├── static ├── classic_archive.zip ├── metadata.en.html ├── metadata.fr.html └── www │ └── index.html ├── .gitignore ├── src ├── locales.ts ├── api │ ├── tools │ │ ├── index.ts │ │ └── archive.ts │ ├── users │ │ ├── callback_twitter.ts │ │ ├── index.ts │ │ ├── delete.ts │ │ ├── credentials.ts │ │ ├── tokens.ts │ │ ├── request.ts │ │ ├── revoke_token.ts │ │ ├── credentials_twitter.ts │ │ └── access.ts │ ├── cloud │ │ ├── index.ts │ │ ├── download.ts │ │ └── upload.ts │ ├── tasks │ │ ├── index.ts │ │ ├── task_stop.ts │ │ ├── task_worker │ │ │ ├── worker.ts │ │ │ └── WorkerTask.ts │ │ ├── task_create.ts │ │ ├── task_infos.ts │ │ ├── task_server.ts │ │ └── Task.ts │ ├── batch │ │ ├── index.ts │ │ ├── dm_media_proxy.ts │ │ ├── tiny_tweets.ts │ │ ├── tweets.ts │ │ └── users.ts │ ├── jwt.ts │ └── index.ts ├── twitter_const.ts ├── interfaces.ts ├── constants.ts ├── logger.ts ├── errors.ts ├── static_server.ts ├── models.ts ├── index.ts ├── uploader.ts ├── cli.ts ├── twitter_lite_clone │ └── twitter_lite.ts └── helpers.ts ├── deploy.sh ├── settings.sample.json ├── publish.sh ├── package.json ├── README.md ├── tsconfig.json └── LICENSE /static/classic_archive.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alkihis/archive-explorer-node/HEAD/static/classic_archive.zip -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .ssh 3 | build 4 | misc 5 | static/old_www 6 | static/www 7 | log.log 8 | logs 9 | deploy/* 10 | settings.json 11 | utils 12 | start.js 13 | .idea 14 | uploads/ 15 | -------------------------------------------------------------------------------- /src/locales.ts: -------------------------------------------------------------------------------- 1 | export default { 2 | fr: 'static/metadata.fr.html', 3 | fr_FR: 'static/metadata.fr.html', 4 | fr_CA: 'static/metadata.fr.html', 5 | en: 'static/metadata.en.html', 6 | 7 | } as { [lang: string]: string }; 8 | -------------------------------------------------------------------------------- /src/api/tools/index.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import BaseArchiveRouter from './archive'; 3 | 4 | const ToolsRouter = Router(); 5 | 6 | ToolsRouter.use('/archive.json', BaseArchiveRouter); 7 | 8 | export default ToolsRouter; 9 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | rm -rf ./static/www/* 2 | 3 | cp -R ../archive-explorer-web/build/* ./static/www 4 | 5 | mkdir deploy 6 | 7 | cp -R build misc .ssh static package.json settings.json deploy/ 8 | 9 | zip -r deploy.zip deploy 10 | 11 | rm -rf deploy 12 | -------------------------------------------------------------------------------- /src/api/users/callback_twitter.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | 3 | const route = Router(); 4 | 5 | route.all('/', (req, res) => { 6 | res.json({ 7 | query: req.query, 8 | body: req.body 9 | }); 10 | }) 11 | 12 | export default route; 13 | -------------------------------------------------------------------------------- /src/api/cloud/index.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import UploadArchiveCloud from './upload'; 3 | import DownloadArchiveCloud from './download'; 4 | 5 | const CloudRouter = Router(); 6 | 7 | CloudRouter.use('/upload', UploadArchiveCloud); 8 | CloudRouter.use('/download', DownloadArchiveCloud); 9 | 10 | export default CloudRouter; 11 | -------------------------------------------------------------------------------- /src/api/tasks/index.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import create_task from './task_create'; 3 | import task_info from './task_infos'; 4 | import task_delete from './task_stop'; 5 | 6 | const route = Router(); 7 | 8 | route.use('/create.json', create_task); 9 | route.use('/details', task_info); 10 | route.use('/destroy', task_delete); 11 | 12 | export default route; 13 | -------------------------------------------------------------------------------- /src/twitter_const.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from "fs"; 2 | import { IS_DEV_MODE } from './index'; 3 | 4 | const config_file = JSON.parse(readFileSync(__dirname + "/../settings.json", "utf-8")); 5 | 6 | export const CONSUMER_KEY = config_file.consumer; 7 | export const CONSUMER_SECRET = config_file.consumer_secret; 8 | export const CALLBACK_URL = () => IS_DEV_MODE ? config_file.oauth_callback_dev : config_file.oauth_callback; 9 | -------------------------------------------------------------------------------- /src/api/tools/archive.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { methodNotAllowed } from "../../helpers"; 3 | import { CLASSIC_ARCHIVE_PATH } from "../../constants"; 4 | 5 | const BaseArchiveRouter = Router(); 6 | 7 | BaseArchiveRouter.get('/', (_, res) => { 8 | res.sendFile(CLASSIC_ARCHIVE_PATH); 9 | }); 10 | 11 | BaseArchiveRouter.all('/', methodNotAllowed('GET')); 12 | 13 | export default BaseArchiveRouter; 14 | -------------------------------------------------------------------------------- /settings.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "consumer": "xxx", 3 | "consumer_secret": "xxx", 4 | "oauth_callback": "oob", 5 | "oauth_callback_dev": "http://localhost:3000/finalize", 6 | "signin_public_key_file": ".ssh/key_new", 7 | "signin_private_key_file": ".ssh/key_new.pem", 8 | "signin_passphrase_key_file": ".ssh/passphrase", 9 | "https_key_directory": "/etc/letsencrypt/live//", 10 | "allowed_users_to_cloud": [] 11 | } 12 | -------------------------------------------------------------------------------- /src/api/batch/index.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import tweets from './tweets'; 3 | import tiny_tweets from './tiny_tweets'; 4 | import users from './users'; 5 | import dmImage from './dm_media_proxy'; 6 | 7 | const route = Router(); 8 | 9 | route.use('/tweets.json', tweets); 10 | route.use('/speed_tweets.json', tiny_tweets); 11 | route.use('/users.json', users); 12 | route.use('/dm_proxy', dmImage); 13 | 14 | export default route; 15 | -------------------------------------------------------------------------------- /src/interfaces.ts: -------------------------------------------------------------------------------- 1 | export interface JSONWebTokenPartial { 2 | /** Issued at */ 3 | iat: string; 4 | /** Expiration (timestamp) */ 5 | exp: string; 6 | /** Issuer */ 7 | iss: string; 8 | /** ID */ 9 | jti: string; 10 | } 11 | 12 | export interface TokenPayload { 13 | user_id: string, 14 | screen_name: string, 15 | login_ip: string 16 | } 17 | 18 | export type JSONWebToken = JSONWebTokenPartial & TokenPayload; 19 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | current=$1 2 | new=$2 3 | 4 | if [ -z "$1" ] 5 | then 6 | echo "Error: No current version supplied" 7 | exit 8 | fi 9 | 10 | if [ -z "$2" ] 11 | then 12 | echo "Error: No new version supplied" 13 | exit 14 | fi 15 | 16 | unzip deploy.zip 17 | mv deploy "$new" 18 | cd "$new" 19 | 20 | cp "../$current/settings.json" . 21 | cp "../$current/misc/deleted_count.json" ./misc 22 | npm i 23 | 24 | echo "Server is ready. Please set the screen" 25 | -------------------------------------------------------------------------------- /src/api/users/index.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import access from './access'; 3 | import request from './request'; 4 | import credentials from './credentials'; 5 | import twitter_cred from './credentials_twitter'; 6 | import tokens from './tokens'; 7 | import delete_user from './delete'; 8 | import revoke_token from './revoke_token'; 9 | 10 | const route = Router(); 11 | 12 | route.use('/request.json', request); 13 | route.use('/access.json', access); 14 | route.use('/credentials.json', credentials); 15 | route.use('/twitter.json', twitter_cred); 16 | route.use('/destroy.json', delete_user); 17 | 18 | const token_route = Router(); 19 | token_route.use('/show.json', tokens); 20 | token_route.use('/revoke.json', revoke_token); 21 | 22 | route.use('/tokens', token_route); 23 | 24 | export default route; 25 | -------------------------------------------------------------------------------- /src/api/users/delete.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { methodNotAllowed, deleteUser } from "../../helpers"; 3 | import logger from "../../logger"; 4 | import AEError, { sendError } from "../../errors"; 5 | import Task from "../tasks/Task"; 6 | 7 | const route = Router(); 8 | 9 | route.post('/', (req, res) => { 10 | // WARNING: THIS WILL COMPLETELY DELETE USER 11 | // AND INVALIDATE ALL HIS TOKENS 12 | 13 | // Cancel every task from user 14 | const tasks_of_user = Task.tasksOf(req.user!.user_id); 15 | 16 | for (const task of tasks_of_user) { 17 | task.cancel(); 18 | } 19 | 20 | // Delete user 21 | deleteUser(req.user!.user_id) 22 | .then(() => { 23 | res.json({ status: true }); 24 | }) 25 | .catch(e => { 26 | logger.error("Unable to delete user:", e); 27 | sendError(AEError.server_error, res); 28 | }); 29 | }); 30 | 31 | route.all('/', methodNotAllowed('POST')); 32 | 33 | export default route; 34 | -------------------------------------------------------------------------------- /src/api/jwt.ts: -------------------------------------------------------------------------------- 1 | import jwt from 'express-jwt'; 2 | import { isTokenInvalid } from '../helpers'; 3 | import { SECRET_PUBLIC_KEY } from '../constants'; 4 | import logger from '../logger'; 5 | 6 | export default jwt({ 7 | algorithms: ['RS256'], 8 | getToken: function fromHeaderOrQuerystring(req) { 9 | if (req.headers.authorization && req.headers.authorization.split(' ')[0] === 'Bearer') { 10 | return req.headers.authorization.split(' ')[1]; 11 | } else if (req.cookies && req.cookies.login_token) { 12 | return req.cookies.login_token; 13 | } 14 | return null; 15 | }, 16 | secret: SECRET_PUBLIC_KEY, 17 | credentialsRequired: true, 18 | isRevoked: (req, payload, done) => { 19 | isTokenInvalid(payload.jti, req, payload) 20 | .then(is_revoked => { done(null, is_revoked); }) 21 | .catch(e => { logger.error("Unable to check token validity", e); done(e); }); 22 | } 23 | }).unless( 24 | { path: ["/api/users/request.json", "/api/users/access.json", "/api/callback_twitter", "/api", "/api/deleted_count.json"] } 25 | ); 26 | -------------------------------------------------------------------------------- /src/api/users/credentials.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { getCompleteUserFromId, sanitizeMongoObj, methodNotAllowed } from "../../helpers"; 3 | import logger from "../../logger"; 4 | import AEError, { sendError } from "../../errors"; 5 | import { CONFIG_FILE } from "../../constants"; 6 | 7 | const route = Router(); 8 | 9 | route.get('/', (req, res) => { 10 | // Retourne des infos sur l'utilisateur connecté 11 | const user = getCompleteUserFromId(req.user!.user_id); 12 | 13 | user 14 | .then(u => { 15 | if (u) { 16 | const s_user = sanitizeMongoObj(u); 17 | s_user.can_cloud = CONFIG_FILE.allowed_users_to_cloud.includes(u.user_id); 18 | res.json(s_user); 19 | } 20 | else { 21 | sendError(AEError.forbidden, res); 22 | } 23 | }) 24 | .catch(e => { 25 | logger.error("Error while fetching user:", e); 26 | sendError(AEError.server_error, res); 27 | }); 28 | }); 29 | 30 | route.all('/', methodNotAllowed('GET')); 31 | 32 | export default route; 33 | -------------------------------------------------------------------------------- /src/api/users/tokens.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { getTokensFromUser, sanitizeMongoObj, methodNotAllowed } from "../../helpers"; 3 | import logger from "../../logger"; 4 | import AEError, { sendError } from "../../errors"; 5 | import { IToken } from "../../models"; 6 | 7 | const route = Router(); 8 | 9 | route.get('/', (req, res) => { 10 | // Retourne tous les tokens de l'utilisateur actuellement connecté 11 | const tokens = getTokensFromUser(req.user!.user_id); 12 | 13 | tokens 14 | .then(u => { 15 | const list: IToken[] = u.map(e => sanitizeMongoObj(e)); 16 | 17 | for (const e of list) { 18 | if (e.token === req.user!.jti) { 19 | // @ts-ignore 20 | e.current = true; 21 | } 22 | } 23 | 24 | res.json(list); 25 | }) 26 | .catch(e => { 27 | logger.error("Error while fetching tokens:", e); 28 | sendError(AEError.server_error, res); 29 | }); 30 | }); 31 | 32 | route.all('/', methodNotAllowed('GET')); 33 | 34 | export default route; 35 | -------------------------------------------------------------------------------- /src/api/users/request.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import twitter from '../../twitter_lite_clone/twitter_lite'; 3 | import { CONSUMER_KEY, CONSUMER_SECRET, CALLBACK_URL } from "../../twitter_const"; 4 | import AEError, { sendError } from "../../errors"; 5 | import { methodNotAllowed } from "../../helpers"; 6 | 7 | // Ask a request token 8 | 9 | const route = Router(); 10 | 11 | route.post('/', (_, res) => { 12 | // req.user will not be accessible here 13 | // Generating twitter button for client 14 | const data = (new twitter({ 15 | consumer_key: CONSUMER_KEY, 16 | consumer_secret: CONSUMER_SECRET 17 | })).getRequestToken(CALLBACK_URL()); 18 | 19 | data.then(data => { 20 | res.json({ 21 | oauth_token: data.oauth_token, 22 | oauth_token_secret: data.oauth_token_secret, 23 | url: 'https://api.twitter.com/oauth/authenticate?oauth_token=' + data.oauth_token 24 | }); 25 | }) 26 | .catch(() => { 27 | sendError(AEError.server_error, res); 28 | }); 29 | }); 30 | 31 | route.all('/', methodNotAllowed('POST')); 32 | 33 | export default route; 34 | -------------------------------------------------------------------------------- /static/metadata.en.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 11 | 12 | 13 | 14 | 15 | 16 | 20 | 21 | 22 | 23 | 27 | 31 | -------------------------------------------------------------------------------- /src/api/users/revoke_token.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { invalidateToken, methodNotAllowed, getTokenInstanceFromString } from "../../helpers"; 3 | import logger from "../../logger"; 4 | import AEError, { sendError } from "../../errors"; 5 | 6 | const route = Router(); 7 | 8 | route.post('/', (req, res) => { 9 | const token = req.body && req.body.token ? req.body.token : req.user!.jti; 10 | 11 | // Checking the authenticity of desired token 12 | (async () => { 13 | const full_token = await getTokenInstanceFromString(token); 14 | 15 | if (!full_token) { 16 | sendError(AEError.inexistant, res); 17 | return; 18 | } 19 | 20 | if (full_token.user_id !== req.user!.user_id) { 21 | sendError(AEError.forbidden, res); 22 | return; 23 | } 24 | 25 | const resp = await invalidateToken(token); 26 | 27 | if (resp.ok) { 28 | logger.debug(`Token ${token} revoked.`); 29 | 30 | res.json({ status: true }); 31 | } 32 | else { 33 | sendError(AEError.inexistant, res); 34 | } 35 | })().catch(e => { 36 | sendError(AEError.server_error, res); 37 | logger.error("Server error", e); 38 | }); 39 | }); 40 | 41 | route.all('/', methodNotAllowed('POST')); 42 | 43 | export default route; 44 | -------------------------------------------------------------------------------- /static/metadata.fr.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 11 | 12 | 13 | 14 | 15 | 16 | 20 | 21 | 22 | 23 | 27 | -------------------------------------------------------------------------------- /src/api/tasks/task_stop.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import AEError, { sendError } from '../../errors'; 3 | import { methodNotAllowed } from '../../helpers'; 4 | import Task from './Task'; 5 | 6 | const route = Router(); 7 | 8 | // Arrêter toutes les tâches 9 | route.post('/all.json', (req, res) => { 10 | // Récupérer user ID via token 11 | const user_id = req.user!.user_id; 12 | 13 | const user_tasks = Task.tasksOf(user_id); 14 | 15 | // Si l'utilisateur a des tâches 16 | for (const t of user_tasks) { 17 | t.cancel(); 18 | } 19 | 20 | res.send(); 21 | }); 22 | 23 | route.all('/all.json', methodNotAllowed('GET')); 24 | 25 | // Arrêter une tâche par ID 26 | route.post('/:id.json', (req, res) => { 27 | if (req.params.id) { 28 | const user_id = req.user!.user_id; 29 | 30 | try { 31 | var id = BigInt(req.params.id); 32 | } catch (e) { 33 | sendError(AEError.invalid_data, res); 34 | return; 35 | } 36 | 37 | // Recherche si la tâche existe 38 | const task = Task.get(id); 39 | 40 | if (!task) { 41 | sendError(AEError.inexistant, res); 42 | return; 43 | } 44 | 45 | if (task!.owner !== user_id) { 46 | sendError(AEError.forbidden, res); 47 | return; 48 | } 49 | 50 | task!.cancel(); 51 | 52 | res.json(); 53 | } 54 | else { 55 | sendError(AEError.invalid_data, res); 56 | } 57 | }); 58 | 59 | route.all('/:id.json', methodNotAllowed('POST')); 60 | 61 | export default route; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "archive-explorer-server", 3 | "version": "1.1.0", 4 | "description": "", 5 | "main": "index.js", 6 | "license": "CC-BY-NC-SA-4.0", 7 | "repository": "https://github.com/alkihis/archive-explorer-node", 8 | "bugs": "https://github.com/alkihis/archive-explorer-node/issues", 9 | "scripts": { 10 | "test": "echo \"Error: no test specified\" && exit 1", 11 | "start-dev": "node start.js", 12 | "deploy": "./deploy.sh" 13 | }, 14 | "keywords": [], 15 | "author": "", 16 | "dependencies": { 17 | "@types/cors": "^2.8.9", 18 | "@types/express": "^4.17.11", 19 | "@types/express-jwt": "0.0.42", 20 | "@types/jsonwebtoken": "^8.5.0", 21 | "@types/mongoose": "^5.10.3", 22 | "@types/node": "^13.13.40", 23 | "@types/socket.io": "^2.1.12", 24 | "@types/spdy": "^3.4.4", 25 | "@types/uuid": "^3.4.9", 26 | "body-parser": "^1.19.0", 27 | "commander": "^3.0.2", 28 | "cookie-parser": "^1.4.5", 29 | "cors": "^2.8.5", 30 | "express": "^4.17.1", 31 | "express-jwt": "^6.0.0", 32 | "express-mongo-sanitize": "^1.3.2", 33 | "interactive-cli-helper": "^2.0.3", 34 | "jsonwebtoken": "^8.5.1", 35 | "md5-file": "^5.0.0", 36 | "mongoose": "^5.11.13", 37 | "multer": "^1.4.2", 38 | "socket.io": "^2.4.1", 39 | "timerize": "^1.0.3", 40 | "twitter-d": "^0.4.0", 41 | "twitter-lite": "^0.9.4", 42 | "uuid": "^3.4.0", 43 | "winston": "^3.3.3" 44 | }, 45 | "devDependencies": { 46 | "@types/cookie-parser": "^1.4.2", 47 | "@types/multer": "^1.4.5", 48 | "typescript": "^4.1.3" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/api/tasks/task_worker/worker.ts: -------------------------------------------------------------------------------- 1 | import { parentPort } from 'worker_threads'; 2 | import WorkerTaskMaker, { WorkerTask, TaskJobs } from './WorkerTask'; 3 | 4 | const DEBUG = false; 5 | let task: WorkerTaskMaker; 6 | 7 | parentPort!.on('message', (data: WorkerTask) => { 8 | if (data.type === "task") { 9 | // console.log("New task on worker of type", data.task_type); 10 | 11 | const maker = TaskJobs[data.task_type]; 12 | 13 | // If we have a job available for this kind of task 14 | if (maker) { 15 | task = new WorkerTaskMaker(data.tweets, data.credentials, maker, parentPort!); 16 | if (typeof data.debug !== 'undefined') { 17 | task.debug_mode = data.debug; 18 | } 19 | else { 20 | task.debug_mode = DEBUG; 21 | } 22 | 23 | task.start() 24 | .then(() => { 25 | // console.log("Worker task end"); 26 | parentPort!.postMessage({ type: "end" }); 27 | }) 28 | .catch(e => { 29 | console.error("Worker task end with error", e); 30 | parentPort!.postMessage({ type: "error", error: e }); 31 | }); 32 | } 33 | else { 34 | console.error("Worker task end with error", "Unexpected task type"); 35 | parentPort!.postMessage({ type: "error", error: "Unexpected task type" }); 36 | } 37 | } 38 | else if (data.type === "stop") { 39 | // console.log("Request worker end"); 40 | if (task) { 41 | task.stop(); 42 | } 43 | } 44 | }); 45 | 46 | -------------------------------------------------------------------------------- /src/api/users/credentials_twitter.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { getCompleteUserFromId, sanitizeMongoObj, methodNotAllowed, sendTwitterError } from "../../helpers"; 3 | import logger from "../../logger"; 4 | import AEError, { sendError } from "../../errors"; 5 | import Twitter from '../../twitter_lite_clone/twitter_lite'; 6 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 7 | 8 | const route = Router(); 9 | 10 | route.get('/', (req, res) => { 11 | // Retourne des infos sur l'utilisateur connecté 12 | const user = getCompleteUserFromId(req.user!.user_id); 13 | 14 | (async () => { 15 | const u = await user; 16 | 17 | if (u) { 18 | // Check Twitter credentials 19 | const twi = new Twitter({ 20 | consumer_key: CONSUMER_KEY, 21 | consumer_secret: CONSUMER_SECRET, 22 | access_token_key: u.oauth_token, 23 | access_token_secret: u.oauth_token_secret 24 | }); 25 | 26 | try { 27 | const resp = await twi.get('account/verify_credentials'); 28 | delete resp._headers; 29 | res.json({ user: sanitizeMongoObj(u), twitter: resp }); 30 | } catch (e) { 31 | sendTwitterError(e, res); 32 | } 33 | } 34 | else { 35 | sendError(AEError.forbidden, res); 36 | } 37 | })().catch(e => { 38 | logger.error("Error while fetching user:", e); 39 | sendError(AEError.server_error, res); 40 | }); 41 | 42 | 43 | }); 44 | 45 | route.all('/', methodNotAllowed('GET')); 46 | 47 | export default route; 48 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync, writeFileSync } from "fs"; 2 | import logger from "./logger"; 3 | import path from 'path'; 4 | 5 | export const VERSION = "1.8.0"; 6 | export const CONFIG_FILE: any = {} 7 | export const CLASSIC_ARCHIVE_PATH = path.resolve(__dirname + "/../static/classic_archive.zip"); 8 | 9 | export function reloadSettings() { 10 | const settings = JSON.parse(readFileSync(__dirname + "/../settings.json", "utf-8")); 11 | 12 | for (const key in settings) { 13 | if (settings.hasOwnProperty(key)) { 14 | CONFIG_FILE[key] = settings[key]; 15 | } 16 | } 17 | } 18 | 19 | reloadSettings(); 20 | 21 | export const TweetCounter = new class { 22 | protected count_file: { deleted: number }; 23 | protected filename = __dirname + "/../misc/deleted_count.json"; 24 | protected timer: NodeJS.Timeout | undefined; 25 | 26 | constructor() { 27 | try { 28 | this.count_file = JSON.parse(readFileSync(this.filename, "utf-8")); 29 | } catch (e) { 30 | logger.info("Count file does not exists, creating misc/" + this.filename + "..."); 31 | this.count_file = { deleted: 0 }; 32 | } 33 | } 34 | 35 | inc(of_amount: number = 1) { 36 | this.count_file.deleted += of_amount; 37 | 38 | if (this.timer) { 39 | clearTimeout(this.timer); 40 | } 41 | 42 | this.timer = setTimeout(() => this.sync(), 2500); 43 | } 44 | 45 | get count() { 46 | return this.count_file.deleted; 47 | } 48 | 49 | sync() { 50 | writeFileSync(this.filename, JSON.stringify(this.count_file)); 51 | 52 | if (this.timer) 53 | clearTimeout(this.timer); 54 | 55 | this.timer = undefined; 56 | } 57 | }; 58 | 59 | const passphrase_file = __dirname + "/../" + CONFIG_FILE.signin_passphrase_key_file; 60 | let real_passphrase: string = ""; 61 | // Try to get passphrase 62 | try { 63 | real_passphrase = readFileSync(passphrase_file, "utf-8").trimRight(); 64 | } catch {} 65 | 66 | export const SECRET_PUBLIC_KEY = readFileSync(__dirname + "/../" + CONFIG_FILE.signin_public_key_file, "utf-8"); 67 | export const SECRET_PRIVATE_KEY = readFileSync(__dirname + "/../" + CONFIG_FILE.signin_private_key_file, "utf-8"); 68 | export const SECRET_PASSPHRASE = real_passphrase; 69 | export const MAX_TASK_PER_USER = 3; 70 | export const MAX_TASK_PER_USER_SPECIAL = 10; 71 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import Winston from 'winston'; 2 | 3 | /* 4 | * This is the logger module using winston package. Redirecting some logs into the standard output (Console). 5 | * Setting up a log level need to be implemented before uses logs. 6 | * Use the #levelMin variable to set up the minimum log level that will be used in the entire program. 7 | * The default value of the log level is 'INFO'. 8 | * Require this module with: 9 | * import win = require('./lib/logger'); 10 | * 11 | * Using examples: 12 | * - win.logger.log('CRITICAL', ) - Higher level of logger, critical error 13 | * - win.logger.log('ERROR', ) - Second level of logger, error 14 | * - win.logger.log('WARNING', ) - Third level of logger, warning message 15 | * - win.logger.log('SUCCESS', ) - 4th level of logger, success message 16 | * - win.logger.log('INFO', ) - 5th level of logger, info message 17 | * - win.logger.log('DEBUG', ) - Lower level of logger, debug mode 18 | */ 19 | const myCustomLevels = { 20 | levels: { 21 | fatal: 0, 22 | error: 1, 23 | warn: 2, 24 | success: 3, 25 | info: 4, 26 | verbose: 5, 27 | debug: 6, 28 | silly: 7, 29 | }, 30 | colors: { 31 | fatal: 'red', 32 | error: 'magenta', 33 | warn: 'yellow', 34 | success: 'green', 35 | info: 'cyan', 36 | verbose: 'grey', 37 | debug: 'blue', 38 | silly: 'white' 39 | } 40 | }; 41 | // See winston format API at https://github.com/winstonjs/logform 42 | export const FORMAT_CONSOLE = Winston.format.combine( 43 | Winston.format.colorize(), 44 | Winston.format.timestamp(), 45 | Winston.format.printf(info => `[${info.timestamp.split('T', 2).join(' ').split('Z')[0]}] ${info.level}: ${info.message}`) 46 | ); 47 | 48 | export const FORMAT_FILE = Winston.format.combine( 49 | Winston.format.timestamp(), 50 | Winston.format.printf(info => `[${info.timestamp.split('T', 2).join(' ').split('Z')[0]}] ${info.level}: ${info.message}`) 51 | ); 52 | 53 | export const logger = Winston.createLogger({ 54 | levels: myCustomLevels.levels, 55 | transports: [new Winston.transports.Console({ 56 | stderrLevels : ['fatal', 'error', 'warn'], 57 | format: FORMAT_CONSOLE 58 | })] 59 | }); 60 | 61 | Winston.addColors(myCustomLevels.colors); 62 | 63 | export default logger; 64 | -------------------------------------------------------------------------------- /src/api/tasks/task_create.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import AEError, { sendError } from '../../errors'; 3 | import Task, { isValidTaskType } from './Task'; 4 | import { methodNotAllowed, getCompleteUserFromId } from '../../helpers'; 5 | import logger from '../../logger'; 6 | import { MAX_TASK_PER_USER, MAX_TASK_PER_USER_SPECIAL } from '../../constants'; 7 | 8 | const route = Router(); 9 | 10 | route.post('/', (req, res) => { 11 | (async () => { 12 | const user = await getCompleteUserFromId(req.user!.user_id); 13 | 14 | if (!user) { 15 | sendError(AEError.forbidden, res); 16 | return; 17 | } 18 | 19 | const tasks = Task.tasksOf(user.user_id); 20 | 21 | if (user.special) { 22 | // Special allow a derogation to enable more active tasks 23 | if (tasks.size >= MAX_TASK_PER_USER_SPECIAL) { 24 | sendError(AEError.too_many_tasks, res); 25 | return; 26 | } 27 | } 28 | else { 29 | // Classic user 30 | if (tasks.size >= MAX_TASK_PER_USER) { 31 | sendError(AEError.too_many_tasks, res); 32 | return; 33 | } 34 | } 35 | 36 | // IDs are in req.body.ids, splitted by comma 37 | if (req.body && req.body.ids && typeof req.body.ids === 'string' && req.body.type && isValidTaskType(req.body.type)) { 38 | const ids = (req.body.ids as string).split(','); 39 | 40 | if (ids.length) { 41 | // Création tâche (elle s'enregistre correctement automatiquement) 42 | const task = new Task(ids, { 43 | user_id: user.user_id, 44 | oauth_token: user.oauth_token, 45 | oauth_token_secret: user.oauth_token_secret, 46 | screen_name: user.twitter_screen_name, 47 | }, req.body.type); 48 | 49 | res.json({ status: true, task: String(task.id) }); 50 | } 51 | else { 52 | sendError(AEError.invalid_data, res); 53 | } 54 | } 55 | else { 56 | sendError(AEError.invalid_data, res); 57 | } 58 | })().catch(e => { 59 | logger.error('Error:', e); 60 | sendError(AEError.server_error, res); 61 | }); 62 | }); 63 | 64 | route.all('/', methodNotAllowed('POST')); 65 | 66 | export default route; 67 | -------------------------------------------------------------------------------- /src/errors.ts: -------------------------------------------------------------------------------- 1 | import { Response } from "express"; 2 | 3 | // Specify here the error internal name. 4 | // A code will be automatically given (numeric order, the first is 1). 5 | enum AEError { 6 | inexistant = 1, invalid_route, server_error, invalid_data, invalid_request, 7 | forbidden, invalid_token, invalid_verifier, invalid_method, twitter_error, 8 | twitter_credentials_expired, twitter_rate_limit, too_many_tasks, size_mismatch, 9 | too_many_chunks, 10 | }; 11 | 12 | // Specify here the corresponding error message and HTTP code for the error 13 | const errors: { [errorCode: string]: [string, number] } = { 14 | [AEError.inexistant]: ["The page or desired document can't be found", 404], 15 | [AEError.invalid_route]: ["Specified route does not exists", 404], 16 | [AEError.server_error]: ["Internal server error", 500], 17 | [AEError.invalid_data]: ["Provided data is invalid", 400], 18 | [AEError.invalid_request]: ["Query or URL is invalid", 400], 19 | [AEError.forbidden]: ["Your current credentials does not allow you to access requested document or endpoint", 403], 20 | [AEError.invalid_token]: ["Invalid or inexistant token, that is required to access this resource", 403], 21 | [AEError.invalid_verifier]: ["OAuth verifier is invalid, please renew your request with valid credentials", 400], 22 | [AEError.invalid_method]: ["Invalid HTTP method", 405], 23 | [AEError.twitter_error]: ["Twitter error. See .error for explicit details", 400], 24 | [AEError.twitter_credentials_expired]: ["Twitter tokens has expired or has been revoked. Please log in again.", 403], 25 | [AEError.twitter_rate_limit]: ["Twitter send a too many requests error. Please try again later.", 429], 26 | [AEError.too_many_tasks]: ["Too many tasks are already started for you. Try again later.", 429], 27 | [AEError.size_mismatch]: ["Sent chunks size doesn't match size originally given.", 400], 28 | [AEError.too_many_chunks]: ["You already sent too many chunks.", 400], 29 | }; 30 | 31 | export default AEError; 32 | 33 | export function sendError(code: AEError, res: Response, custom_error?: any) { 34 | if (String(code) in errors) { 35 | const e = { 36 | code, message: errors[code][0] 37 | }; 38 | 39 | res.status(errors[code][1]).json(custom_error ? {...e, error: custom_error} : e); 40 | } 41 | else { 42 | sendError(AEError.server_error, res); 43 | throw new Error(`Invalid error code: ${code}.`); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/api/tasks/task_infos.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import AEError, { sendError } from '../../errors'; 3 | import { methodNotAllowed } from '../../helpers'; 4 | import Task, { isValidTaskType } from './Task'; 5 | 6 | const route = Router(); 7 | 8 | route.get('/all.json', (req, res) => { 9 | // récupérer user ID via token 10 | const user_id = req.user!.user_id; 11 | 12 | const tasks = Task.tasksOf(user_id); 13 | 14 | // Répertorie toutes les tâches de l'utilisateur et renvoie leur progression actuelle 15 | res.json([...tasks].map(t => t!.current_progression)); 16 | }); 17 | 18 | route.all('/all.json', methodNotAllowed('GET')); 19 | 20 | route.get('/type/:type.json', (req, res) => { 21 | if (req.params.type) { 22 | const type: string = req.params.type; 23 | 24 | if (isValidTaskType(type)) { 25 | const user_id = req.user!.user_id; 26 | const tasks = Task.typeOf(type, user_id); 27 | 28 | res.json([...tasks].map(t => t!.current_progression)); 29 | } 30 | else { 31 | sendError(AEError.invalid_data, res); 32 | } 33 | } 34 | else { 35 | sendError(AEError.invalid_request, res); 36 | } 37 | 38 | // récupérer user ID via token 39 | const user_id = req.user!.user_id; 40 | 41 | const tasks = Task.tasksOf(user_id); 42 | 43 | // Répertorie toutes les tâches de l'utilisateur et renvoie leur progression actuelle 44 | res.json([...tasks].map(t => t!.current_progression)); 45 | }); 46 | 47 | route.all('/type/:type.json', methodNotAllowed('GET')); 48 | 49 | route.get('/:id.json', (req, res) => { 50 | if (req.params.id) { 51 | // récupérer user ID via token 52 | const user_id = req.user!.user_id; 53 | 54 | // Recherche la tâche :id 55 | try { 56 | var id = BigInt(req.params.id); 57 | } catch (e) { 58 | // Invalid conversation 59 | sendError(AEError.invalid_request, res); 60 | return; 61 | } 62 | 63 | const task = Task.get(id); 64 | 65 | if (task) { 66 | if (task.owner === user_id) { 67 | res.json(task.current_progression); 68 | } 69 | else { 70 | sendError(AEError.forbidden, res); 71 | } 72 | } 73 | else { 74 | sendError(AEError.inexistant, res); 75 | } 76 | } 77 | else { 78 | sendError(AEError.invalid_request, res); 79 | } 80 | }); 81 | 82 | route.all('/:id.json', methodNotAllowed('GET')); 83 | 84 | export default route; -------------------------------------------------------------------------------- /src/api/index.ts: -------------------------------------------------------------------------------- 1 | import express, { Router } from 'express'; 2 | import AEError, { sendError } from '../errors'; 3 | import task_route from './tasks/index'; 4 | import users_route from './users/index'; 5 | import batch_route from './batch/index'; 6 | import { TweetCounter } from '../constants'; 7 | import { JSONWebToken } from '../interfaces'; 8 | import bodyParser from 'body-parser'; 9 | import logger from '../logger'; 10 | import cookieParser from 'cookie-parser'; 11 | import jwt from './jwt'; 12 | import ToolsRouter from './tools'; 13 | import CloudRouter from './cloud'; 14 | 15 | const route = Router(); 16 | 17 | route.use(cookieParser()); 18 | 19 | // Declare jwt use 20 | route.use(jwt); 21 | 22 | route.use((req, res, next) => { 23 | if (req.__ask_refresh__) { 24 | res.setHeader('X-Upgrade-Token', req.__ask_refresh__); 25 | delete req.__ask_refresh__; 26 | } 27 | next(); 28 | }); 29 | 30 | // Extends Express request 31 | declare module 'express-serve-static-core' { 32 | interface Request { 33 | user?: JSONWebToken; 34 | __ask_refresh__?: string; 35 | } 36 | } 37 | 38 | // parse application/x-www-form-urlencoded 39 | route.use(bodyParser.urlencoded({ extended: true })); 40 | 41 | // parse application/json 42 | route.use(bodyParser.json({ limit: "50mb" })); 43 | 44 | // Pas de type system (vieux paquet) 45 | const mongoSanitize = require('express-mongo-sanitize'); 46 | 47 | // Or, to replace prohibited characters with _, use: 48 | route.use(mongoSanitize({ 49 | replaceWith: '_' 50 | })); 51 | 52 | // Defining routers 53 | route.use('/tasks', task_route); 54 | route.use('/users', users_route); 55 | route.use('/batch', batch_route); 56 | route.use('/tools', ToolsRouter); 57 | route.use('/cloud', CloudRouter); 58 | route.get('/deleted_count.json', (_, res) => { 59 | res.json({ count: TweetCounter.count }); 60 | }); 61 | 62 | route.all('/', (_, res) => { 63 | sendError(AEError.invalid_route, res); 64 | }); 65 | 66 | // Catch JWT erros 67 | // Can't be used in router, must be declared in top-level 68 | export function apiErrors(err: any, _: express.Request, res: express.Response, next: Function) { 69 | logger.debug("An error occurred: " + err.name); 70 | logger.verbose(err.stack); 71 | 72 | if (err.name === 'UnauthorizedError') { 73 | logger.debug("Token identification error: " + err.name); 74 | sendError(AEError.invalid_token, res); 75 | } 76 | else { 77 | next(err); 78 | } 79 | } 80 | 81 | // Catch all API invalid routes 82 | route.use((_, res) => { 83 | sendError(AEError.inexistant, res); 84 | }); 85 | 86 | export default route; 87 | -------------------------------------------------------------------------------- /src/api/cloud/download.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import { deleteCloudedArchive, methodNotAllowed } from "../../helpers"; 3 | import AEError, { sendError } from '../../errors'; 4 | import logger from '../../logger'; 5 | import { CloudedArchiveModel, ICloudedArchive } from '../../models'; 6 | import * as fs from 'fs'; 7 | import { UPLOAD_PATH } from "../../uploader"; 8 | 9 | const DownloadArchiveCloud = Router(); 10 | 11 | DownloadArchiveCloud.get('/archive/:file_id', (req, res) => { 12 | const file_id = String(req.params.file_id); 13 | 14 | if (!file_id) { 15 | return sendError(AEError.invalid_data, res); 16 | } 17 | 18 | (async () => { 19 | const archive_info = await CloudedArchiveModel.findOne({ file_id }) as ICloudedArchive; 20 | 21 | if (!archive_info || archive_info.user_id !== req.user!.user_id) { 22 | return sendError(AEError.inexistant, res); 23 | } 24 | 25 | const filepath = UPLOAD_PATH + '/' + archive_info.path; 26 | const filename = archive_info.filename; 27 | 28 | res.setHeader('Content-Type', 'application/zip'); 29 | res.attachment(filename); 30 | fs.createReadStream(filepath, { autoClose: true }).pipe(res); 31 | })().catch(e => { 32 | sendError(AEError.server_error, res); 33 | logger.error("Server error", e); 34 | }); 35 | }); 36 | 37 | DownloadArchiveCloud.all('/archive/:file_id', methodNotAllowed('GET')); 38 | 39 | DownloadArchiveCloud.get('/list.json', (req, res) => { 40 | (async () => { 41 | const files = await CloudedArchiveModel.find({ user_id: req.user!.user_id }) as ICloudedArchive[]; 42 | 43 | res.json({ 44 | files: files.map(f => ({ 45 | id: f.file_id, 46 | name: f.filename, 47 | info: f.info, 48 | date: f.date.toISOString(), 49 | })), 50 | }); 51 | })().catch(e => { 52 | sendError(AEError.server_error, res); 53 | logger.error("Server error", e); 54 | }); 55 | }); 56 | 57 | DownloadArchiveCloud.all('/list.json', methodNotAllowed('GET')); 58 | 59 | DownloadArchiveCloud.delete('/destroy/:file_id', (req, res) => { 60 | (async () => { 61 | const file = await CloudedArchiveModel.findOne({ 62 | user_id: req.user!.user_id, 63 | file_id: req.params.file_id, 64 | }) as ICloudedArchive; 65 | 66 | if (file) { 67 | await deleteCloudedArchive(file); 68 | } 69 | 70 | res.send(); 71 | })().catch(e => { 72 | sendError(AEError.server_error, res); 73 | logger.error("Server error", e); 74 | }); 75 | }); 76 | 77 | DownloadArchiveCloud.all('/destroy/:file_id', methodNotAllowed('DELETE')); 78 | 79 | export default DownloadArchiveCloud; 80 | -------------------------------------------------------------------------------- /src/api/batch/dm_media_proxy.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import AEError, { sendError } from "../../errors"; 3 | import { getCompleteUserFromId, methodNotAllowed } from "../../helpers"; 4 | import Twitter from '../../twitter_lite_clone/twitter_lite'; 5 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 6 | import logger from "../../logger"; 7 | import { Request, Response } from 'express'; 8 | 9 | //// PROXY A DM IMAGE 10 | 11 | const route = Router(); 12 | 13 | async function getImageFromUrl(url: string, req: Request, res: Response) { 14 | if (!url.startsWith('https://ton.twitter.com/dm/')) { 15 | sendError(AEError.invalid_request, res); 16 | return; 17 | } 18 | 19 | const bdd_user = await getCompleteUserFromId(req.user!.user_id); 20 | 21 | // If user does not exists 22 | if (!bdd_user) { 23 | sendError(AEError.invalid_token, res); 24 | return; 25 | } 26 | 27 | // Create Twitter object with credentials 28 | const user = new Twitter({ 29 | consumer_key: CONSUMER_KEY, 30 | consumer_secret: CONSUMER_SECRET, 31 | access_token_key: bdd_user.oauth_token, 32 | access_token_secret: bdd_user.oauth_token_secret 33 | }); 34 | 35 | // Send response 36 | const resp = await user.dmImage(url); 37 | 38 | if (!resp.ok) { 39 | if (resp.status === 404) { 40 | sendError(AEError.inexistant, res); 41 | } 42 | else { 43 | sendError(AEError.twitter_error, res); 44 | } 45 | return; 46 | } 47 | 48 | const headers_to_copy = ['Content-Type', 'Transfer-Encoding']; 49 | for (const header of headers_to_copy) { 50 | if (resp.headers.has(header)) { 51 | res.setHeader(header, resp.headers.get(header) as string); 52 | } 53 | } 54 | 55 | const buffer = await resp.arrayBuffer(); 56 | res.send(Buffer.from(buffer)); 57 | } 58 | 59 | route.get('/', (req, res) => { 60 | // Download a single image from Twitter 61 | if (req.user && req.query && req.query.url) { 62 | const url: string = req.query.url as string; 63 | 64 | // Get URL from Twitter 65 | getImageFromUrl(url, req, res).catch(e => { 66 | logger.error("Fetch error", e); 67 | sendError(AEError.server_error, res); 68 | }); 69 | } 70 | else { 71 | sendError(AEError.invalid_request, res); 72 | } 73 | }); 74 | 75 | route.post('/', (req, res) => { 76 | // Download a single image from Twitter 77 | if (req.user && req.body && req.body.url) { 78 | const url: string = req.body.url; 79 | 80 | // Get URL from Twitter 81 | getImageFromUrl(url, req, res).catch(e => { 82 | logger.error("Fetch error", e); 83 | sendError(AEError.server_error, res); 84 | }); 85 | } 86 | else { 87 | sendError(AEError.invalid_request, res); 88 | } 89 | }); 90 | 91 | route.all('/', methodNotAllowed(['GET', 'POST'])); 92 | 93 | export default route; 94 | -------------------------------------------------------------------------------- /src/static_server.ts: -------------------------------------------------------------------------------- 1 | import express, { Router } from "express"; 2 | import path from 'path'; 3 | import Locales from './locales'; 4 | import { readFile } from "fs"; 5 | import logger from "./logger"; 6 | 7 | const router = Router(); 8 | 9 | let CACHE_FOR_LANGS: any = {}; 10 | 11 | export function clearCache() { 12 | CACHE_FOR_LANGS = {}; 13 | } 14 | 15 | function serveHtmlMiddleware(req: express.Request, res: express.Response, next: express.NextFunction) { 16 | const lang = req.acceptsLanguages('fr', 'fr_FR', 'fr_CA', 'en'); 17 | 18 | if (lang && !lang.includes("en") && lang in Locales) { 19 | const filename_changer = Locales[lang]; 20 | 21 | if (filename_changer in CACHE_FOR_LANGS) { 22 | logger.debug("Serving page for lang " + lang + " from cache"); 23 | res.send(CACHE_FOR_LANGS[filename_changer]); 24 | return; 25 | } 26 | 27 | const change = path.join(__dirname, "..", filename_changer); 28 | const original = path.join(__dirname, "../static/www/index.html"); 29 | 30 | const original_string = new Promise((resolve, reject) => { 31 | readFile(original, (err, data) => { 32 | if (err) { 33 | reject(err); 34 | return; 35 | } 36 | 37 | resolve(data.toString('utf-8')); 38 | }); 39 | }) as Promise; 40 | 41 | const replacement_string = new Promise((resolve, reject) => { 42 | readFile(change, (err, data) => { 43 | if (err) { 44 | reject(err); 45 | return; 46 | } 47 | 48 | resolve(data.toString('utf-8')); 49 | }); 50 | }) as Promise; 51 | 52 | const replacement_regex = new RegExp(`(.+)`, 's') 53 | 54 | // Envoie un fichier remplacé 55 | Promise.all([original_string, replacement_string]) 56 | .then(d => { 57 | logger.debug("Serving page for lang " + lang); 58 | let real_lang = lang; 59 | if (lang && lang.includes('_')) { 60 | real_lang = lang.split('_')[0]; 61 | } 62 | 63 | res.send( 64 | CACHE_FOR_LANGS[filename_changer] = d[0].replace(replacement_regex, d[1]).replace('lang="en"', `lang="${real_lang}"`) 65 | ); 66 | }) 67 | .catch(err => next(err)); 68 | } 69 | else { 70 | // Aucune modification nécessaire 71 | res.sendFile(path.join(__dirname, "../static/www/index.html")); 72 | } 73 | } 74 | 75 | const static_serve_middleware = express.static(path.join(__dirname, "../static/www")); 76 | 77 | router.use('/', (req, res, next) => { 78 | if (req.path === "/" || req.path === "/index.html") { 79 | serveHtmlMiddleware(req, res, next); 80 | } 81 | else { 82 | static_serve_middleware(req, res, next); 83 | } 84 | }); 85 | 86 | router.use('*', serveHtmlMiddleware); 87 | 88 | export default router; 89 | -------------------------------------------------------------------------------- /src/models.ts: -------------------------------------------------------------------------------- 1 | import mongoose from 'mongoose'; 2 | import { Status, FullUser } from 'twitter-d'; 3 | 4 | export interface IUser extends mongoose.Document { 5 | oauth_token: string, 6 | oauth_token_secret: string, 7 | user_id: string, 8 | last_login: Date, 9 | twitter_name: string, 10 | twitter_screen_name: string, 11 | twitter_id: string, 12 | profile_picture: string, 13 | created_at: Date, 14 | special?: boolean, 15 | } 16 | const user_schema = new mongoose.Schema({ 17 | oauth_token: String, 18 | oauth_token_secret: String, 19 | user_id: String, 20 | last_login: Date, 21 | twitter_name: String, 22 | twitter_screen_name: String, 23 | twitter_id: String, 24 | profile_picture: String, 25 | created_at: Date, 26 | special: Boolean, 27 | }); 28 | 29 | export interface IToken extends mongoose.Document { 30 | user_id: string, 31 | token: string, 32 | login_ip: string, 33 | date: Date, 34 | last_use: Date 35 | } 36 | const token_schema = new mongoose.Schema({ 37 | user_id: String, 38 | token: String, 39 | login_ip: String, 40 | date: Date, 41 | last_use: Date 42 | }); 43 | 44 | export interface ICloudedArchive extends mongoose.Document { 45 | file_id: string, 46 | user_id: string, 47 | filename: string, 48 | path: string, 49 | hash: string, 50 | date: Date, 51 | info: any, 52 | } 53 | const clouded_archive_schema = new mongoose.Schema({ 54 | file_id: String, 55 | user_id: String, 56 | filename: String, 57 | path: String, 58 | hash: String, 59 | date: Date, 60 | info: Object, 61 | }); 62 | 63 | export type ITweetPartial = mongoose.Document & Status; 64 | 65 | export interface ITweet extends ITweetPartial { 66 | inserted_time: Date; 67 | } 68 | 69 | const tweet_schema = new mongoose.Schema({ 70 | id_str: String, 71 | inserted_time: Date, 72 | user: {} 73 | }, { strict: false }); 74 | 75 | export type IUserPartial = mongoose.Document & FullUser; 76 | 77 | export interface ITwitterUser extends IUserPartial { 78 | inserted_time: Date; 79 | } 80 | 81 | const user_twitter_schema = new mongoose.Schema({ 82 | id_str: String, 83 | inserted_time: Date, 84 | }, { strict: false }); 85 | 86 | export const UserModel = mongoose.model('UserModel', user_schema, 'ae_user'); 87 | 88 | export const TokenModel = mongoose.model('TokenModel', token_schema, 'ae_token'); 89 | 90 | export const TweetModel = mongoose.model('TweetModel', tweet_schema, 'ae_tweets'); 91 | 92 | export const CloudedArchiveModel = mongoose.model('CloudedArchiveModel', clouded_archive_schema, 'ae_archives'); 93 | 94 | export const TwitterUserModel = mongoose.model('TwitterUserModel', user_twitter_schema, 'ae_twitter_users'); 95 | 96 | export const COLLECTIONS = { 97 | 'ae_user': UserModel, 98 | 'ae_token': TokenModel, 99 | 'ae_tweets': TweetModel, 100 | 'ae_twitter_users': TwitterUserModel, 101 | 'ae_archives': CloudedArchiveModel, 102 | }; 103 | -------------------------------------------------------------------------------- /src/api/batch/tiny_tweets.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import AEError, { sendError } from "../../errors"; 3 | import { batchTweets, getCompleteUserFromId, methodNotAllowed, sendTwitterError, suppressUselessTweetProperties } from "../../helpers"; 4 | import Twitter from '../../twitter_lite_clone/twitter_lite'; 5 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 6 | import { Status } from "twitter-d"; 7 | import logger from "../../logger"; 8 | 9 | //// BULKING TWEETS (100 max) 10 | 11 | const route = Router(); 12 | 13 | route.post('/', (req, res) => { 14 | // Download tweets from twitter 15 | 16 | if (req.user && req.body && req.body.ids && Array.isArray(req.body.ids)) { 17 | const ids: string[] = req.body.ids; 18 | 19 | // Test if rq is OK 20 | if (!req.body.ids.length) { 21 | sendError(AEError.invalid_request, res, "Needs tweets attached to request"); 22 | return; 23 | } 24 | if (req.body.ids.length > 100) { 25 | sendError(AEError.invalid_request, res, "Up to 100 tweets could be agregated in a request."); 26 | return; 27 | } 28 | if (!ids.every(e => typeof e === 'string' && e.length < 32)) { 29 | sendError(AEError.invalid_request, res, "Tweet IDs should be representated with strings of length < 32 chars"); 30 | return; 31 | } 32 | 33 | // Get tweets from DB or/and Twitter 34 | (async () => { 35 | const existings = await batchTweets(ids) as any as Status[]; 36 | 37 | const ids_existings = new Set(existings.map(e => e.id_str)); 38 | 39 | // array diff 40 | let to_retrieve = ids.filter(e => !ids_existings.has(e)) 41 | 42 | let error = false; 43 | 44 | if (to_retrieve.length) { 45 | // Max 100 tweets allowed 46 | if (to_retrieve.length > 100) { 47 | sendError(AEError.invalid_request, res); 48 | return; 49 | } 50 | 51 | logger.debug(`Batching ${to_retrieve.length} tweets from Twitter`); 52 | 53 | const bdd_user = await getCompleteUserFromId(req.user!.user_id); 54 | 55 | // If user does not exists 56 | if (!bdd_user) { 57 | sendError(AEError.invalid_token, res); 58 | return; 59 | } 60 | 61 | // Create Twitter object with credentials 62 | const user = new Twitter({ 63 | consumer_key: CONSUMER_KEY, 64 | consumer_secret: CONSUMER_SECRET, 65 | access_token_key: bdd_user.oauth_token, 66 | access_token_secret: bdd_user.oauth_token_secret 67 | }); 68 | 69 | // Batch tweets using lookup endpoint (100 max) 70 | const twitter_tweets = await user.post('statuses/lookup', { 71 | id: to_retrieve.join(','), 72 | include_entities: true, 73 | tweet_mode: "extended" 74 | }) 75 | // Otherwise, send Twitter error 76 | .catch(e => { 77 | sendTwitterError(e, res); 78 | error = true; 79 | }) as Status[]; 80 | 81 | if (twitter_tweets) { 82 | existings.push(...twitter_tweets); 83 | } 84 | } 85 | 86 | // Send response 87 | if (!error) 88 | res.json(existings.map(e => suppressUselessTweetProperties(e))); 89 | })().catch(e => { 90 | logger.error("Batch error", e); 91 | sendError(AEError.server_error, res); 92 | }); 93 | } 94 | else { 95 | sendError(AEError.invalid_request, res); 96 | } 97 | }); 98 | 99 | route.all('/', methodNotAllowed('POST')); 100 | 101 | export default route; 102 | -------------------------------------------------------------------------------- /src/api/batch/tweets.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import AEError, { sendError } from "../../errors"; 3 | import { batchTweets, getCompleteUserFromId, saveTweets, sanitizeMongoObj, methodNotAllowed, sendTwitterError } from "../../helpers"; 4 | import Twitter from '../../twitter_lite_clone/twitter_lite'; 5 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 6 | import { Status } from "twitter-d"; 7 | import logger from "../../logger"; 8 | 9 | //// BULKING TWEETS (100 max) 10 | 11 | const route = Router(); 12 | 13 | route.post('/', (req, res) => { 14 | // Download tweets from twitter 15 | 16 | if (req.user && req.body && req.body.ids && Array.isArray(req.body.ids)) { 17 | const ids: string[] = req.body.ids; 18 | 19 | // Test if rq is OK 20 | if (!req.body.ids.length) { 21 | sendError(AEError.invalid_request, res, "Needs tweets attached to request"); 22 | return; 23 | } 24 | if (req.body.ids.length > 100) { 25 | sendError(AEError.invalid_request, res, "Up to 100 tweets could be agregated in a request."); 26 | return; 27 | } 28 | if (!ids.every(e => typeof e === 'string' && e.length < 32)) { 29 | sendError(AEError.invalid_request, res, "Tweet IDs should be representated with strings of length < 32 chars"); 30 | return; 31 | } 32 | 33 | // Get tweets from DB or/and Twitter 34 | (async () => { 35 | const existings = await batchTweets(ids); 36 | 37 | const ids_existings = new Set(existings.map(e => e.id_str)); 38 | 39 | // array diff 40 | let to_retrieve = ids.filter(e => !ids_existings.has(e)) 41 | 42 | let error = false; 43 | 44 | if (to_retrieve.length) { 45 | // Max 100 tweets allowed 46 | if (to_retrieve.length > 100) { 47 | sendError(AEError.invalid_request, res); 48 | return; 49 | } 50 | 51 | logger.debug(`Batching ${to_retrieve.length} tweets from Twitter`); 52 | 53 | const bdd_user = await getCompleteUserFromId(req.user!.user_id); 54 | 55 | // If user does not exists 56 | if (!bdd_user) { 57 | sendError(AEError.invalid_token, res); 58 | return; 59 | } 60 | 61 | // Create Twitter object with credentials 62 | const user = new Twitter({ 63 | consumer_key: CONSUMER_KEY, 64 | consumer_secret: CONSUMER_SECRET, 65 | access_token_key: bdd_user.oauth_token, 66 | access_token_secret: bdd_user.oauth_token_secret 67 | }); 68 | 69 | // Batch tweets using lookup endpoint (100 max) 70 | const twitter_tweets = await user.post('statuses/lookup', { 71 | id: to_retrieve.join(','), 72 | include_entities: true, 73 | tweet_mode: "extended" 74 | }) 75 | // Save every tweet in mangoose (and catch insert errors) 76 | .then((statuses: Status[]) => saveTweets(statuses).catch(e => { 77 | logger.error("Unable to save tweets.", e); 78 | sendError(AEError.server_error, res); 79 | })) 80 | // Otherwise, send Twitter error 81 | .catch(e => { 82 | sendTwitterError(e, res); 83 | error = true; 84 | }); 85 | 86 | if (twitter_tweets) { 87 | existings.push(...twitter_tweets); 88 | } 89 | } 90 | 91 | // Send response 92 | if (!error) 93 | res.json(existings.map(e => sanitizeMongoObj(e))); 94 | })().catch(e => { 95 | logger.error("Batch error", e); 96 | sendError(AEError.server_error, res); 97 | }); 98 | } 99 | else { 100 | sendError(AEError.invalid_request, res); 101 | } 102 | }); 103 | 104 | route.all('/', methodNotAllowed('POST')); 105 | 106 | export default route; 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # archive-explorer-node 2 | 3 | Serve requests for the Archive Explorer project. 4 | 5 | This is the back-end side of the Archive Explorer project. See [archive-explorer-web](https://github.com/alkihis/archive-explorer-web) for the front-end website. 6 | 7 | Website is available in [archive-explorer.com](https://archive-explorer.com) ! 8 | 9 | ## Foreword 10 | 11 | None of the pages served to clients are server-side rendered. This server provides **static** access to the bundled React website, and gives access to an API developed with the **Express** framework. 12 | 13 | API exposes multiple REST endpoints and a WebSocket powered by **Socket.io**. 14 | 15 | This server needs a open **mongoDB** server to store user credentials and tweets/twitter users cache, internally opered by **mongoose**. 16 | 17 | The whole project is developed in **TypeScript**. 18 | 19 | **Worker threads** are used to dispatch tweet delete requests, so make sure you have a Node version that support the `worker_threads` module ! 20 | 21 | Supported Node versions are **Node 10.5+** with `--experimental-worker` flag, or **Node 12+** without flags. 22 | 23 | ## Getting started for development 24 | 25 | Clone repository and install dependencies with NPM. 26 | 27 | ```bash 28 | git clone https://github.com/alkihis/archive-explorer-node.git 29 | cd archive-explorer-node 30 | npm i 31 | ``` 32 | 33 | ### Setting up constants 34 | 35 | Some constants are not built-in, to provide security. 36 | 37 | #### Twitter constants 38 | 39 | You must have a working Twitter application in order to run server. This could be long to obtain, but I just can't share my own credentials. 40 | 41 | Duplicate the file `settings.sample.json` and name it `settings.json`. 42 | 43 | Set up inside the newly created file your Twitter application token (`consumer` key), the application secret key (`consumer_secret` key), and the callback after Sign in with Twitter login flow (`localhost:3000/finalize` should be fine for development, change it for production). 44 | 45 | #### Public and private keys 46 | 47 | This server use `JWT` (JSON Web Tokens) as token system. This kind of credentials requires a couple of public/private keys in order to sign tokens. 48 | 49 | Create a `.ssh` folder in the directory root. 50 | Go to this directory. 51 | Create a public and private key **with a passphrase**. 52 | 53 | All the created files will **not** be gitted to your fork or repo. 54 | ```bash 55 | # Creating dir 56 | mkdir .ssh 57 | cd .ssh 58 | 59 | # Generating private key (do not forget to enter a passphrase when asked) 60 | ssh-keygen -t rsa -b 4096 -m PEM -f key_new.pem 61 | 62 | # Generating public key 63 | openssl rsa -in key_new.pem -pubout -outform PEM -out key_new 64 | 65 | # Register the passphrase in the file "passphrase" 66 | echo "my_choosen_passphrase" > passphrase 67 | ``` 68 | 69 | Project is ready ! 70 | 71 | ### Compiling and running project 72 | 73 | You need to first compile the project in order to start the index.js file with Node. Make sure *TypeScript* npm package is installed globally. 74 | 75 | ```bash 76 | tsc 77 | ``` 78 | 79 | Now, make sure the **mongoDB** server is running, then start the server: 80 | 81 | ```bash 82 | node build/index.js -l -p -m 83 | ``` 84 | 85 | Default values are: 86 | - `-l info` 87 | - `-p 3128` 88 | - `-m 3281` 89 | 90 | You can ask for help with `--help`. 91 | 92 | ## Deploy 93 | 94 | By default, server will emit to port 443 (HTTPS). 95 | You just need to specify `NODE_ENV=production` env variable to `build/index.js`. 96 | 97 | ```bash 98 | # You may need to be sudo to emit to port 443. 99 | # Otherwise, start in dev mode and use Nginx with reverse proxy. 100 | NODE_ENV=production node build/index.js 101 | ``` 102 | 103 | Certificates should be in directory mentionned in settings.json's `https_key_directory`. It assume you will use Let's Encrypt. 104 | -------------------------------------------------------------------------------- /src/api/tasks/task_server.ts: -------------------------------------------------------------------------------- 1 | import io from '../../index'; 2 | import { Socket } from 'socket.io'; 3 | import { checkToken } from '../../helpers'; 4 | import logger from '../../logger'; 5 | import Task from './Task'; 6 | 7 | // Task server (uses socket.io) 8 | 9 | // Key is socket 10 | export const socket_to_tasks: Map> = new Map; 11 | 12 | export function startIo() { 13 | io.on('connection', socket => { 14 | // Souscription à une tâche (progression) 15 | socket.on('task', async (id: string, user_token: string) => { 16 | // Verify user and obtain user id from token... 17 | try { 18 | logger.debug(`User ask subscription to ${id}, verifing token...`); 19 | var payload = await checkToken(user_token); 20 | } catch (e) { } 21 | 22 | if (!payload || !payload.user_id) { 23 | socket.emit('task error', { id, msg: "Can't verify token or user is invalid" }); 24 | return; 25 | } 26 | 27 | const user_id = payload.user_id; 28 | logger.debug(`Logged user ${user_id} subscribing to task ${id}`); 29 | 30 | if (id) { 31 | try { 32 | var id_int = BigInt(id); 33 | } catch (e) { 34 | socket.emit('task error', { id, msg: "Task ID is invalid" }); 35 | return; 36 | } 37 | 38 | // Récupération de la tâche 39 | const task = Task.get(id_int); 40 | 41 | if (task) { 42 | // Si elle n'appartient pas à l'utilisateur 43 | if (task.owner !== user_id) { 44 | socket.emit('task error', { id, msg: "Task does not belong to you" }); 45 | return; 46 | } 47 | 48 | // Si ce socket n'est pas déjà inscrit 49 | if (!socket_to_tasks.has(socket)) { 50 | socket_to_tasks.set(socket, new Set); 51 | } 52 | 53 | socket_to_tasks.get(socket)!.add(id_int); 54 | 55 | task.subscribe(socket); 56 | } 57 | else { 58 | // La tâche n'existe pas / plus 59 | socket.emit('task error', { id, msg: "Task does not exists" }); 60 | } 61 | } 62 | }); 63 | 64 | socket.on('remove', async (id: string, user_token: string) => { 65 | // Verify user and obtain user id from token... 66 | try { 67 | logger.debug(`User ask unsub to ${id}, verifing token...`); 68 | var payload = await checkToken(user_token); 69 | } catch (e) { } 70 | 71 | if (!payload || !payload.user_id) { 72 | socket.emit('error', { id, msg: "Can't verify token or user is invalid" }); 73 | return; 74 | } 75 | 76 | const user_id = payload.user_id; 77 | logger.debug(`Logged user ${user_id} unsub task ${id}`); 78 | 79 | if (id) { 80 | try { 81 | var id_int = BigInt(id); 82 | } catch (e) { 83 | socket.emit('error', { id, msg: "Task ID is invalid" }); 84 | return; 85 | } 86 | 87 | // Récupération de la tâche 88 | const task = Task.get(id_int); 89 | 90 | if (task) { 91 | // Si elle n'appartient pas à l'utilisateur 92 | if (task.owner !== user_id) { 93 | return; 94 | } 95 | 96 | // Si ce socket n'est pas inscrit 97 | if (!socket_to_tasks.has(socket)) { 98 | return; 99 | } 100 | 101 | socket_to_tasks.get(socket)!.delete(id_int); 102 | 103 | task.unsubscribe(socket); 104 | } 105 | } 106 | }); 107 | 108 | // Déconnexion du socket 109 | socket.on('disconnect', () => { 110 | const tasks = socket_to_tasks.get(socket); 111 | if (tasks) { 112 | // Suppression de toutes les tâches affectées à ce socket 113 | for (const t of tasks) { 114 | const task = Task.get(t); 115 | 116 | if (task) { 117 | task.unsubscribe(socket); 118 | } 119 | } 120 | } 121 | 122 | socket_to_tasks.delete(socket); 123 | }); 124 | }); 125 | } 126 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import commander from 'commander'; 3 | import { mkdirSync } from 'fs'; 4 | import winston from 'winston'; 5 | import { VERSION } from './constants'; 6 | import logger, { FORMAT_FILE } from './logger'; 7 | import APIIndex, { apiErrors as APIErrors } from './api/index'; 8 | import socket_io from 'socket.io'; 9 | import http_base from 'http'; 10 | import { startIo } from './api/tasks/task_server'; 11 | import mongoose from 'mongoose'; 12 | import cors from 'cors'; 13 | import { COLLECTIONS } from './models'; 14 | import { purgeCollections } from './helpers'; 15 | import StaticServer from './static_server'; 16 | import { CliSettings, startCli } from './cli'; 17 | import { inspect } from 'util'; 18 | 19 | // archive-explorer-server main file 20 | // Meant to serve archive-explorer website, 21 | // and provide an API in order to delete tweets 22 | 23 | export let IS_DEV_MODE = true; 24 | 25 | commander 26 | .version(VERSION) 27 | .option('-p, --port ', 'Server port', Number, 3128) 28 | .option('-m, --mongo-port ', 'Mongo server port', Number, 3281) 29 | .option('-p, --purge', 'Purge all mongo collection, then quit') 30 | .option('--prod') 31 | .option('--file-logging') 32 | .option('-l, --log-level [logLevel]', 'Log level [debug|silly|verbose|info|warn|error]', /^(debug|silly|verbose|info|warn|error)$/, 'info') 33 | .parse(process.argv); 34 | 35 | if (process.env.NODE_ENV === 'production' || commander.prod) { 36 | IS_DEV_MODE = false; 37 | } 38 | 39 | if (commander.logLevel) { 40 | logger.level = commander.logLevel; 41 | } 42 | 43 | const app = express(); 44 | http_base.globalAgent.maxSockets = Infinity; 45 | 46 | let http_server: http_base.Server; 47 | let file_logging = commander.fileLogging; 48 | http_server = http_base.createServer(app); 49 | 50 | if (!IS_DEV_MODE) { 51 | console.log("Starting with prod mode."); 52 | logger.exitOnError = false; 53 | } 54 | else { 55 | console.log("Starting with dev mode."); 56 | 57 | if (file_logging === undefined) { 58 | file_logging = true; 59 | } 60 | 61 | // Define cors request for dev 62 | app.use(cors({ credentials: true, origin: '*', allowedHeaders: "*", exposedHeaders: "*" })); 63 | // @ts-ignore 64 | app.options('*', cors({ credentials: true, origin: '*' })); 65 | } 66 | 67 | if (file_logging) { 68 | // Activate file logger 69 | try { 70 | mkdirSync(__dirname + '/../logs'); 71 | } catch (e) { } 72 | 73 | logger.add(new winston.transports.File({ 74 | filename: __dirname + '/../logs/info.log', 75 | level: 'info', 76 | eol: "\n", 77 | format: FORMAT_FILE 78 | })); 79 | logger.add(new winston.transports.File({ 80 | filename: __dirname + '/../logs/warn.log', 81 | level: 'warn', 82 | eol: "\n", 83 | format: FORMAT_FILE 84 | })); 85 | logger.add(new winston.transports.File({ 86 | filename: __dirname + '/../logs/error.log', 87 | level: 'error', 88 | eol: "\n", 89 | format: FORMAT_FILE 90 | })); 91 | logger.exceptions.handle(new winston.transports.File({ 92 | filename: __dirname + '/../logs/exceptions.log', 93 | eol: "\n", 94 | format: FORMAT_FILE 95 | })); 96 | } 97 | 98 | const io = socket_io(http_server); 99 | export default io; 100 | 101 | logger.debug("Establishing MongoDB connection"); 102 | 103 | mongoose.connect('mongodb://localhost:' + commander.mongoPort + '/ae', { useNewUrlParser: true, useUnifiedTopology: true }); 104 | 105 | const db = mongoose.connection; 106 | 107 | db.on('error', (msg: any) => { 108 | logger.error("Incoming error message from MongoDB:", msg); 109 | }); 110 | 111 | db.once('open', function() { 112 | CliSettings.db_ok = true; 113 | 114 | if (commander.purge) { 115 | purgeCollections(COLLECTIONS, db, mongoose); 116 | return; 117 | } 118 | 119 | logger.verbose("MongoDB connection is open"); 120 | 121 | // tmp redirect to .com 122 | app.use('*', (req, res, next) => { 123 | if (req.hostname && req.hostname.includes('archive-explorer.fr')) { 124 | res.redirect(301, 'https://archive-explorer.com'); 125 | } 126 | else { 127 | next(); 128 | } 129 | }); 130 | 131 | logger.debug("Serving API"); 132 | app.use('/api', APIIndex); 133 | app.use('/api', APIErrors); 134 | 135 | logger.debug("Serving static website"); 136 | // File should be in build/ 137 | app.use(StaticServer); 138 | 139 | // 404 not found for all others pages 140 | app.use((_, res) => { 141 | res.status(404).send(); 142 | }); 143 | 144 | // Use http, not app ! 145 | http_server.listen(commander.port, () => { 146 | logger.info(`Archive Explorer Server ${VERSION} is listening on port ${commander.port}`); 147 | startCli(); 148 | }); 149 | 150 | startIo(); 151 | }); 152 | 153 | process.on('unhandledRejection', reason => { 154 | logger.error('Unhandled rejection :('); 155 | logger.error(inspect(reason)); 156 | }); 157 | -------------------------------------------------------------------------------- /static/www/index.html: -------------------------------------------------------------------------------- 1 | Twitter Archive Explorer
-------------------------------------------------------------------------------- /src/api/users/access.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import twitter from '../../twitter_lite_clone/twitter_lite'; 3 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 4 | import AEError, { sendError } from "../../errors"; 5 | import { getCompleteUserFromTwitterId, signToken, methodNotAllowed } from "../../helpers"; 6 | import { IUser, UserModel, TokenModel } from "../../models"; 7 | import { FullUser as TwitterUser } from 'twitter-d'; 8 | import uuid from "uuid"; 9 | import logger from "../../logger"; 10 | 11 | // Meant to be called when oauth callback is done 12 | const route = Router(); 13 | 14 | route.post('/', (req, res) => { 15 | if ( 16 | req.body && 17 | req.body.oauth_token && 18 | req.body.oauth_token_secret && 19 | req.body.oauth_verifier 20 | ) { 21 | const t = new twitter({ 22 | consumer_key: CONSUMER_KEY, 23 | consumer_secret: CONSUMER_SECRET 24 | }); 25 | 26 | // Encapsule la fonction async pour express 27 | (async () => { 28 | try { 29 | var access = await t.getAccessToken({ 30 | key: req.body.oauth_token, 31 | secret: req.body.oauth_token_secret, 32 | verifier: req.body.oauth_verifier 33 | }); 34 | } catch (e) { 35 | sendError(AEError.invalid_verifier, res); 36 | return; 37 | } 38 | 39 | // On a les données ! il faut vérifier que l'utilisateur existe 40 | var user: IUser | null; 41 | try { 42 | user = await getCompleteUserFromTwitterId(access.user_id as string); 43 | } catch (e) { 44 | // On doit créer l'utilisateur 45 | user = null; 46 | } 47 | 48 | // Obtention des données pour cet utilisateur 49 | const tmp_user = new twitter({ 50 | consumer_key: CONSUMER_KEY, 51 | consumer_secret: CONSUMER_SECRET, 52 | access_token_key: access.oauth_token as string, 53 | access_token_secret: access.oauth_token_secret as string 54 | }); 55 | 56 | // Obtention des credentials 57 | let t_user: TwitterUser; 58 | try { 59 | t_user = await tmp_user.get('account/verify_credentials'); 60 | } catch (e) { 61 | // invalid user 62 | sendError(AEError.invalid_verifier, res); 63 | return; 64 | } 65 | 66 | let token: string; 67 | const uniq_id = uuid.v4(); 68 | 69 | // On génère un JWT 70 | try { 71 | token = await signToken({ 72 | user_id: t_user.id_str, 73 | screen_name: t_user.screen_name, 74 | login_ip: req.connection.remoteAddress! 75 | }, uniq_id); 76 | } catch (e) { 77 | logger.error("Unable to sign token", e); 78 | sendError(AEError.server_error, res); 79 | return; 80 | } 81 | 82 | // Puis on enregistre le token en BDD ! 83 | const token_model = new TokenModel({ 84 | token: uniq_id, 85 | user_id: t_user.id_str, 86 | date: new Date, 87 | last_use: new Date, 88 | login_ip: req.connection.remoteAddress 89 | }); 90 | token_model.save(); 91 | 92 | if (user === null) { 93 | // Création 94 | // On enregistre l'utilisateur 95 | logger.info(`Creating account for user @${t_user.screen_name}.`); 96 | 97 | user = new UserModel({ 98 | oauth_token: access.oauth_token, 99 | oauth_token_secret: access.oauth_token_secret, 100 | twitter_id: access.user_id, 101 | twitter_name: t_user.name, 102 | twitter_screen_name: t_user.screen_name, 103 | profile_picture: t_user.profile_image_url_https, 104 | user_id: t_user.id_str, 105 | last_login: new Date, 106 | created_at: new Date, 107 | }); 108 | user.save(); 109 | } 110 | else { 111 | // Mise à jour de l'utilisateur avec les nouvelles données 112 | user.twitter_name = t_user.name; 113 | user.profile_picture = t_user.profile_image_url_https; 114 | user.oauth_token = access.oauth_token as string; 115 | user.oauth_token_secret = access.oauth_token_secret as string; 116 | user.twitter_screen_name = t_user.screen_name; 117 | user.last_login = new Date; 118 | user.save(); 119 | } 120 | 121 | logger.verbose(`User @${user!.twitter_screen_name} logged and token ${uniq_id} created.`); 122 | 123 | // L'utilisateur est prêt ! 124 | res.json({ 125 | status: true, 126 | token 127 | }); 128 | })().catch((e) => { 129 | logger.error("Uncaught error when trying to generate access token", e); 130 | sendError(AEError.server_error, res); 131 | }); 132 | } 133 | else { 134 | sendError(AEError.invalid_data, res); 135 | } 136 | }); 137 | 138 | route.all('/', methodNotAllowed('POST')); 139 | 140 | export default route; 141 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | // "incremental": true, /* Enable incremental compilation */ 5 | "target": "es2018", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 6 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 7 | // "lib": [], /* Specify library files to be included in the compilation. */ 8 | // "allowJs": true, /* Allow javascript files to be compiled. */ 9 | // "checkJs": true, /* Report errors in .js files. */ 10 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 11 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 12 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 13 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 14 | // "outFile": "./", /* Concatenate and emit output to single file. */ 15 | "outDir": "./build", /* Redirect output structure to the directory. */ 16 | "rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 17 | // "composite": true, /* Enable project compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | "skipLibCheck": true, 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | // "strictNullChecks": true, /* Enable strict null checks. */ 29 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 46 | // "typeRoots": [], /* List of folders to include type definitions from. */ 47 | // "types": [], /* Type declaration files to be included in compilation. */ 48 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 49 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 50 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 51 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 52 | 53 | /* Source Map Options */ 54 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 55 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 56 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 57 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 58 | 59 | /* Experimental Options */ 60 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 61 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/uploader.ts: -------------------------------------------------------------------------------- 1 | import multer from 'multer'; 2 | import { mkdirSync, promises as fs } from 'fs'; 3 | import path from 'path'; 4 | import { v4 as uuid } from 'uuid'; 5 | 6 | // 512 * 200: 100MB par archive max 7 | export const MAX_ALLOWED_CHUNKS = 200; 8 | export const MAX_CLOUDED_ARCHIVES = 10; 9 | export const UPLOAD_PATH = __dirname + '/../uploads/'; 10 | try { 11 | mkdirSync(UPLOAD_PATH, { recursive: true }); 12 | } catch {} 13 | 14 | export const uploader = multer({ 15 | dest: UPLOAD_PATH, 16 | limits: { fileSize: 1024 * 1024 }, 17 | }); 18 | export default uploader; 19 | 20 | export class UploadManager { 21 | static cleanup(file: Express.Multer.File | Express.Multer.File[]) { 22 | if (!file) { 23 | return; 24 | } 25 | 26 | if (Array.isArray(file)) { 27 | return file.map(f => fs.unlink(f.path).catch(() => {})); 28 | } 29 | else { 30 | return fs.unlink(file.path).catch(() => {}); 31 | } 32 | } 33 | 34 | static copyTo(file: Express.Multer.File, name: string) { 35 | const dirname = path.dirname(file.path); 36 | const newpath = path.normalize(dirname + '/' + name); 37 | 38 | return fs.copyFile(file.path, newpath); 39 | } 40 | } 41 | 42 | export interface ChunkManifest { 43 | id: string; 44 | size: number; 45 | sent_size: number; 46 | chunks: string[]; 47 | name: string; 48 | info: any; 49 | started_at: string; 50 | last_update: string; 51 | } 52 | 53 | export class ChunkManager { 54 | constructor(protected file_id: string) {} 55 | 56 | static getNewFileId() { 57 | return uuid(); 58 | } 59 | 60 | static async removeOutdatedManifests() { 61 | const manifests = await fs.readdir(UPLOAD_PATH); 62 | const threshold = new Date(); 63 | // Max life: 15 minutes 64 | threshold.setMinutes(threshold.getMinutes() - 15); 65 | const to_delete: [string, ChunkManifest][] = []; 66 | 67 | for (const manifest of manifests.filter(m => m.endsWith('.manifest'))) { 68 | const manifest_data: ChunkManifest = JSON.parse(await fs.readFile(UPLOAD_PATH + '/' + manifest, 'utf-8')); 69 | const last_updated = new Date(manifest_data.last_update); 70 | 71 | // If last update inferior to threshold time 72 | if (last_updated.getTime() < threshold.getTime()) { 73 | to_delete.push([manifest, manifest_data]); 74 | } 75 | } 76 | 77 | return Promise.all(to_delete.map(async ([item, manifest]) => { 78 | // Delete chunk attached to manifest 79 | await this.deleteFromManifest(manifest); 80 | // Delete manifest 81 | await fs.unlink(UPLOAD_PATH + '/' + item).catch(e => {}); 82 | })); 83 | } 84 | 85 | protected static deleteFromManifest(manifest: ChunkManifest) { 86 | return Promise.all( 87 | manifest.chunks.map(e => 88 | fs 89 | .unlink(path.normalize(UPLOAD_PATH + '/' + e)) 90 | .catch(d => {}) 91 | ) 92 | ); 93 | } 94 | 95 | async reconstructFile(manifest: ChunkManifest) { 96 | let i = 0; 97 | const indexed_chunks: [number, string][] = manifest.chunks.map(e => [this.getChunkIndex(e), e]); 98 | const sorted_chunks = indexed_chunks.sort((a, b) => { 99 | return a[0] - b[0]; 100 | }); 101 | const filename = this.file_id + '.zip'; 102 | const final_path = path.normalize(UPLOAD_PATH + '/' + filename); 103 | 104 | for (const [chunk_index, chunk_name] of sorted_chunks) { 105 | if (chunk_index !== i) { 106 | throw new RangeError('Index order does not match'); 107 | } 108 | 109 | const chunk_path = path.normalize(UPLOAD_PATH + '/' + chunk_name); 110 | // Append chunk to final file 111 | await fs.appendFile(final_path, await fs.readFile(chunk_path)); 112 | 113 | i++; 114 | } 115 | 116 | // Ok, file reconstructed 117 | // Remove every chunk + manifest 118 | await ChunkManager.deleteFromManifest(manifest); 119 | await fs.unlink(this.manifest_path).catch(e => {}); 120 | 121 | return filename; 122 | } 123 | 124 | getChunkPath(chunk_id: string) { 125 | return this.file_id + '__' + chunk_id + '.chunk'; 126 | } 127 | 128 | getManifest() : Promise { 129 | return fs.readFile(this.manifest_path, 'utf-8').then(JSON.parse); 130 | } 131 | 132 | saveManifest(manifest: ChunkManifest) { 133 | manifest.last_update = new Date().toISOString(); 134 | return fs.writeFile(this.manifest_path, JSON.stringify(manifest)).then(() => manifest); 135 | } 136 | 137 | registerManifest(file_size: number, name: string, info: object) { 138 | return this.saveManifest({ 139 | id: this.file_id, 140 | size: file_size, 141 | sent_size: 0, 142 | chunks: [], 143 | name, 144 | info, 145 | started_at: new Date().toISOString(), 146 | last_update: '', 147 | }); 148 | } 149 | 150 | static registerManifest(file_size: number, name: string, info: object) { 151 | const cm = new ChunkManager(ChunkManager.getNewFileId()); 152 | return cm.registerManifest(file_size, name, info); 153 | } 154 | 155 | protected get manifest_path() { 156 | return path.normalize(UPLOAD_PATH + '/' + this.file_id + '.manifest'); 157 | } 158 | 159 | protected getChunkIndex(chunk_path: string) { 160 | const chunk_name = chunk_path.split(this.file_id + '__')[1]; 161 | return Number(chunk_name.split('.chunk')[0]); 162 | } 163 | } 164 | 165 | // Set timer to delete outdated manifests and chunks 166 | setInterval(() => { 167 | ChunkManager.removeOutdatedManifests(); 168 | }, 120 * 1000); 169 | -------------------------------------------------------------------------------- /src/api/batch/users.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import AEError, { sendError } from "../../errors"; 3 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 4 | import { getCompleteUserFromId, batchUsers, saveTwitterUsers, sanitizeMongoObj, methodNotAllowed, sendTwitterError } from "../../helpers"; 5 | import Twitter from '../../twitter_lite_clone/twitter_lite'; 6 | import { FullUser } from "twitter-d"; 7 | import logger from "../../logger"; 8 | 9 | //// BULKING USERS (100 max) 10 | const user_ids_that_dont_exists = new Set(); 11 | const screen_names_that_dont_exists = new Map(); 12 | 13 | // reset ID set every 24 hours 14 | setInterval(() => { 15 | user_ids_that_dont_exists.clear(); 16 | }, 1000 * 60 * 60 * 24); 17 | 18 | // Clear not found screen names set every hour 19 | setInterval(() => { 20 | const entries = [...screen_names_that_dont_exists.entries()]; 21 | // Minimal date to validate, 5 days ago 22 | const threshold = new Date().getTime() - (1000 * 60 * 60 * 24 * 5); 23 | for (const [sn, date] of entries) { 24 | if (date.getTime() < threshold) { 25 | screen_names_that_dont_exists.delete(sn); 26 | } 27 | } 28 | }, 1000 * 60 * 60); 29 | 30 | const route = Router(); 31 | 32 | route.post('/', (req, res) => { 33 | // Download users from twitter 34 | const fetch_id = req.body && req.body.ids && Array.isArray(req.body.ids); 35 | 36 | if (req.user && req.body && ( 37 | fetch_id 38 | || (req.body.sns && Array.isArray(req.body.sns)) 39 | )) { 40 | const ids: string[] = fetch_id ? req.body.ids : req.body.sns; 41 | 42 | // Test if rq is OK 43 | if (!ids.length) { 44 | sendError(AEError.invalid_request, res, "Needs users attached to request"); 45 | return; 46 | } 47 | if (ids.length > 100) { 48 | sendError(AEError.invalid_request, res, "Up to 100 users could be agregated in a request."); 49 | return; 50 | } 51 | if (!ids.every(e => typeof e === 'string' && e.length < 32)) { 52 | sendError(AEError.invalid_request, res, "Users should be representated with strings of length < 32 chars"); 53 | return; 54 | } 55 | 56 | // Get users from DB or/and Twitter 57 | (async () => { 58 | const existings = await batchUsers(ids, !fetch_id); 59 | 60 | const ids_existings = new Set( 61 | existings.map(e => { 62 | if (fetch_id) { 63 | return e.id_str; 64 | } 65 | return e.screen_name.toLowerCase(); 66 | }) 67 | ); 68 | 69 | // array diff 70 | let to_retrieve = ids.map(e => e.toLowerCase()).filter(e => !ids_existings.has(e)); 71 | let error = false; 72 | 73 | if (fetch_id) { 74 | to_retrieve = to_retrieve.filter(id => !user_ids_that_dont_exists.has(id)); 75 | } 76 | else { 77 | // fetch screen name 78 | to_retrieve = to_retrieve.filter(screen_name => !screen_names_that_dont_exists.has(screen_name)); 79 | } 80 | 81 | if (to_retrieve.length) { 82 | // Max 100 users allowed 83 | if (to_retrieve.length > 100) { 84 | sendError(AEError.invalid_request, res); 85 | return; 86 | } 87 | 88 | logger.debug(`Batching ${to_retrieve.length} users from Twitter`); 89 | 90 | const bdd_user = await getCompleteUserFromId(req.user!.user_id); 91 | 92 | // If user does not exists 93 | if (!bdd_user) { 94 | sendError(AEError.invalid_token, res); 95 | return; 96 | } 97 | 98 | // Create Twitter object with credentials 99 | const user = new Twitter({ 100 | consumer_key: CONSUMER_KEY, 101 | consumer_secret: CONSUMER_SECRET, 102 | access_token_key: bdd_user.oauth_token, 103 | access_token_secret: bdd_user.oauth_token_secret 104 | }); 105 | 106 | // Batch tweets using lookup endpoint (100 max) 107 | const parameters: any = { include_entities: true }; 108 | if (fetch_id) { 109 | parameters.user_id = to_retrieve.join(','); 110 | } 111 | else { 112 | parameters.screen_name = to_retrieve.join(','); 113 | } 114 | 115 | const twitter_users = await user.post('users/lookup', parameters) 116 | // Save every tweet in mangoose (and catch insert errors) 117 | .then((users: FullUser[]) => saveTwitterUsers(users).catch(e => { 118 | logger.error("Unable to save users.", e); 119 | sendError(AEError.server_error, res); 120 | error = true; 121 | })) 122 | // Otherwise, send Twitter error 123 | .catch(e => { 124 | if (e.errors && e.errors[0].code === 17) { 125 | // No user match, skipping 126 | return; 127 | } 128 | 129 | sendTwitterError(e, res); 130 | error = true; 131 | }); 132 | 133 | if (twitter_users) { 134 | existings.push(...twitter_users); 135 | } 136 | 137 | // Check which ones does not exists 138 | if (fetch_id) { 139 | // Find users to retrieve that are not in 140 | const retrieved = new Set((twitter_users || []).map(e => e.id_str)); 141 | const not_found = to_retrieve.filter(id => !retrieved.has(id)); 142 | 143 | if (not_found.length) { 144 | logger.debug(`${not_found.length} users has been marqued as not found.`); 145 | for (const e of not_found) { 146 | user_ids_that_dont_exists.add(e); 147 | } 148 | } 149 | } 150 | else { 151 | // fetch screen name : find users to retrieve that are not in 152 | const retrieved = new Set((twitter_users || []).map(e => e.screen_name)); 153 | const not_found = to_retrieve.filter(sn => !retrieved.has(sn)); 154 | 155 | if (not_found.length) { 156 | logger.debug(`${not_found.length} users has been marqued as not found.`); 157 | for (const e of not_found) { 158 | screen_names_that_dont_exists.set(e, new Date); 159 | } 160 | } 161 | } 162 | } 163 | 164 | // Send response 165 | if (!error) 166 | res.json(existings.map(e => sanitizeMongoObj(e))); 167 | })().catch(e => { 168 | logger.error("Batch error", e); 169 | sendError(AEError.server_error, res); 170 | }); 171 | } 172 | else { 173 | sendError(AEError.invalid_request, res); 174 | } 175 | }); 176 | 177 | route.all('/', methodNotAllowed('POST')); 178 | 179 | export default route; 180 | -------------------------------------------------------------------------------- /src/api/cloud/upload.ts: -------------------------------------------------------------------------------- 1 | import { Response, Router } from 'express'; 2 | import { methodNotAllowed } from "../../helpers"; 3 | import uploader, { ChunkManager, ChunkManifest, MAX_ALLOWED_CHUNKS, MAX_CLOUDED_ARCHIVES, UploadManager, UPLOAD_PATH } from '../../uploader'; 4 | import AEError, { sendError } from '../../errors'; 5 | import logger from '../../logger'; 6 | import { CloudedArchiveModel } from '../../models'; 7 | import md5File from 'md5-file'; 8 | import { CONFIG_FILE } from '../../constants'; 9 | import path from 'path'; 10 | 11 | const UploadArchiveCloud = Router(); 12 | 13 | UploadArchiveCloud.get('/allowed.json', (_, res) => { 14 | res.json({ 15 | allowed: CONFIG_FILE.allowed_users_to_cloud, 16 | }); 17 | }); 18 | 19 | UploadArchiveCloud.all('/allowed.json', methodNotAllowed('GET')); 20 | 21 | UploadArchiveCloud.post('/start.json', (req, res) => { 22 | const size = Number(req.body.size); 23 | const filename = req.body.filename; 24 | const info = req.body.info; 25 | 26 | if (!size || size < 0 || !filename) { 27 | return sendError(AEError.invalid_data, res); 28 | } 29 | if (typeof info !== 'object') { 30 | return sendError(AEError.invalid_data, res); 31 | } 32 | const allowed_users = CONFIG_FILE.allowed_users_to_cloud as string[]; 33 | if (!allowed_users.includes(req.user!.user_id)) { 34 | console.log(`User ${req.user!.user_id} is not allowed to register cloud archives.`); 35 | return sendError(AEError.forbidden, res); 36 | } 37 | 38 | (async () => { 39 | const existing_count = await CloudedArchiveModel.countDocuments({ user_id: req.user!.user_id }) as number; 40 | if (existing_count >= MAX_CLOUDED_ARCHIVES) { 41 | return sendError(AEError.forbidden, res); 42 | } 43 | 44 | const existing_hash = await CloudedArchiveModel.countDocuments({ 45 | user_id: req.user!.user_id, 46 | 'info.hash': info.hash, 47 | }) as number; 48 | if (existing_hash) { 49 | return res.json({ 50 | already_sent: true, 51 | }); 52 | } 53 | 54 | const manifest = await ChunkManager.registerManifest(size, filename, info); 55 | 56 | res.json({ 57 | id: manifest.id, 58 | }); 59 | })().catch(e => { 60 | sendError(AEError.server_error, res); 61 | logger.error("Server error", e); 62 | }); 63 | }); 64 | 65 | UploadArchiveCloud.all('/start.json', methodNotAllowed('POST')); 66 | 67 | UploadArchiveCloud.post('/chunk.json', uploader.single('chunk'), (req, res) => { 68 | function errorAndCleanup(error: AEError, res: Response) { 69 | sendError(error, res); 70 | UploadManager.cleanup(chunk); 71 | } 72 | 73 | const allowed_users = CONFIG_FILE.allowed_users_to_cloud as string[]; 74 | if (!allowed_users.includes(req.user!.user_id)) { 75 | return sendError(AEError.forbidden, res); 76 | } 77 | 78 | const chunk = req.file; 79 | const file_id = req.body.file_id; 80 | const chunk_id = Number(req.body.chunk_id); 81 | 82 | if (!file_id || isNaN(chunk_id) || file_id.includes('/')) { 83 | return errorAndCleanup(AEError.invalid_data, res); 84 | } 85 | 86 | (async () => { 87 | const manager = new ChunkManager(file_id); 88 | let manifest: ChunkManifest; 89 | 90 | try { 91 | manifest = await manager.getManifest(); 92 | } catch (e) { 93 | return sendError(AEError.inexistant, res); 94 | } 95 | 96 | if (manifest.chunks.length >= MAX_ALLOWED_CHUNKS) { 97 | return sendError(AEError.too_many_chunks, res); 98 | } 99 | 100 | const chunk_path = manager.getChunkPath(chunk_id.toString()); 101 | 102 | if (manifest.chunks.includes(chunk_path)) { 103 | return sendError(AEError.forbidden, res); 104 | } 105 | 106 | manifest.chunks.push(chunk_path); 107 | await manager.saveManifest(manifest); 108 | 109 | // Store the file 110 | try { 111 | await UploadManager.copyTo(chunk, chunk_path); 112 | } catch (e) { 113 | manifest.chunks = manifest.chunks.slice(0, manifest.chunks.length - 1); 114 | // Might overwrite changes... 115 | manager.saveManifest(manifest); 116 | 117 | sendError(AEError.server_error, res); 118 | logger.error("Server error", e); 119 | return; 120 | } 121 | 122 | manifest.sent_size += chunk.size; 123 | await manager.saveManifest(manifest); 124 | 125 | res.json({ 126 | id: manifest.id, 127 | }); 128 | })().catch(e => { 129 | sendError(AEError.server_error, res); 130 | logger.error("Server error", e); 131 | }).finally(() => UploadManager.cleanup(chunk)); 132 | }); 133 | 134 | UploadArchiveCloud.all('/chunk.json', methodNotAllowed('POST')); 135 | 136 | UploadArchiveCloud.post('/terminate.json', (req, res) => { 137 | const allowed_users = CONFIG_FILE.allowed_users_to_cloud as string[]; 138 | if (!allowed_users.includes(req.user!.user_id)) { 139 | return sendError(AEError.forbidden, res); 140 | } 141 | 142 | const file_id = req.body.file_id; 143 | 144 | if (!file_id || file_id.includes('/')) { 145 | return sendError(AEError.invalid_data, res); 146 | } 147 | 148 | (async () => { 149 | const manager = new ChunkManager(file_id); 150 | let manifest: ChunkManifest; 151 | 152 | try { 153 | manifest = await manager.getManifest(); 154 | } catch (e) { 155 | return sendError(AEError.inexistant, res); 156 | } 157 | 158 | if (manifest.sent_size !== manifest.size) { 159 | return sendError(AEError.size_mismatch, res); 160 | } 161 | 162 | // Reconstruct original file 163 | const final_path = await manager.reconstructFile(manifest); 164 | const hash = await md5File(path.normalize(UPLOAD_PATH + '/' + final_path)); 165 | 166 | // Register file into mongodb 167 | await CloudedArchiveModel.create({ 168 | file_id, 169 | user_id: req.user!.user_id, 170 | filename: manifest.name, 171 | path: final_path, 172 | hash, 173 | date: new Date(), 174 | info: manifest.info, 175 | }); 176 | 177 | res.json({ 178 | status: 'ok', 179 | id: file_id, 180 | name: manifest.name, 181 | }); 182 | })().catch(e => { 183 | sendError(AEError.server_error, res); 184 | logger.error("Server error", e); 185 | }); 186 | }); 187 | 188 | UploadArchiveCloud.all('/terminate.json', methodNotAllowed('POST')); 189 | 190 | export default UploadArchiveCloud; 191 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | import CliHelper from 'interactive-cli-helper'; 2 | import mongoose from 'mongoose'; 3 | import Task, { isValidTaskType } from './api/tasks/Task'; 4 | import { COLLECTIONS } from './models'; 5 | import { purgePartial, getCompleteUserFromTwitterScreenName } from './helpers'; 6 | import { reloadSettings, TweetCounter } from './constants'; 7 | import { clearCache } from './static_server'; 8 | 9 | export const CliSettings = { 10 | db_ok: false 11 | }; 12 | 13 | export function startCli() { 14 | const db = mongoose.connection; 15 | 16 | function printTask(task: Task) { 17 | return `@${task.owner_screen_name} [${task.type} #${task.id}]: ${task.current_progression.remaining}/${task.current_progression.failed}/${task.current_progression.total} [${task.worker_count} threads]`; 18 | } 19 | 20 | const cli = new CliHelper({ 21 | onNoMatch: "Command not recognized. Available commands are: reload-settings, coll, exit, task.", 22 | suggestions: true 23 | }); 24 | 25 | cli.command('reload-settings', () => { 26 | reloadSettings(); 27 | return 'Settings reloaded.'; 28 | }); 29 | 30 | cli.command('reset-cache', () => { 31 | clearCache(); 32 | }); 33 | 34 | const collection = cli.command( 35 | 'coll', 36 | (_, __, ___, validator) => validator ? "Available commands for coll: list, drop" : "Can't access collections: Database is not ready.", 37 | { onValidateBefore: () => CliSettings.db_ok } 38 | ); 39 | 40 | // Drop a collection 41 | collection.command( 42 | /^drop ?(.+)?/, 43 | (_, __, matches) => { 44 | if (matches && matches[1]) { 45 | const coll = matches[1].split(/\s/g) as (keyof typeof COLLECTIONS)[]; 46 | const to_delete = coll.filter(e => e in COLLECTIONS); 47 | 48 | if (to_delete.length) { 49 | const colls: { [name: string]: any } = {}; 50 | 51 | for (const name of to_delete) { 52 | colls[name] = COLLECTIONS[name]; 53 | } 54 | 55 | return purgePartial(colls, db).then(() => "Purge completed."); 56 | } 57 | else { 58 | return `Collection(s) ${coll.join(', ')} don't exist.`; 59 | } 60 | } 61 | else { 62 | return `Usage: coll drop `; 63 | } 64 | } 65 | ); 66 | // List all the collections 67 | collection.command( 68 | 'list', 69 | `Available collections are: ${Object.keys(COLLECTIONS).join(', ')}` 70 | ); 71 | 72 | // Exit the server 73 | cli.command('exit', () => { 74 | console.log("Goodbye.") 75 | TweetCounter.sync(); 76 | process.exit(0); 77 | }); 78 | 79 | // ----------------- 80 | // | TASK LISTENER | 81 | // ----------------- 82 | // Task listener: get info about running task, run a new task... 83 | const task_listener = cli.command( 84 | 'task', 85 | () => `There are ${Task.count} running tasks. Available commands: list, create, stop` 86 | ); 87 | 88 | // ------------ 89 | // INFOS / LIST 90 | // Get info about all tasks 91 | const list_task_listener = task_listener.command( 92 | 'list', 93 | () => `Running tasks (${Task.count}).\n${ 94 | [...Task.all()] 95 | .map(printTask) 96 | .join('\n') 97 | }` 98 | ); 99 | // List all tasks from user 100 | list_task_listener.command( 101 | /^@(.+)/, 102 | async (_, __, matches) => { 103 | if (matches && matches[1]) { 104 | const user_sn = matches[1]; 105 | const user_object = await getCompleteUserFromTwitterScreenName(user_sn); 106 | 107 | if (user_object) { 108 | return [...Task.tasksOf(user_object.user_id)].map(printTask).join('\n'); 109 | } 110 | return `User ${user_sn} does not exists.`; 111 | } 112 | } 113 | ); 114 | // List a task by ID 115 | list_task_listener.command( 116 | /^\d+$/, 117 | rest => { 118 | const id = rest.trim(); 119 | if (Task.exists(id)) { 120 | return printTask(Task.get(id)!); 121 | } 122 | return "This task does not exists."; 123 | } 124 | ); 125 | 126 | // ---------- 127 | // STOP TASKS 128 | const task_stop_listener = task_listener.command( 129 | 'stop', 130 | rest => { 131 | Task.get(rest.trim())?.cancel(); 132 | return "Task stopped."; 133 | } 134 | ); 135 | // Stop all tasks 136 | task_stop_listener.command( 137 | 'all', 138 | () => { 139 | for (const task of Task.all()) { 140 | task.cancel(); 141 | } 142 | 143 | return `All tasks has been stopped.`; 144 | } 145 | ); 146 | // Stop all tasks from user 147 | task_stop_listener.command( 148 | /@(.+)/, 149 | async (_, __, matches) => { 150 | if (matches && matches[1]) { 151 | const user_sn = matches[1]; 152 | const user_object = await getCompleteUserFromTwitterScreenName(user_sn); 153 | 154 | if (user_object) { 155 | for (const task of Task.tasksOf(user_object.user_id)) { 156 | task.cancel(); 157 | } 158 | } 159 | return `Tasks of ${user_sn} has been stopped.`; 160 | } 161 | } 162 | ); 163 | 164 | // ------------ 165 | // CREATE TASKS 166 | task_listener.command( 167 | /create @(\S+)/, 168 | async (rest, _, matches) => { 169 | if (!matches || !matches[1]) { 170 | return "User not found."; 171 | } 172 | 173 | // User to create with 174 | const user_object = await getCompleteUserFromTwitterScreenName(matches[1]); 175 | if (!user_object) { 176 | return "User does not exists."; 177 | } 178 | 179 | const [type, ids] = rest.split(' ', 2); 180 | 181 | if (!type) { 182 | return "Task type is required."; 183 | } 184 | if (!ids) { 185 | return "IDs are required."; 186 | } 187 | 188 | if (!isValidTaskType(type)) { 189 | return "Invalid task type."; 190 | } 191 | 192 | const spaced = ids.split(' ').filter(e => e).map(e => e.split(',').filter(e => e)); 193 | const all_ids = [...new Set(Array().concat(...spaced))]; 194 | 195 | // Create the task 196 | const task = new Task(all_ids, { 197 | oauth_token: user_object.oauth_token, 198 | oauth_token_secret: user_object.oauth_token_secret, 199 | user_id: user_object.user_id, 200 | screen_name: user_object.twitter_screen_name 201 | }, type); 202 | 203 | return `Task #${task.id} created for user @${user_object.twitter_screen_name}.`; 204 | } 205 | ); 206 | 207 | cli.listen(); 208 | } 209 | -------------------------------------------------------------------------------- /src/api/tasks/task_worker/WorkerTask.ts: -------------------------------------------------------------------------------- 1 | import Twitter from '../../../twitter_lite_clone/twitter_lite'; 2 | import { MessagePort } from 'worker_threads'; 3 | 4 | /** 5 | * Represents a current Twitter user, used to make requests. 6 | */ 7 | export interface TwitterCredentials { 8 | consumer_token: string; 9 | consumer_secret: string; 10 | oauth_token: string; 11 | oauth_token_secret: string; 12 | } 13 | 14 | /** 15 | * Message sended to worker to start a new task. 16 | */ 17 | export interface WorkerTask { 18 | type: "task" | "stop", 19 | credentials: TwitterCredentials, 20 | tweets: string[], 21 | task_type: TaskType, 22 | debug?: boolean; 23 | } 24 | 25 | /** 26 | * Object used to start a new task with the WorkerTaskMaker 27 | */ 28 | export interface TaskStarter { 29 | method: string; 30 | request: (id: string) => ({ endpoint: string, parameters: object }); 31 | chunk_length: number; 32 | retry_on_rate_limit_exceeded?: boolean; 33 | } 34 | 35 | /** 36 | * Link a TaskType to its TaskStarter object. 37 | */ 38 | export const TaskJobs = { 39 | tweet: { 40 | method: 'POST', 41 | request: (id: string) => ({ endpoint: 'statuses/destroy/' + id, parameters: { trim_user: true } }), 42 | chunk_length: 100, 43 | retry_on_rate_limit_exceeded: true 44 | }, 45 | mute: { 46 | method: 'POST', 47 | request: (id: string) => ({ endpoint: 'mutes/users/destroy', parameters: { user_id: id, skip_status: true, include_entities: false } }), 48 | chunk_length: 75, 49 | retry_on_rate_limit_exceeded: true 50 | }, 51 | block: { 52 | method: 'POST', 53 | request: (id: string) => ({ endpoint: 'blocks/destroy', parameters: { user_id: id, include_entities: false, skip_status: true } }), 54 | chunk_length: 75, 55 | retry_on_rate_limit_exceeded: true 56 | }, 57 | fav: { 58 | method: 'POST', 59 | request: (id: string) => ({ endpoint: 'favorites/destroy', parameters: { id, include_entities: false } }), 60 | chunk_length: 75, 61 | retry_on_rate_limit_exceeded: true 62 | }, 63 | dm: { 64 | method: 'DELETE', 65 | request: (id: string) => ({ endpoint: 'direct_messages/events/destroy', parameters: { id } }), 66 | chunk_length: 100, 67 | retry_on_rate_limit_exceeded: true 68 | } 69 | }; 70 | 71 | export type TaskType = keyof typeof TaskJobs; 72 | 73 | 74 | /** 75 | * Represents a current task handled by the current worker. 76 | */ 77 | export default class WorkerTaskMaker { 78 | protected task_maker: (id: string) => Promise; 79 | 80 | public debug_mode = false; 81 | protected stopped = false; 82 | 83 | constructor( 84 | protected ids: string[], 85 | credentials: TwitterCredentials, 86 | protected starter: TaskStarter, 87 | protected message_port: MessagePort, 88 | ) { 89 | const user = new Twitter({ 90 | consumer_key: credentials.consumer_token, 91 | consumer_secret: credentials.consumer_secret, 92 | access_token_key: credentials.oauth_token, 93 | access_token_secret: credentials.oauth_token_secret 94 | }); 95 | 96 | let fn: (endpoint: string, parameters: object) => Promise; 97 | switch (starter.method) { 98 | case "POST": 99 | fn = user.post.bind(user); 100 | break; 101 | case "DELETE": 102 | fn = user.delete.bind(user); 103 | break; 104 | default: 105 | fn = user.get.bind(user); 106 | } 107 | 108 | this.task_maker = async id => { 109 | const { endpoint, parameters } = starter.request(id); 110 | const res = await fn(endpoint, parameters); 111 | return res; 112 | }; 113 | } 114 | 115 | async start() { 116 | // concurrent running tasks 117 | let chunk_len = this.starter.chunk_length; 118 | let do_task = this.task_maker; 119 | const error_stack = new Set(); 120 | 121 | // DEBUG 122 | if (this.debug_mode) { 123 | chunk_len = 3; 124 | do_task = () => new Promise(resolve => setTimeout(resolve, 500)); 125 | 126 | this.message_port!.postMessage({ 127 | type: "misc", ids: this.ids 128 | }); 129 | } 130 | 131 | // do the task... 132 | let current_i = 0; 133 | 134 | let promises: Promise[] = []; 135 | 136 | let chunk = this.ids.slice(current_i, current_i + chunk_len); 137 | current_i += chunk_len; 138 | 139 | let current = { done: 0, failed: 0 }; 140 | let last_errors: {[code: string]: [number, string]} = {}; 141 | 142 | const done_pp_fn = () => { current.done++; }; 143 | const failed_pp_fn: (e: any) => Promise | undefined = (e: any) => { 144 | // Check errors 145 | if (this.starter.retry_on_rate_limit_exceeded && e && e.errors && e.errors[0].code === 88) { 146 | // Rate limit exceeded 147 | return Promise.reject(88); 148 | } 149 | else if (e && e.errors) { 150 | // No status found with that ID. 151 | if (e.errors[0].code === 144) { 152 | current.done++; 153 | return; 154 | } 155 | // Other errors 156 | if (e.errors[0].code in last_errors) { 157 | last_errors[e.errors[0].code][0]++; 158 | } 159 | else { 160 | last_errors[e.errors[0].code] = [1, e.errors[0].message]; 161 | } 162 | } 163 | else if ( 164 | e instanceof Error && 165 | e.stack && 166 | e.stack.startsWith('FetchError') && 167 | e.stack.includes('https://api.twitter.com/1.1/statuses/destroy/') 168 | ) { 169 | // Try to get the ID (statuses) 170 | const part2 = e.stack.split('https://api.twitter.com/1.1/statuses/destroy/').pop(); 171 | if (part2) { 172 | const id = part2.split('.json', 2)[0]; 173 | 174 | if (!error_stack.has(id) && Number(id)) { 175 | error_stack.add(id); 176 | 177 | // Retry the task 178 | return do_task(id) 179 | .then(done_pp_fn) 180 | .catch(failed_pp_fn); 181 | } 182 | } 183 | } 184 | 185 | current.failed++; 186 | }; 187 | 188 | while (chunk.length) { 189 | if (this.stopped) 190 | return; 191 | 192 | for (const id of chunk) { 193 | try { 194 | BigInt(id); 195 | } catch (e) { 196 | current.failed++; 197 | continue; 198 | } 199 | 200 | promises.push( 201 | do_task(id) 202 | .then(done_pp_fn) 203 | .catch(failed_pp_fn) 204 | ); 205 | } 206 | 207 | if (this.stopped) 208 | return; 209 | 210 | try { 211 | await Promise.all(promises); 212 | } catch (e) { 213 | // Un tweet est en rate limit exceeded 214 | if (e === 88 && this.starter.retry_on_rate_limit_exceeded) { 215 | console.log(`Rate limit exceeded for worker`); 216 | 217 | // Reset des var de boucle 218 | promises = []; 219 | 220 | // Attends 5 minutes avant de recommencer 221 | await sleep(1000 * 60 * 5); 222 | 223 | current.done = 0; 224 | current.failed = 0; 225 | last_errors = {}; 226 | 227 | continue; 228 | } 229 | } 230 | 231 | // Emet le message d'avancement 232 | this.message_port.postMessage({ 233 | type: 'info', 234 | info: { done: current.done, failed: current.failed } 235 | }); 236 | 237 | promises = []; 238 | current.done = 0; 239 | current.failed = 0; 240 | 241 | // Signale les erreurs rencontrées 242 | if (Object.keys(last_errors).length) { 243 | this.message_port.postMessage({ 244 | type: "twitter_error", error: last_errors 245 | }); 246 | } 247 | last_errors = {}; 248 | 249 | chunk = this.ids.slice(current_i, current_i + chunk_len); 250 | current_i += chunk_len; 251 | } 252 | } 253 | 254 | stop() { 255 | this.stopped = true; 256 | } 257 | } 258 | 259 | function sleep(ms: number) { 260 | return new Promise(resolve => setTimeout(resolve, ms)); 261 | } 262 | 263 | -------------------------------------------------------------------------------- /src/twitter_lite_clone/twitter_lite.ts: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | import Fetch from 'cross-fetch'; 3 | const OAuth = require('oauth-1.0a'); 4 | import querystring from 'querystring'; 5 | 6 | const getUrl = (subdomain: string, endpoint = '1.1') => 7 | `https://${subdomain}.twitter.com/${endpoint}`; 8 | 9 | const createOauthClient = ({ key, secret }: { key: string, secret: string }) => { 10 | const client = OAuth({ 11 | consumer: { key, secret }, 12 | signature_method: 'HMAC-SHA1', 13 | hash_function(baseString: string, key: string) { 14 | return crypto 15 | .createHmac('sha1', key) 16 | .update(baseString) 17 | .digest('base64'); 18 | }, 19 | }); 20 | 21 | return client; 22 | }; 23 | 24 | const defaults: { 25 | subdomain?: string, 26 | consumer_key?: string | null, 27 | consumer_secret?: string | null, 28 | access_token_key?: string | null, 29 | access_token_secret?: string | null, 30 | bearer_token?: string | null, 31 | } = { 32 | subdomain: 'api', 33 | consumer_key: null, 34 | consumer_secret: null, 35 | access_token_key: null, 36 | access_token_secret: null, 37 | bearer_token: null, 38 | }; 39 | 40 | type TwitterConstructType = typeof defaults; 41 | 42 | // Twitter expects POST body parameters to be URL-encoded: https://developer.twitter.com/en/docs/basics/authentication/guides/creating-a-signature 43 | // However, some endpoints expect a JSON payload - https://developer.twitter.com/en/docs/direct-messages/sending-and-receiving/api-reference/new-event 44 | // It appears that JSON payloads don't need to be included in the signature, 45 | // because sending DMs works without signing the POST body 46 | const JSON_ENDPOINTS = [ 47 | 'direct_messages/events/new', 48 | 'direct_messages/welcome_messages/new', 49 | 'direct_messages/welcome_messages/rules/new', 50 | ]; 51 | 52 | const baseHeaders = { 53 | 'Content-Type': 'application/json', 54 | Accept: 'application/json', 55 | }; 56 | 57 | function percentEncode(string: string) { 58 | // From OAuth.prototype.percentEncode 59 | return string 60 | .replace(/!/g, '%21') 61 | .replace(/\*/g, '%2A') 62 | .replace(/'/g, '%27') 63 | .replace(/\(/g, '%28') 64 | .replace(/\)/g, '%29'); 65 | } 66 | 67 | export default class Twitter { 68 | public authType: string; 69 | public client: any; 70 | public token: { key: string, secret: string }; 71 | public url: string; 72 | public oauth: string; 73 | public config: TwitterConstructType; 74 | 75 | constructor(options: TwitterConstructType) { 76 | const config = Object.assign({}, defaults, options); 77 | this.authType = config.bearer_token ? 'App' : 'User'; 78 | this.client = createOauthClient({ 79 | key: config.consumer_key!, 80 | secret: config.consumer_secret!, 81 | }); 82 | 83 | this.token = { 84 | key: config.access_token_key!, 85 | secret: config.access_token_secret!, 86 | }; 87 | 88 | this.url = getUrl(config.subdomain!); 89 | this.oauth = getUrl(config.subdomain!, 'oauth'); 90 | this.config = config; 91 | } 92 | 93 | /** 94 | * Parse the JSON from a Response object and add the Headers under `_headers` 95 | * @param {Response} response - the Response object returned by Fetch 96 | * @return {Promise} 97 | * @private 98 | */ 99 | static _handleResponse(response: Response) { 100 | // @ts-ignore 101 | const headers = response.headers.raw(); // TODO: see #44 102 | // Return empty response on 204 "No content" 103 | if (response.status === 204) 104 | return { 105 | _headers: headers, 106 | }; 107 | // Otherwise, parse JSON response 108 | return response.json().then(res => { 109 | res._headers = headers; // TODO: this creates an array-like object when it adds _headers to an array response 110 | return res; 111 | }); 112 | } 113 | 114 | async getBearerToken() { 115 | const headers = { 116 | Authorization: 117 | 'Basic ' + 118 | Buffer.from( 119 | this.config.consumer_key + ':' + this.config.consumer_secret 120 | ).toString('base64'), 121 | 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', 122 | }; 123 | 124 | const results = await Fetch('https://api.twitter.com/oauth2/token', { 125 | method: 'POST', 126 | body: 'grant_type=client_credentials', 127 | headers, 128 | }).then(Twitter._handleResponse); 129 | 130 | return results; 131 | } 132 | 133 | async getRequestToken(twitterCallbackUrl: string) { 134 | const requestData = { 135 | url: `${this.oauth}/request_token`, 136 | method: 'POST', 137 | }; 138 | 139 | let parameters = {}; 140 | if (twitterCallbackUrl) parameters = { oauth_callback: twitterCallbackUrl }; 141 | if (parameters) requestData.url += '?' + querystring.stringify(parameters); 142 | 143 | const headers = this.client.toHeader( 144 | this.client.authorize(requestData, {}) 145 | ); 146 | 147 | const results = await Fetch(requestData.url, { 148 | method: 'POST', 149 | headers: Object.assign({}, baseHeaders, headers), 150 | }) 151 | .then(res => res.text()) 152 | .then(txt => querystring.parse(txt)); 153 | 154 | return results; 155 | } 156 | 157 | async getAccessToken(options: { verifier: string, key: string, secret: string }) { 158 | const requestData = { 159 | url: `${this.oauth}/access_token`, 160 | method: 'POST', 161 | }; 162 | 163 | let parameters = { oauth_verifier: options.verifier }; 164 | if (parameters) requestData.url += '?' + querystring.stringify(parameters); 165 | 166 | const headers = this.client.toHeader( 167 | this.client.authorize(requestData, { 168 | key: options.key, 169 | secret: options.secret, 170 | }) 171 | ); 172 | 173 | const results = await Fetch(requestData.url, { 174 | method: 'POST', 175 | headers: Object.assign({}, baseHeaders, headers), 176 | }) 177 | .then(res => res.text()) 178 | .then(txt => querystring.parse(txt)); 179 | 180 | return results; 181 | } 182 | 183 | /** 184 | * Construct the data and headers for an authenticated HTTP request to the Twitter API 185 | * @param {string} method - 'GET' or 'POST' 186 | * @param {string} resource - the API endpoint 187 | * @param {object} parameters 188 | * @param {boolean} rawRequest 189 | * @return {{requestData: {url: string, method: string}, headers: ({Authorization: string}|OAuth.Header)}} 190 | * @private 191 | */ 192 | _makeRequest(method: string, resource: string, parameters?: any, rawRequest = false) { 193 | const requestData: any = { 194 | url: rawRequest ? resource : `${this.url}/${resource}.json`, 195 | method, 196 | }; 197 | if (parameters) 198 | if (method === 'POST') requestData.data = parameters; 199 | else requestData.url += '?' + querystring.stringify(parameters); 200 | 201 | let headers = {}; 202 | if (this.authType === 'User') { 203 | headers = this.client.toHeader( 204 | this.client.authorize(requestData, this.token) 205 | ); 206 | } else { 207 | headers = { 208 | Authorization: `Bearer ${this.config.bearer_token}`, 209 | }; 210 | } 211 | return { 212 | requestData, 213 | headers, 214 | }; 215 | } 216 | 217 | /** 218 | * Fetch a DM image from the given URL. 219 | * @param {string} url 220 | * @returns {Response} 221 | */ 222 | dmImage(url: string) { 223 | const { requestData, headers } = this._makeRequest( 224 | 'GET', 225 | url, 226 | undefined, 227 | true 228 | ); 229 | 230 | return Fetch(requestData.url, { headers }); 231 | } 232 | 233 | /** 234 | * Send a GET request 235 | * @param {string} resource - endpoint, e.g. `followers/ids` 236 | * @param {object} [parameters] - optional parameters 237 | * @returns {Promise} Promise resolving to the response from the Twitter API. 238 | * The `_header` property will be set to the Response headers (useful for checking rate limits) 239 | */ 240 | get(resource: string, parameters?: any) { 241 | const { requestData, headers } = this._makeRequest( 242 | 'GET', 243 | resource, 244 | parameters 245 | ); 246 | 247 | return Fetch(requestData.url, { headers }) 248 | .then(Twitter._handleResponse) 249 | .then(results => 250 | 'errors' in results ? Promise.reject(results) : results 251 | ); 252 | } 253 | 254 | /** 255 | * Send a DELETE request 256 | * @param {string} resource - endpoint, e.g. `followers/ids` 257 | * @param {object} [parameters] - optional parameters 258 | * @returns {Promise} Promise resolving to the response from the Twitter API. 259 | * The `_header` property will be set to the Response headers (useful for checking rate limits) 260 | */ 261 | delete(resource: string, parameters?: any) { 262 | const { requestData, headers } = this._makeRequest( 263 | 'DELETE', 264 | resource, 265 | parameters 266 | ); 267 | 268 | return Fetch(requestData.url, { headers }) 269 | .then(Twitter._handleResponse) 270 | .then(results => 271 | 'errors' in results ? Promise.reject(results) : results 272 | ); 273 | } 274 | 275 | /** 276 | * Send a POST request 277 | * @param {string} resource - endpoint, e.g. `users/lookup` 278 | * @param {object} body - POST parameters object. 279 | * Will be encoded appropriately (JSON or urlencoded) based on the resource 280 | * @returns {Promise} Promise resolving to the response from the Twitter API. 281 | * The `_header` property will be set to the Response headers (useful for checking rate limits) 282 | */ 283 | post(resource: string, body?: any) { 284 | const { requestData, headers } = this._makeRequest( 285 | 'POST', 286 | resource, 287 | JSON_ENDPOINTS.includes(resource) ? null : body // don't sign JSON bodies; only parameters 288 | ); 289 | 290 | const postHeaders = Object.assign({}, baseHeaders, headers); 291 | if (JSON_ENDPOINTS.includes(resource)) { 292 | body = JSON.stringify(body); 293 | } else { 294 | body = percentEncode(querystring.stringify(body)); 295 | postHeaders['Content-Type'] = 'application/x-www-form-urlencoded'; 296 | } 297 | 298 | return Fetch(requestData.url, { 299 | method: 'POST', 300 | headers: postHeaders, 301 | body, 302 | }) 303 | .then(Twitter._handleResponse) 304 | .then(results => 305 | 'errors' in results ? Promise.reject(results) : results 306 | ); 307 | } 308 | } 309 | 310 | module.exports = Twitter; 311 | -------------------------------------------------------------------------------- /src/api/tasks/Task.ts: -------------------------------------------------------------------------------- 1 | import { Socket } from "socket.io"; 2 | import { Worker } from 'worker_threads'; 3 | import logger from "../../logger"; 4 | import { CONSUMER_KEY, CONSUMER_SECRET } from "../../twitter_const"; 5 | import { TweetCounter } from "../../constants"; 6 | import Timer from 'timerize'; 7 | 8 | Timer.default_format = "s"; 9 | 10 | export interface TaskProgression { 11 | percentage: number; 12 | done: number; 13 | remaining: number; 14 | failed: number; 15 | total: number; 16 | id: string; 17 | error?: string; 18 | type: TaskType; 19 | } 20 | 21 | export interface TaskEnd { 22 | id: string; 23 | type: TaskType; 24 | errors?: { [code: string]: [number, string] }; 25 | } 26 | 27 | interface WorkerTask { 28 | type: "task" | "stop", 29 | credentials: TwitterCredentials, 30 | tweets: string[], 31 | task_type: TaskType, 32 | debug?: boolean, 33 | } 34 | 35 | interface WorkerMessage { 36 | type: string; 37 | // SINCE THE LAST MESSAGE 38 | info?: { 39 | done: number; 40 | failed: number; 41 | }; 42 | error?: any; 43 | } 44 | 45 | interface TwitterCredentials { 46 | consumer_token: string; 47 | consumer_secret: string; 48 | oauth_token: string; 49 | oauth_token_secret: string; 50 | } 51 | 52 | interface Credentials { 53 | user_id: string; 54 | oauth_token: string; 55 | oauth_token_secret: string; 56 | screen_name?: string; 57 | } 58 | 59 | export type TaskType = "tweet" | "mute" | "block" | "fav" | "dm"; 60 | 61 | export function isValidTaskType(type: string) : type is TaskType { 62 | return type === "tweet" || type === "mute" || type === "block" || type === "fav" || type === "dm"; 63 | } 64 | 65 | export default class Task { 66 | // @ts-ignore 67 | protected static current_id = 1n; 68 | // Key is Task ID 69 | protected static readonly tasks_to_objects: Map = new Map; 70 | protected static readonly users_to_tasks: Map> = new Map; 71 | 72 | static readonly DEFAULT_THREAD_NUMBER = 2; 73 | 74 | // STATIC METHODS 75 | static get(id: string | BigInt) { 76 | if (typeof id === 'string') { 77 | id = BigInt(id); 78 | } 79 | 80 | return this.tasks_to_objects.get(id); 81 | } 82 | 83 | static exists(id: string | BigInt) { 84 | return this.get(id) !== undefined; 85 | } 86 | 87 | static tasksOf(user_id: string) { 88 | if (this.users_to_tasks.has(user_id)) { 89 | return this.users_to_tasks.get(user_id)!; 90 | } 91 | 92 | return new Set(); 93 | } 94 | 95 | static typeOf(type: TaskType, user_id: string) { 96 | const tasks = this.tasksOf(user_id); 97 | 98 | const t = new Set(); 99 | 100 | for (const task of tasks) { 101 | if (task.type === type) { 102 | t.add(task); 103 | } 104 | } 105 | 106 | return t; 107 | } 108 | 109 | static get count() { 110 | return this.tasks_to_objects.size; 111 | } 112 | 113 | static *all() { 114 | for (const [, tasks] of this.users_to_tasks) { 115 | yield* tasks; 116 | } 117 | } 118 | 119 | protected static register(task: Task) { 120 | this.tasks_to_objects.set(task.id, task); 121 | 122 | // USER TASK 123 | if (!this.users_to_tasks.has(task.owner)) { 124 | this.users_to_tasks.set(task.owner, new Set); 125 | } 126 | this.users_to_tasks.get(task.owner)!.add(task); 127 | } 128 | 129 | protected static unregister(task: Task) { 130 | this.tasks_to_objects.delete(task.id); 131 | 132 | // USER TASK 133 | const tasks = this.users_to_tasks.get(task.owner); 134 | if (tasks) { 135 | tasks.delete(task); 136 | 137 | if (!tasks.size) { 138 | this.users_to_tasks.delete(task.owner); 139 | } 140 | } 141 | } 142 | 143 | // INSTANCE PROPERTIES & METHODS 144 | 145 | readonly id: BigInt; 146 | 147 | protected sockets: Set = new Set; 148 | 149 | protected pool: Worker[] = []; 150 | 151 | protected done = 0; 152 | protected remaining = 0; 153 | protected failed = 0; 154 | 155 | protected timer?: Timer = new Timer; 156 | 157 | protected last: TaskProgression; 158 | 159 | /** 160 | * Log Twitter errors encountered during execution (code (as string) => [number of occurences, message for code]) 161 | */ 162 | protected twitter_errors_encountered: { [code: string]: [number, string] } = {}; 163 | 164 | constructor( 165 | items_ids: string[], 166 | protected user: Credentials, 167 | public readonly type: TaskType, 168 | thread_number: number = Task.DEFAULT_THREAD_NUMBER, 169 | ) { 170 | // Auto increment internal ID 171 | const c = Task.current_id; 172 | Task.current_id++; 173 | this.id = c; 174 | 175 | logger.info(`Creation of task #${this.id}, type ${type} for user @${user.screen_name} (${items_ids.length} elements)`); 176 | 177 | this.last = { 178 | id: String(this.id), 179 | remaining: items_ids.length, 180 | done: 0, 181 | failed: 0, 182 | percentage: 0, 183 | total: items_ids.length, 184 | type: this.type 185 | }; 186 | 187 | this.remaining = items_ids.length; 188 | 189 | // Register task 190 | Task.register(this); 191 | 192 | // Spawn worker thread(s)... 193 | // Découpage en {thread_number} parties le tableau de tweets 194 | if (items_ids.length <= thread_number ||items_ids.length < 50) { 195 | // Si il y a moins d'items que de threads, alors on lance un seul thread (y'en a pas beaucoup) 196 | // Ou alors si il y a peu d'items 197 | this.startWorker(items_ids); 198 | } 199 | else { 200 | const chunk_length = Math.ceil(items_ids.length / thread_number); 201 | let i = 0; 202 | let items_ids_part: string[]; 203 | 204 | while ((items_ids_part = items_ids.slice(i, i + chunk_length)).length) { 205 | this.startWorker(items_ids_part); 206 | i += chunk_length; 207 | } 208 | } 209 | } 210 | 211 | subscribe(socket: Socket) { 212 | this.sockets.add(socket); 213 | socket.emit('progression', this.last); 214 | } 215 | 216 | unsubscribe(socket: Socket) { 217 | this.sockets.delete(socket); 218 | } 219 | 220 | clearSubs() { 221 | this.sockets.clear(); 222 | } 223 | 224 | cancel() { 225 | logger.debug("Canceling task", this.id); 226 | this.sendMessageToSockets('task cancel', { 227 | id: String(this.id), 228 | type: this.type 229 | }); 230 | 231 | this.end(false); 232 | } 233 | 234 | end(with_end_message = true) { 235 | for (const worker of this.pool) { 236 | worker.removeAllListeners(); 237 | } 238 | 239 | // Send end message to sockets 240 | if (with_end_message) { 241 | this.sendMessageToSockets('task end', { 242 | id: String(this.id), 243 | type: this.type, 244 | errors: this.twitter_errors_encountered, 245 | }); 246 | } 247 | 248 | logger.debug("Terminating workers"); 249 | // Send stop message to workers then terminate 250 | for (const worker of this.pool) { 251 | worker.postMessage({ type: 'stop' }); 252 | process.nextTick(() => worker.terminate()); 253 | } 254 | 255 | // Empty pool of workers 256 | this.pool = []; 257 | 258 | this.clearSubs(); 259 | 260 | // Unregister task from Maps 261 | Task.unregister(this); 262 | 263 | let computed_stats = `${this.done} ok${this.failed ? `, ${this.failed} failed` : ""} over ${this.length}`; 264 | if (this.remaining) { 265 | computed_stats += ` (Remaining ${this.remaining})`; 266 | } 267 | 268 | logger.info(`Task #${this.id} of type ${this.type} from @${this.user.screen_name} has ended. Taken ${this.timer?.elapsed}s. ${computed_stats}`); 269 | this.timer = undefined; 270 | 271 | if (this.has_twitter_errors_encountered) { 272 | logger.warn(`Task #${this.id}: Twitter errors has been encountered: ${ 273 | Object.entries(this.twitter_errors_encountered) 274 | .map(([code, count]) => `code ${code} '${count[1]}' [${count[0]} times]`) 275 | .join(', ') 276 | }`); 277 | } 278 | } 279 | 280 | get current_progression() { 281 | return this.last; 282 | } 283 | 284 | get owner() { 285 | return this.user.user_id; 286 | } 287 | 288 | get owner_screen_name() { 289 | return this.user.screen_name!; 290 | } 291 | 292 | get worker_count() { 293 | return this.pool.length; 294 | } 295 | 296 | get length() { 297 | return this.done + this.remaining + this.failed; 298 | } 299 | 300 | get has_twitter_errors_encountered() { 301 | return Object.keys(this.twitter_errors_encountered).length > 0; 302 | } 303 | 304 | protected startWorker(items: string[]) { 305 | logger.silly(`Task #${this.id}: Starting worker ${this.pool.length + 1} with ${items.length} items.`); 306 | 307 | const worker = new Worker(__dirname + '/task_worker/worker.js'); 308 | const task_to_worker: WorkerTask = { 309 | tweets: items, 310 | credentials: { 311 | consumer_token: CONSUMER_KEY, 312 | consumer_secret: CONSUMER_SECRET, 313 | oauth_token: this.user.oauth_token, 314 | oauth_token_secret: this.user.oauth_token_secret 315 | }, 316 | type: "task", 317 | task_type: this.type, 318 | debug: this.debug_mode, 319 | }; 320 | 321 | // Assignation des listeners 322 | worker.on('message', this.onWorkerMessage(worker)); 323 | 324 | // Envoi de la tâche quand le worker est prêt 325 | worker.once('online', () => { 326 | worker.postMessage(task_to_worker); 327 | }); 328 | 329 | this.pool.push(worker); 330 | } 331 | 332 | protected onWorkerMessage(worker: Worker) { 333 | return (data: WorkerMessage) => { 334 | logger.silly("Recieved message from worker:", data); 335 | 336 | if (data.type === "info") { 337 | // Envoi d'un message de progression de la suppression 338 | this.done += data.info!.done; 339 | 340 | // Incrémente le compteur si la tâche est de type tweet 341 | if (this.type === "tweet") 342 | TweetCounter.inc(data.info!.done); 343 | 344 | this.remaining -= (data.info!.done + data.info!.failed); 345 | this.failed += data.info!.failed; 346 | 347 | this.emitProgress(this.done, this.remaining, this.failed); 348 | } 349 | else if (data.type === "end") { 350 | // End if all workers end 351 | this.pool = this.pool.filter(w => w !== worker); 352 | logger.verbose(`Terminating a worker on task #${this.id}.`); 353 | process.nextTick(() => worker.terminate()); 354 | 355 | if (this.pool.length === 0) { 356 | // All is over ! 357 | this.end(); 358 | } 359 | } 360 | else if (data.type === "error") { 361 | this.emitError(data.error); 362 | // Termine le worker 363 | this.end(false); 364 | } 365 | else if (data.type === "misc") { 366 | logger.debug("Worker misc data", data); 367 | } 368 | else if (data.type === "twitter_error") { 369 | const errors = data.error as {[code: string]: [number, string]}; 370 | 371 | for (const error in errors) { 372 | if (error in this.twitter_errors_encountered) { 373 | this.twitter_errors_encountered[error][0] += errors[error][0]; 374 | } 375 | else { 376 | this.twitter_errors_encountered[error] = errors[error]; 377 | } 378 | } 379 | } 380 | }; 381 | } 382 | 383 | protected emit(progression: TaskProgression) { 384 | this.last = progression; 385 | this.sendMessageToSockets('progression', progression); 386 | } 387 | 388 | protected sendMessageToSockets(name: string, message: any) { 389 | logger.debug(`Sending message ${name} to all sockets for task ${this.id}`, message); 390 | 391 | for (const s of this.sockets) { 392 | s.emit(name, message); 393 | } 394 | } 395 | 396 | protected emitProgress(done: number, remaining: number, failed: number) { 397 | const total = done + remaining + failed; 398 | 399 | this.emit({ 400 | done, 401 | remaining, 402 | id: String(this.id), 403 | failed, 404 | total, 405 | percentage: ((done + failed) / total) * 100, 406 | type: this.type 407 | }); 408 | } 409 | 410 | protected emitError(reason = "Unknown error") { 411 | logger.warn(`Error in worker for task #${this.id}: ${reason}`); 412 | this.emit({ 413 | done: 0, 414 | remaining: 0, 415 | id: String(this.id), 416 | total: this.length, 417 | failed: 0, 418 | percentage: 0, 419 | error: reason, 420 | type: this.type 421 | }); 422 | } 423 | 424 | protected get debug_mode() { 425 | return logger.level === "debug"; 426 | } 427 | } -------------------------------------------------------------------------------- /src/helpers.ts: -------------------------------------------------------------------------------- 1 | import { UserModel, TokenModel, IUser, TweetModel, ITweet, TwitterUserModel, ITwitterUser, IToken, CloudedArchiveModel, ICloudedArchive } from "./models"; 2 | import { SECRET_PRIVATE_KEY, SECRET_PASSPHRASE, SECRET_PUBLIC_KEY } from "./constants"; 3 | import JsonWebToken from 'jsonwebtoken'; 4 | import TwitterLite from "./twitter_lite_clone/twitter_lite"; 5 | import { CONSUMER_KEY, CONSUMER_SECRET } from "./twitter_const"; 6 | import express from 'express'; 7 | import Mongoose from "mongoose"; 8 | import AEError, { sendError } from "./errors"; 9 | import { Status, FullUser } from "twitter-d"; 10 | import { TokenPayload, JSONWebToken } from "./interfaces"; 11 | import logger from "./logger"; 12 | import * as fs from "fs"; 13 | import { UPLOAD_PATH } from "./uploader"; 14 | 15 | export function methodNotAllowed(allow: string | string[]) { 16 | return (_: any, res: express.Response) => { 17 | res.setHeader('Allow', typeof allow === 'string' ? allow : allow.join(', ')); 18 | sendError(AEError.invalid_method, res); 19 | }; 20 | } 21 | 22 | export function sanitizeMongoObj(data: T) : any { 23 | try { 24 | const original_clean = 'toJSON' in data ? data.toJSON() : data; 25 | 26 | for (const prop in original_clean) { 27 | if (prop.startsWith('_')) { 28 | // @ts-ignore 29 | delete original_clean[prop]; 30 | } 31 | } 32 | 33 | return original_clean; 34 | } catch {} 35 | 36 | return data; 37 | } 38 | 39 | export function getTokenInstanceFromString(token: string) : Promise { 40 | return TokenModel.findOne({ token }); 41 | } 42 | 43 | export function getTokensFromUser(user_id: string) : Promise { 44 | return TokenModel.find({ user_id }); 45 | } 46 | 47 | export function getCompleteUserFromId(user_id: string) : Promise { 48 | return UserModel.findOne({ user_id }); 49 | } 50 | 51 | export function getCompleteUserFromTwitterId(twitter_id: string) : Promise { 52 | return UserModel.findOne({ twitter_id }); 53 | } 54 | 55 | export function getCompleteUserFromTwitterScreenName(twitter_screen_name: string) : Promise { 56 | return UserModel.findOne({ twitter_screen_name: { $regex: "^" + twitter_screen_name + "$", $options: "i" }}); 57 | } 58 | 59 | export function batchTweets(ids: string[]) : Promise { 60 | return TweetModel.find({ id_str: { $in: ids } }) 61 | .then((statuses: ITweet[]) => { 62 | const current_date_minus = new Date; 63 | // Expiration: 2 semaines 64 | current_date_minus.setDate(current_date_minus.getDate() - (2 * 7)); 65 | 66 | // Check the tweets that are obsoletes 67 | statuses = statuses.filter(e => e.inserted_time.getTime() >= current_date_minus.getTime()); 68 | 69 | // Delete obsoletes tweets 70 | TweetModel.deleteMany({ inserted_time: { $lte: current_date_minus } }); 71 | 72 | // Return valids 73 | logger.debug(`${statuses.length} valid tweets batched from MongoDB (${ids.length} fetched)`); 74 | return statuses.map(s => s.toObject()); 75 | }); 76 | } 77 | 78 | export function batchUsers(ids: string[], as_screen_names = false) { 79 | let user_prom: Mongoose.DocumentQuery; 80 | if (as_screen_names) { 81 | user_prom = TwitterUserModel.find({ screen_name: { 82 | $regex: new RegExp('(^' + ids.join('$)|(^') + '$)'), 83 | $options: "i" 84 | }}); 85 | } 86 | else { 87 | user_prom = TwitterUserModel.find({ id_str: { $in: ids } }); 88 | } 89 | 90 | return user_prom 91 | .then(users => { 92 | const current_date_minus = new Date; 93 | // Expiration: 1 jour 94 | current_date_minus.setDate(current_date_minus.getDate() - 1); 95 | 96 | // Check the tweets that are obsoletes 97 | users = users.filter(e => e.inserted_time.getTime() >= current_date_minus.getTime()); 98 | 99 | // Delete obsoletes tweets 100 | TwitterUserModel.deleteMany({ inserted_time: { $lte: current_date_minus } }); 101 | 102 | // Return valids 103 | logger.debug(`${users.length} valid Twitter users batched from MongoDB (${ids.length} fetched)`); 104 | return users.map(u => u.toObject() as ITwitterUser); 105 | }); 106 | } 107 | 108 | export async function saveTweets(tweets: Status[]) { 109 | if (!tweets.length) { 110 | return []; 111 | } 112 | 113 | logger.debug(`Saving ${tweets.length} tweets in database.`); 114 | // Delete tweets in DB that are already existant 115 | await TweetModel.deleteMany({ id_str: { $in: tweets.map(t => t.id_str) } }) 116 | 117 | return TweetModel.insertMany( 118 | tweets 119 | .map(e => suppressUselessTweetProperties(e)) 120 | .map(t => ({ ...t, inserted_time: new Date })) 121 | ); 122 | } 123 | 124 | export async function saveTwitterUsers(users: FullUser[]) { 125 | if (!users.length) { 126 | return []; 127 | } 128 | 129 | logger.debug(`Saving ${users.length} twitter users in database (${users.map(u => `@${u.screen_name}`)}).`); 130 | // Delete users in DB that are already existant (maybe useless and slow) 131 | await TwitterUserModel.deleteMany({ id_str: { $in: users.map(t => t.id_str) } }) 132 | 133 | return TwitterUserModel.insertMany( 134 | users 135 | .map(u => suppressUselessTUserProperties(u)) 136 | .map(u => ({ ...u, inserted_time: new Date })) 137 | ); 138 | } 139 | 140 | export function invalidateToken(token: string) { 141 | return TokenModel.deleteOne({ token }); 142 | } 143 | 144 | export function invalidateTokensFromUser(user_id: string) { 145 | return TokenModel.deleteMany({ user_id }); 146 | } 147 | 148 | export function isTokenInvalid(token: string, req?: express.Request, full_payload?: JSONWebToken) { 149 | return getTokenInstanceFromString(token) 150 | .then(async model => { 151 | if (model) { 152 | // Actualise le last_use 153 | model.last_use = new Date; 154 | if (req) { 155 | model.login_ip = req.connection.remoteAddress!; 156 | } 157 | // Sauvegarde sans considérer une erreur 158 | model.save().catch(e => e); 159 | 160 | // Test if token should be renewed 161 | if (full_payload && req) { 162 | const exp = new Date(Number(full_payload.exp) * 1000); 163 | const now = new Date; 164 | 165 | // If token is not expired 166 | if (exp.getTime() >= now.getTime()) { 167 | now.setDate(now.getDate() + 14); 168 | 169 | // Token expires in less than 14 days 170 | if (exp.getTime() < now.getTime()) { 171 | try { 172 | const token = await makeTokenFromUser( 173 | full_payload.jti, 174 | full_payload.user_id, 175 | full_payload.screen_name, 176 | full_payload.login_ip 177 | ); 178 | 179 | req.__ask_refresh__ = token; 180 | } catch (e) { 181 | logger.error("Unable to create renewed token:", e); 182 | } 183 | } 184 | } 185 | else { 186 | return true; 187 | } 188 | } 189 | 190 | return false; 191 | } 192 | return true; 193 | }) 194 | .catch(() => true); 195 | } 196 | 197 | export async function checkToken(token: string) { 198 | const decoded: JSONWebToken = await new Promise((resolve, reject) => { 199 | JsonWebToken.verify(token, SECRET_PUBLIC_KEY, (err, data) => { 200 | if (err) reject(err); 201 | resolve(data as any); 202 | }); 203 | }); 204 | 205 | if (await isTokenInvalid(decoded.jti)) { 206 | return undefined; 207 | } 208 | return decoded; 209 | } 210 | 211 | export function removeUser(user: IUser) { 212 | invalidateTokensFromUser(user.user_id); 213 | return user.remove(); 214 | } 215 | 216 | export function signToken(payload: TokenPayload, id: string) { 217 | return new Promise((resolve, reject) => { 218 | // Signe le token 219 | JsonWebToken.sign( 220 | payload, // Données custom 221 | { key: SECRET_PRIVATE_KEY, passphrase: SECRET_PASSPHRASE }, // Clé RSA privée 222 | { 223 | algorithm: 'RS256', 224 | expiresIn: "90d", // 3 months durability 225 | issuer: "Archive Explorer Server 1", 226 | jwtid: id, // ID généré avec uuid 227 | }, 228 | (err, encoded) => { // Quand le token est généré (ou non), accepte/rejette la promesse 229 | if (err) reject(err); 230 | else resolve(encoded as string); 231 | } 232 | ); 233 | }) as Promise; 234 | } 235 | 236 | export function makeTokenFromUser(jti: string, user_id: string, screen_name: string, login_ip: string) { 237 | return signToken({ 238 | user_id, 239 | screen_name, 240 | login_ip 241 | }, jti); 242 | } 243 | 244 | export function createTwitterObjectFromUser(user: IUser) { 245 | return new TwitterLite({ 246 | consumer_key: CONSUMER_KEY, 247 | consumer_secret: CONSUMER_SECRET, 248 | access_token_key: user.oauth_token, 249 | access_token_secret: user.oauth_token_secret 250 | }); 251 | } 252 | 253 | export function suppressUselessTweetProperties(tweet: Status) { 254 | delete tweet.contributors; 255 | delete tweet.coordinates; 256 | delete tweet.current_user_retweet; 257 | // @ts-expect-error 258 | delete tweet.favorited; 259 | delete tweet.place; 260 | // @ts-expect-error 261 | delete tweet.retweeted; 262 | delete tweet.scopes; 263 | delete tweet.withheld_copyright; 264 | delete tweet.withheld_in_countries; 265 | delete tweet.withheld_scope; 266 | 267 | if (tweet.quoted_status) { 268 | tweet.quoted_status = suppressUselessTweetProperties(tweet.quoted_status); 269 | } 270 | if (tweet.retweeted_status) { 271 | tweet.retweeted_status = suppressUselessTweetProperties(tweet.retweeted_status); 272 | } 273 | 274 | tweet.user = suppressUselessTUserProperties(tweet.user as FullUser); 275 | 276 | return tweet; 277 | } 278 | 279 | export function suppressUselessTUserProperties(user: FullUser) { 280 | // @ts-expect-error 281 | delete user.entities; 282 | // @ts-expect-error 283 | delete user.listed_count; 284 | delete user.status; 285 | delete user.withheld_in_countries; 286 | delete user.withheld_scope; 287 | // @ts-expect-error 288 | delete user.statuses_count; 289 | // @ts-ignore 290 | delete user.follow_request_sent; 291 | // @ts-expect-error 292 | delete user.default_profile_image; 293 | // @ts-expect-error 294 | delete user.default_profile; 295 | // @ts-ignore 296 | delete user.profile_background_image_url; 297 | // @ts-ignore 298 | delete user.profile_background_image_url_https; 299 | // @ts-ignore 300 | delete user.profile_background_tile; 301 | // @ts-ignore 302 | delete user.profile_background_color; 303 | // @ts-ignore 304 | delete user.geo_enabled; 305 | // @ts-ignore 306 | delete user.utc_offset; 307 | // @ts-ignore 308 | delete user.time_zone; 309 | // @ts-ignore 310 | delete user.contributors_enabled; 311 | // @ts-ignore 312 | delete user.is_translator; 313 | // @ts-ignore 314 | delete user.is_translation_enabled; 315 | // @ts-ignore 316 | delete user.profile_use_background_image; 317 | // @ts-ignore 318 | delete user.has_extended_profile; 319 | // @ts-ignore 320 | delete user.notifications; 321 | // @ts-ignore 322 | delete user.translator_type; 323 | 324 | return user; 325 | } 326 | 327 | export function sendTwitterError(e: any, res: express.Response) { 328 | if (e.errors) { 329 | if (e.errors[0].code === 32 || e.errors[0].code === 89 || e.errors[0].code === 99) { 330 | sendError(AEError.twitter_credentials_expired, res); 331 | } 332 | else if (e.errors[0].code === 88) { 333 | sendError(AEError.twitter_rate_limit, res); 334 | } 335 | else { 336 | sendError(AEError.twitter_error, res, e); 337 | } 338 | } 339 | else { 340 | sendError(AEError.server_error, res); 341 | } 342 | } 343 | 344 | export async function deleteUser(user_id: string) { 345 | // Invalidate all tokens 346 | await invalidateTokensFromUser(user_id); 347 | // Delete user 348 | await UserModel.deleteOne({ user_id }); 349 | // Delete clouded archives 350 | await deleteEveryCloudedArchive(user_id); 351 | } 352 | 353 | export async function deleteEveryCloudedArchive(user_id: string) { 354 | const archives = await CloudedArchiveModel.find({ user_id }) as ICloudedArchive[]; 355 | await Promise.all(archives.map(deleteCloudedArchive)); 356 | } 357 | 358 | export async function deleteCloudedArchive(archive: ICloudedArchive) { 359 | // Delete related ZIP file then model 360 | await fs.promises.unlink(UPLOAD_PATH + '/' + archive.path).catch(() => {}); 361 | await archive.remove(); 362 | } 363 | 364 | export async function changeSpecial(user_id: string, special = false) { 365 | const user = await getCompleteUserFromId(user_id); 366 | 367 | if (user) { 368 | user.special = special; 369 | user.save(); 370 | } 371 | } 372 | 373 | export async function purgeCollections(COLLECTIONS: any, db: any, mongoose: any) { 374 | const drops: Promise[] = []; 375 | for (const collection of Object.keys(COLLECTIONS)) { 376 | drops.push(db.db.dropCollection(collection) 377 | .then(() => console.log(`Collection ${collection} dropped.`)) 378 | .catch(() => logger.warn(`Unable to drop collection ${collection}. (maybe it hasn't been created yet)`))); 379 | } 380 | 381 | return Promise.all(drops).then(() => db.close()).then(() => mongoose.disconnect()); 382 | } 383 | 384 | export async function purgePartial(COLLECTIONS: any, db: any) { 385 | const drops: Promise[] = []; 386 | for (const collection of Object.keys(COLLECTIONS)) { 387 | drops.push(db.db.dropCollection(collection) 388 | .then(() => console.log(`Collection ${collection} dropped.`)) 389 | .catch(() => logger.warn(`Unable to drop collection ${collection}. (maybe it hasn't been created yet)`))); 390 | } 391 | 392 | return Promise.all(drops); 393 | } 394 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Attribution-NonCommercial-ShareAlike 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International 58 | Public License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-NonCommercial-ShareAlike 4.0 International Public License 63 | ("Public License"). To the extent this Public License may be 64 | interpreted as a contract, You are granted the Licensed Rights in 65 | consideration of Your acceptance of these terms and conditions, and the 66 | Licensor grants You such rights in consideration of benefits the 67 | Licensor receives from making the Licensed Material available under 68 | these terms and conditions. 69 | 70 | 71 | Section 1 -- Definitions. 72 | 73 | a. Adapted Material means material subject to Copyright and Similar 74 | Rights that is derived from or based upon the Licensed Material 75 | and in which the Licensed Material is translated, altered, 76 | arranged, transformed, or otherwise modified in a manner requiring 77 | permission under the Copyright and Similar Rights held by the 78 | Licensor. For purposes of this Public License, where the Licensed 79 | Material is a musical work, performance, or sound recording, 80 | Adapted Material is always produced where the Licensed Material is 81 | synched in timed relation with a moving image. 82 | 83 | b. Adapter's License means the license You apply to Your Copyright 84 | and Similar Rights in Your contributions to Adapted Material in 85 | accordance with the terms and conditions of this Public License. 86 | 87 | c. BY-NC-SA Compatible License means a license listed at 88 | creativecommons.org/compatiblelicenses, approved by Creative 89 | Commons as essentially the equivalent of this Public License. 90 | 91 | d. Copyright and Similar Rights means copyright and/or similar rights 92 | closely related to copyright including, without limitation, 93 | performance, broadcast, sound recording, and Sui Generis Database 94 | Rights, without regard to how the rights are labeled or 95 | categorized. For purposes of this Public License, the rights 96 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 97 | Rights. 98 | 99 | e. Effective Technological Measures means those measures that, in the 100 | absence of proper authority, may not be circumvented under laws 101 | fulfilling obligations under Article 11 of the WIPO Copyright 102 | Treaty adopted on December 20, 1996, and/or similar international 103 | agreements. 104 | 105 | f. Exceptions and Limitations means fair use, fair dealing, and/or 106 | any other exception or limitation to Copyright and Similar Rights 107 | that applies to Your use of the Licensed Material. 108 | 109 | g. License Elements means the license attributes listed in the name 110 | of a Creative Commons Public License. The License Elements of this 111 | Public License are Attribution, NonCommercial, and ShareAlike. 112 | 113 | h. Licensed Material means the artistic or literary work, database, 114 | or other material to which the Licensor applied this Public 115 | License. 116 | 117 | i. Licensed Rights means the rights granted to You subject to the 118 | terms and conditions of this Public License, which are limited to 119 | all Copyright and Similar Rights that apply to Your use of the 120 | Licensed Material and that the Licensor has authority to license. 121 | 122 | j. Licensor means the individual(s) or entity(ies) granting rights 123 | under this Public License. 124 | 125 | k. NonCommercial means not primarily intended for or directed towards 126 | commercial advantage or monetary compensation. For purposes of 127 | this Public License, the exchange of the Licensed Material for 128 | other material subject to Copyright and Similar Rights by digital 129 | file-sharing or similar means is NonCommercial provided there is 130 | no payment of monetary compensation in connection with the 131 | exchange. 132 | 133 | l. Share means to provide material to the public by any means or 134 | process that requires permission under the Licensed Rights, such 135 | as reproduction, public display, public performance, distribution, 136 | dissemination, communication, or importation, and to make material 137 | available to the public including in ways that members of the 138 | public may access the material from a place and at a time 139 | individually chosen by them. 140 | 141 | m. Sui Generis Database Rights means rights other than copyright 142 | resulting from Directive 96/9/EC of the European Parliament and of 143 | the Council of 11 March 1996 on the legal protection of databases, 144 | as amended and/or succeeded, as well as other essentially 145 | equivalent rights anywhere in the world. 146 | 147 | n. You means the individual or entity exercising the Licensed Rights 148 | under this Public License. Your has a corresponding meaning. 149 | 150 | 151 | Section 2 -- Scope. 152 | 153 | a. License grant. 154 | 155 | 1. Subject to the terms and conditions of this Public License, 156 | the Licensor hereby grants You a worldwide, royalty-free, 157 | non-sublicensable, non-exclusive, irrevocable license to 158 | exercise the Licensed Rights in the Licensed Material to: 159 | 160 | a. reproduce and Share the Licensed Material, in whole or 161 | in part, for NonCommercial purposes only; and 162 | 163 | b. produce, reproduce, and Share Adapted Material for 164 | NonCommercial purposes only. 165 | 166 | 2. Exceptions and Limitations. For the avoidance of doubt, where 167 | Exceptions and Limitations apply to Your use, this Public 168 | License does not apply, and You do not need to comply with 169 | its terms and conditions. 170 | 171 | 3. Term. The term of this Public License is specified in Section 172 | 6(a). 173 | 174 | 4. Media and formats; technical modifications allowed. The 175 | Licensor authorizes You to exercise the Licensed Rights in 176 | all media and formats whether now known or hereafter created, 177 | and to make technical modifications necessary to do so. The 178 | Licensor waives and/or agrees not to assert any right or 179 | authority to forbid You from making technical modifications 180 | necessary to exercise the Licensed Rights, including 181 | technical modifications necessary to circumvent Effective 182 | Technological Measures. For purposes of this Public License, 183 | simply making modifications authorized by this Section 2(a) 184 | (4) never produces Adapted Material. 185 | 186 | 5. Downstream recipients. 187 | 188 | a. Offer from the Licensor -- Licensed Material. Every 189 | recipient of the Licensed Material automatically 190 | receives an offer from the Licensor to exercise the 191 | Licensed Rights under the terms and conditions of this 192 | Public License. 193 | 194 | b. Additional offer from the Licensor -- Adapted Material. 195 | Every recipient of Adapted Material from You 196 | automatically receives an offer from the Licensor to 197 | exercise the Licensed Rights in the Adapted Material 198 | under the conditions of the Adapter's License You apply. 199 | 200 | c. No downstream restrictions. You may not offer or impose 201 | any additional or different terms or conditions on, or 202 | apply any Effective Technological Measures to, the 203 | Licensed Material if doing so restricts exercise of the 204 | Licensed Rights by any recipient of the Licensed 205 | Material. 206 | 207 | 6. No endorsement. Nothing in this Public License constitutes or 208 | may be construed as permission to assert or imply that You 209 | are, or that Your use of the Licensed Material is, connected 210 | with, or sponsored, endorsed, or granted official status by, 211 | the Licensor or others designated to receive attribution as 212 | provided in Section 3(a)(1)(A)(i). 213 | 214 | b. Other rights. 215 | 216 | 1. Moral rights, such as the right of integrity, are not 217 | licensed under this Public License, nor are publicity, 218 | privacy, and/or other similar personality rights; however, to 219 | the extent possible, the Licensor waives and/or agrees not to 220 | assert any such rights held by the Licensor to the limited 221 | extent necessary to allow You to exercise the Licensed 222 | Rights, but not otherwise. 223 | 224 | 2. Patent and trademark rights are not licensed under this 225 | Public License. 226 | 227 | 3. To the extent possible, the Licensor waives any right to 228 | collect royalties from You for the exercise of the Licensed 229 | Rights, whether directly or through a collecting society 230 | under any voluntary or waivable statutory or compulsory 231 | licensing scheme. In all other cases the Licensor expressly 232 | reserves any right to collect such royalties, including when 233 | the Licensed Material is used other than for NonCommercial 234 | purposes. 235 | 236 | 237 | Section 3 -- License Conditions. 238 | 239 | Your exercise of the Licensed Rights is expressly made subject to the 240 | following conditions. 241 | 242 | a. Attribution. 243 | 244 | 1. If You Share the Licensed Material (including in modified 245 | form), You must: 246 | 247 | a. retain the following if it is supplied by the Licensor 248 | with the Licensed Material: 249 | 250 | i. identification of the creator(s) of the Licensed 251 | Material and any others designated to receive 252 | attribution, in any reasonable manner requested by 253 | the Licensor (including by pseudonym if 254 | designated); 255 | 256 | ii. a copyright notice; 257 | 258 | iii. a notice that refers to this Public License; 259 | 260 | iv. a notice that refers to the disclaimer of 261 | warranties; 262 | 263 | v. a URI or hyperlink to the Licensed Material to the 264 | extent reasonably practicable; 265 | 266 | b. indicate if You modified the Licensed Material and 267 | retain an indication of any previous modifications; and 268 | 269 | c. indicate the Licensed Material is licensed under this 270 | Public License, and include the text of, or the URI or 271 | hyperlink to, this Public License. 272 | 273 | 2. You may satisfy the conditions in Section 3(a)(1) in any 274 | reasonable manner based on the medium, means, and context in 275 | which You Share the Licensed Material. For example, it may be 276 | reasonable to satisfy the conditions by providing a URI or 277 | hyperlink to a resource that includes the required 278 | information. 279 | 3. If requested by the Licensor, You must remove any of the 280 | information required by Section 3(a)(1)(A) to the extent 281 | reasonably practicable. 282 | 283 | b. ShareAlike. 284 | 285 | In addition to the conditions in Section 3(a), if You Share 286 | Adapted Material You produce, the following conditions also apply. 287 | 288 | 1. The Adapter's License You apply must be a Creative Commons 289 | license with the same License Elements, this version or 290 | later, or a BY-NC-SA Compatible License. 291 | 292 | 2. You must include the text of, or the URI or hyperlink to, the 293 | Adapter's License You apply. You may satisfy this condition 294 | in any reasonable manner based on the medium, means, and 295 | context in which You Share Adapted Material. 296 | 297 | 3. You may not offer or impose any additional or different terms 298 | or conditions on, or apply any Effective Technological 299 | Measures to, Adapted Material that restrict exercise of the 300 | rights granted under the Adapter's License You apply. 301 | 302 | 303 | Section 4 -- Sui Generis Database Rights. 304 | 305 | Where the Licensed Rights include Sui Generis Database Rights that 306 | apply to Your use of the Licensed Material: 307 | 308 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 309 | to extract, reuse, reproduce, and Share all or a substantial 310 | portion of the contents of the database for NonCommercial purposes 311 | only; 312 | 313 | b. if You include all or a substantial portion of the database 314 | contents in a database in which You have Sui Generis Database 315 | Rights, then the database in which You have Sui Generis Database 316 | Rights (but not its individual contents) is Adapted Material, 317 | including for purposes of Section 3(b); and 318 | 319 | c. You must comply with the conditions in Section 3(a) if You Share 320 | all or a substantial portion of the contents of the database. 321 | 322 | For the avoidance of doubt, this Section 4 supplements and does not 323 | replace Your obligations under this Public License where the Licensed 324 | Rights include other Copyright and Similar Rights. 325 | 326 | 327 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 328 | 329 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 330 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 331 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 332 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 333 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 334 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 335 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 336 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 337 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 338 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 339 | 340 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 341 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 342 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 343 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 344 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 345 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 346 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 347 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 348 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 349 | 350 | c. The disclaimer of warranties and limitation of liability provided 351 | above shall be interpreted in a manner that, to the extent 352 | possible, most closely approximates an absolute disclaimer and 353 | waiver of all liability. 354 | 355 | 356 | Section 6 -- Term and Termination. 357 | 358 | a. This Public License applies for the term of the Copyright and 359 | Similar Rights licensed here. However, if You fail to comply with 360 | this Public License, then Your rights under this Public License 361 | terminate automatically. 362 | 363 | b. Where Your right to use the Licensed Material has terminated under 364 | Section 6(a), it reinstates: 365 | 366 | 1. automatically as of the date the violation is cured, provided 367 | it is cured within 30 days of Your discovery of the 368 | violation; or 369 | 370 | 2. upon express reinstatement by the Licensor. 371 | 372 | For the avoidance of doubt, this Section 6(b) does not affect any 373 | right the Licensor may have to seek remedies for Your violations 374 | of this Public License. 375 | 376 | c. For the avoidance of doubt, the Licensor may also offer the 377 | Licensed Material under separate terms or conditions or stop 378 | distributing the Licensed Material at any time; however, doing so 379 | will not terminate this Public License. 380 | 381 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 382 | License. 383 | 384 | 385 | Section 7 -- Other Terms and Conditions. 386 | 387 | a. The Licensor shall not be bound by any additional or different 388 | terms or conditions communicated by You unless expressly agreed. 389 | 390 | b. Any arrangements, understandings, or agreements regarding the 391 | Licensed Material not stated herein are separate from and 392 | independent of the terms and conditions of this Public License. 393 | 394 | 395 | Section 8 -- Interpretation. 396 | 397 | a. For the avoidance of doubt, this Public License does not, and 398 | shall not be interpreted to, reduce, limit, restrict, or impose 399 | conditions on any use of the Licensed Material that could lawfully 400 | be made without permission under this Public License. 401 | 402 | b. To the extent possible, if any provision of this Public License is 403 | deemed unenforceable, it shall be automatically reformed to the 404 | minimum extent necessary to make it enforceable. If the provision 405 | cannot be reformed, it shall be severed from this Public License 406 | without affecting the enforceability of the remaining terms and 407 | conditions. 408 | 409 | c. No term or condition of this Public License will be waived and no 410 | failure to comply consented to unless expressly agreed to by the 411 | Licensor. 412 | 413 | d. Nothing in this Public License constitutes or may be interpreted 414 | as a limitation upon, or waiver of, any privileges and immunities 415 | that apply to the Licensor or You, including from the legal 416 | processes of any jurisdiction or authority. 417 | 418 | ======================================================================= 419 | 420 | Creative Commons is not a party to its public 421 | licenses. Notwithstanding, Creative Commons may elect to apply one of 422 | its public licenses to material it publishes and in those instances 423 | will be considered the “Licensor.” The text of the Creative Commons 424 | public licenses is dedicated to the public domain under the CC0 Public 425 | Domain Dedication. Except for the limited purpose of indicating that 426 | material is shared under a Creative Commons public license or as 427 | otherwise permitted by the Creative Commons policies published at 428 | creativecommons.org/policies, Creative Commons does not authorize the 429 | use of the trademark "Creative Commons" or any other trademark or logo 430 | of Creative Commons without its prior written consent including, 431 | without limitation, in connection with any unauthorized modifications 432 | to any of its public licenses or any other arrangements, 433 | understandings, or agreements concerning use of licensed material. For 434 | the avoidance of doubt, this paragraph does not form part of the 435 | public licenses. 436 | 437 | Creative Commons may be contacted at creativecommons.org. 438 | --------------------------------------------------------------------------------