新建任务
106 | 107 |242 | 243 | 244 | 245 | 警告：有部分用户报告，在备份过程中可能会触发豆瓣的反爬虫保护机制，从而导致当前账号被锁定。解锁操作需要使用注册手机发送短信。如果你无法执行解锁操作，请谨慎使用。
246 |
10 |
11 |
16 |
17 |
18 | 加入
19 | 备份数据更新于
20 |
21 |
').text(`[${datetime}] ${levelName}: ${message}`).appendTo(this.$logs); 229 | this.$logs.scrollTop(this.$logs.prop('scrollHeight')); 230 | } 231 | 232 | onStateChange(service) { 233 | let $panel = $(this.panel); 234 | $panel.find('.service-ctrl').addClass('is-hidden'); 235 | 236 | let statusName; 237 | switch (service.status) { 238 | case Service.STATE_STOPPED: 239 | statusName = '已停止'; 240 | this.$start.removeClass('is-hidden'); 241 | break; 242 | case Service.STATE_START_PENDING: 243 | this.$stop.removeClass('is-hidden'); 244 | statusName = '等待任务'; 245 | break; 246 | case Service.STATE_STOP_PENDING: 247 | this.$loading.removeClass('is-hidden'); 248 | statusName = '正在停止'; 249 | break; 250 | case Service.STATE_RUNNING: 251 | this.$stop.removeClass('is-hidden'); 252 | statusName = '运行中'; 253 | break; 254 | } 255 | $panel.find('.service-status') 256 | .data('status', service.status) 257 | .text(statusName); 258 | this.onProgress(service); 259 | } 260 | 261 | onProgress(service) { 262 | this.$job.empty(); 263 | let currentJob = service.currentJob; 264 | if (!currentJob || !currentJob.tasks) { 265 | return; 266 | } 267 | for (let task of currentJob.tasks) { 268 | console.log("类名: ", task.constructor.name); 269 | let $task = $(TEMPLATE_TASK); 270 | if (task === currentJob.currentTask) { 271 | $task.addClass('is-selected'); 272 | } 273 | $task.find('.progress').val(task.completion).attr('max', task.total); 274 | $task.find('.task-name').text(task.name); 275 | $task.appendTo(this.$job); 276 | } 277 | } 278 | 279 | static async render() { 280 | // await new Promise(resolve => { 281 | // chrome.runtime.sendMessage({ action: "ServicePanel" }, resolve); 282 | // }); 283 | let service = await Service.getInstance() 284 | return new ServicePanel('.page-tab-content[name="service"]', service); 285 | } 286 | } 287 | 288 | 289 | TabPanel.render(); 290 | GeneralPanel.render(); 291 | AccountPanel.render(); 292 | ServicePanel.render(); -------------------------------------------------------------------------------- /extension/popup.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
5 | 6 | 7 | 8 | 9 | 71 | 72 | 73 | 74 | 80 | 81 | 87 | 88 | 94 | 100 | 101 | 107 | 108 | 112 | 118 | 119 | 120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /extension/popup.js: -------------------------------------------------------------------------------- 1 | import Service from "./service.js"; 2 | 3 | const URL_OPTIONS = chrome.runtime.getURL('options.html'); 4 | const URL_ABOUT = URL_OPTIONS + '#about'; 5 | const URL_HELP = URL_OPTIONS + '#help'; 6 | const URL_BACKUP = chrome.runtime.getURL('backup.html'); 7 | 8 | class PopupMenu { 9 | constructor(selector, service) { 10 | this.element = document.querySelector(selector); 11 | this.service = service; 12 | } 13 | 14 | clickNew(event) { 15 | window.open(URL_BACKUP + '#new-task'); 16 | } 17 | 18 | clickBackup(event) { 19 | window.open(URL_BACKUP); 20 | } 21 | 22 | async clickStart(event) { 23 | await this.service.start(); 24 | } 25 | 26 | async clickStop(event) { 27 | await this.service.stop(); 28 | } 29 | 30 | clickSettings(event) { 31 | window.open(URL_OPTIONS); 32 | } 33 | 34 | clickHelp(event) { 35 | window.open(URL_HELP); 36 | } 37 | 38 | clickAbout(event) { 39 | console.log("clickAbout"); 40 | window.open(URL_ABOUT); 41 | } 42 | 43 | getItem(name) { 44 | return this.element.querySelector(`[name="${name}"]`); 45 | } 46 | 47 | disable(name) { 48 | this.getItem(name).setAttribute('disabled', true); 49 | } 50 | 51 | enable(name) { 52 | this.getItem(name).removeAttribute('disabled'); 53 | } 54 | 55 | static async render() { 56 | let service = await Service.getInstance(); 57 | let menu = new PopupMenu('.menu', service); 58 | 59 | // 根据状态启用/禁用按钮 60 | switch (service.status) { 61 | case Service.STATE_STOPPED: 62 | menu.enable('Start'); 63 | menu.disable('Stop'); 64 | break; 65 | 66 | case Service.STATE_START_PENDING: 67 | case Service.STATE_RUNNING: 68 | menu.disable('Start'); 69 | menu.enable('Stop'); 70 | break; 71 | 72 | default: 73 | break; 74 | } 75 | 76 | // 绑定点击事件 77 | Zepto('.menu').on('click', '.menu-item', event => { 78 | console.log('Button clicked:', event.currentTarget.getAttribute('name')); 79 | if (event.currentTarget.hasAttribute('disabled')) return false; 80 | let handle = menu['click' + event.currentTarget.getAttribute('name')]; 81 | if (!handle) return false; 82 | handle.apply(menu, [event]); 83 | setTimeout(() => { 84 | window.close(); 85 | }, 100); 86 | }); 87 | } 88 | } 89 | 90 | document.addEventListener('DOMContentLoaded', () => { 91 | PopupMenu.render(); 92 | }); -------------------------------------------------------------------------------- /extension/services/AsyncBlockingQueue.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Class AsyncBlockingQueue 3 | */ 4 | export default class AsyncBlockingQueue { 5 | constructor() { 6 | this.resolves = []; 7 | this.promises = []; 8 | } 9 | 10 | _add() { 11 | this.promises.push( 12 | new Promise(resolve => { 13 | this.resolves.push(resolve); 14 | }) 15 | ); 16 | } 17 | 18 | enqueue(item) { 19 | if (!this.resolves.length) this._add(); 20 | let resolve = this.resolves.shift(); 21 | resolve(item); 22 | } 23 | 24 | dequeue() { 25 | if (!this.promises.length) this._add(); 26 | return this.promises.shift(); 27 | } 28 | 29 | isEmpty() { 30 | return !this.promises.length; 31 | } 32 | 33 | isBlocked() { 34 | return !!this.resolves.length; 35 | } 36 | 37 | clear() { 38 | this.promises.length = 0; 39 | } 40 | 41 | get length() { 42 | return (this.promises.length - this.resolves.length); 43 | } 44 | } -------------------------------------------------------------------------------- /extension/services/Job.js: -------------------------------------------------------------------------------- 1 | import Service from '../service.js'; 2 | import Task from "./Task.js"; 3 | import TaskError from "./TaskError.js"; 4 | import AsyncBlockingQueue from "./AsyncBlockingQueue.js"; 5 | import Storage from "../storage.js"; 6 | 7 | import {taskFromJSON} from "./task_deserialize.js"; 8 | 9 | /** 10 | * Class Job 11 | */ 12 | export default class Job extends EventTarget { 13 | /** 14 | * Constructor 15 | * @param {Service} service 16 | * @param {string|null} targetUserId 17 | * @param {string|null} localUserId 18 | * @param {boolean} isOffline 19 | */ 20 | constructor(service, targetUserId, localUserId, isOffline) { 21 | super(); 22 | this._service = service; 23 | this._targetUserId = targetUserId; 24 | this._localUserId = localUserId; 25 | this._tasks = []; 26 | this._isRunning = false; 27 | this._currentTask = null; 28 | this._id = null; 29 | this._session = null; 30 | this._isOffline = isOffline; 31 | } 32 | 33 | /** 34 | * Get user info 35 | * @param {Object} cookies 36 | * @param {string} userId 37 | */ 38 | async getUserInfo(cookies, userId) { 39 | const URL_USER_INFO = 'https://m.douban.com/rexxar/api/v2/user/{uid}?ck={ck}&for_mobile=1'; 40 | 41 | let userInfoURL = URL_USER_INFO 42 | .replace('{uid}', userId) 43 | .replace('{ck}', cookies.ck); 44 | let fetch = await Service.getFetchURL(this._service); 45 | return await ( 46 | await fetch(userInfoURL, {headers: {'X-Override-Referer': 'https://m.douban.com/'}}) 47 | ).json(); 48 | } 49 | 50 | /** 51 | * Checkin account 52 | * @returns {object} 53 | */ 54 | async checkin() { 55 | const URL_MINE = 'https://m.douban.com/mine/'; 56 | 57 | let response = await fetch(URL_MINE); 58 | if (response.redirected) { 59 | window.open(response.url); 60 | throw new TaskError('未登录豆瓣'); 61 | } 62 | let bodyElement = Task.parseHTML(await response.text(), URL_MINE); 63 | let inputElement = bodyElement.querySelector('#user'); 64 | let username = inputElement.getAttribute('data-name'); 65 | let uid = inputElement.getAttribute('value'); 66 | let homepageLink = bodyElement.querySelector('.profile .detail .basic-info>a'); 67 | let homepageURL = homepageLink.getAttribute('href'); 68 | let userSymbol = homepageURL.match(/\/people\/(.+)/).pop(); 69 | let cookiesNeeded = { 70 | 'ue': '', 71 | 'bid': '', 72 | 'frodotk_db': '', 73 | 'ck': '', 74 | 'dbcl2': '', 75 | }; 76 | let cookies = await new Promise( 77 | resolve => chrome.cookies.getAll({url: 'https://*.douban.com'}, resolve) 78 | ); 79 | for (let cookie of cookies) { 80 | if (cookie.name in cookiesNeeded) { 81 | cookiesNeeded[cookie.name] = cookie.value; 82 | } 83 | } 84 | 85 | let userInfo = await this.getUserInfo(cookiesNeeded, uid); 86 | 87 | return this._session = { 88 | userId: parseInt(uid), 89 | username: username, 90 | userSymbol: userSymbol, 91 | cookies: cookiesNeeded, 92 | userInfo: userInfo, 93 | updated: Date.now(), 94 | isOther: false 95 | } 96 | } 97 | 98 | /** 99 | * Add a task 100 | * @param {Task} task 101 | */ 102 | addTask(task) { 103 | this._tasks.push(task); 104 | } 105 | 106 | /** 107 | * Run the job 108 | */ 109 | async run() { 110 | let logger = this._service.logger 111 | this._isRunning = true; 112 | 113 | let userId, account, targetUser, isOtherUser = false; 114 | 115 | if (this._isOffline) { 116 | userId = this._targetUserId; 117 | isOtherUser = true; 118 | } else { 119 | let session = await this.checkin(); 120 | if (this._targetUserId) { 121 | let userInfo = await this.getUserInfo(session.cookies, this._targetUserId); 122 | this._targetUserId = userId = parseInt(userInfo.id); 123 | account = { 124 | userId: userId, 125 | username: userInfo.name, 126 | userSymbol: userInfo.uid, 127 | cookies: null, 128 | userInfo: userInfo, 129 | updated: Date.now(), 130 | isOther: true 131 | }; 132 | targetUser = userInfo; 133 | isOtherUser = true; 134 | } else { 135 | userId = session.userId; 136 | account = session; 137 | targetUser = session.userInfo; 138 | } 139 | } 140 | 141 | let storage = new Storage(this._localUserId || userId); 142 | await storage.global.open(); 143 | logger.debug('Open global database'); 144 | if (this._isOffline) { 145 | let account = await storage.global.account.get({userId: userId}); 146 | if (!account) { 147 | logger.debug('The account does not exist'); 148 | storage.global.close(); 149 | return; 150 | } 151 | targetUser = account.userInfo; 152 | } else { 153 | await storage.global.account.put(account); 154 | } 155 | logger.debug('Create the account'); 156 | let jobId = await storage.global.job.add({ 157 | userId: userId, 158 | created: Date.now(), 159 | progress: {}, 160 | tasks: JSON.parse(JSON.stringify(this._tasks)), 161 | }); 162 | logger.debug('Create the job'); 163 | storage.global.close(); 164 | logger.debug('Close global database'); 165 | 166 | await storage.local.open(); 167 | logger.debug('Open local database'); 168 | this._id = jobId; 169 | 170 | // 设置最大并发数 171 | const maxConcurrency = 3; // 控制最大并发任务数 172 | const taskQueue = new AsyncBlockingQueue(); 173 | 174 | let fetch = Service.getFetchURL(this._service); 175 | // 将任务添加到队列中 176 | for (let task of this._tasks) { 177 | this._currentTask = task; 178 | task.init( 179 | fetch, 180 | logger, 181 | jobId, 182 | this._session, 183 | storage.local, 184 | targetUser, 185 | isOtherUser 186 | ); 187 | 188 | taskQueue.enqueue(task); 189 | } 190 | 191 | // 处理任务，确保并发执行数不超过 maxConcurrency 192 | let activePromises = []; 193 | for (let i = 0; i < maxConcurrency; i++) { 194 | activePromises.push(this.runTaskQueue(taskQueue, logger)); 195 | } 196 | 197 | // 等待所有任务完成 198 | await Promise.all(activePromises); 199 | 200 | storage.local.close(); 201 | logger.debug('Close local database'); 202 | this._currentTask = null; 203 | this._isRunning = false; 204 | } 205 | 206 | /** 207 | * 处理任务队列中的任务 208 | */ 209 | async runTaskQueue(queue, logger) { 210 | while (!queue.isEmpty()) { 211 | let task = await queue.dequeue(); 212 | try { 213 | await task.run(); 214 | } catch (e) { 215 | console.error(e) 216 | logger.error('Fail to run task:' + e); 217 | } 218 | } 219 | } 220 | 221 | /** 222 | * Whether the job is running 223 | * @returns {boolean} 224 | */ 225 | get isRunning() { 226 | return this._isRunning; 227 | } 228 | 229 | /** 230 | * Get current task 231 | * @returns {Task|null} 232 | */ 233 | get currentTask() { 234 | return this._currentTask; 235 | } 236 | 237 | /** 238 | * Get tasks 239 | * @returns {Array} 240 | */ 241 | get tasks() { 242 | return this._tasks; 243 | } 244 | 245 | /** 246 | * Get job id 247 | * @returns {number|null} 248 | */ 249 | get id() { 250 | return this._id; 251 | } 252 | 253 | /** 254 | * Convert to JSON string 255 | * @returns {string} 256 | */ 257 | toJSON() { 258 | return { 259 | targetUserId: this._targetUserId, 260 | localUserId: this._localUserId, 261 | tasks: this._tasks.map(task => task.toJSON()), // 保存任务 262 | isOffline: this._isOffline, 263 | _id: this._id, 264 | _session: this._session, 265 | _isRunning: this._isRunning, 266 | _currentTask: this._currentTask ? this._currentTask.toJSON() : null, 267 | }; 268 | } 269 | 270 | /** 271 | * Restore job from JSON 272 | * @param {Object} json 273 | * @param {Service} service 274 | * @param storage 275 | * @returns {Job} 276 | */ 277 | static fromJSON(json, service, storage) { 278 | let fetch = Service.getFetchURL(service); 279 | const job = new Job(service, json.targetUserId, json.localUserId, json.isOffline); 280 | job._id = json._id; 281 | job._session = json._session; 282 | job._isRunning = json._isRunning; 283 | job._currentTask = json._currentTask ? taskFromJSON(json._currentTask, fetch, service.logger, storage) : null; 284 | 285 | // 恢复任务 286 | for (let taskJson of json.tasks) { 287 | const task = taskFromJSON( 288 | taskJson, 289 | fetch, // 传入 fetch 290 | service.logger, // 传入 logger 291 | storage // 传入 storage 292 | ); 293 | job.addTask(task); 294 | } 295 | 296 | return job; 297 | } 298 | 299 | } -------------------------------------------------------------------------------- /extension/services/Logger.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Class Logger 3 | */ 4 | export default class Logger extends EventTarget { 5 | /** 6 | * Constructor 7 | */ 8 | constructor() { 9 | super(); 10 | Object.assign(this, { 11 | LEVEL_CRITICAL: 50, 12 | LEVEL_ERROR: 40, 13 | LEVEL_WARNING: 30, 14 | LEVEL_INFO: 20, 15 | LEVEL_DEBUG: 10, 16 | LEVEL_NOTSET: 0, 17 | }); 18 | this._level = this.LEVEL_INFO; 19 | this.entries = []; 20 | } 21 | 22 | /** 23 | * Log error 24 | * @param {string} message 25 | * @param {any} context 26 | * @returns {object} 27 | */ 28 | error(message, context = null) { 29 | return this.log(this.LEVEL_ERROR, message, context); 30 | } 31 | 32 | /** 33 | * Log warning 34 | * @param {string} message 35 | * @param {any} context 36 | * @returns {object} 37 | */ 38 | warning(message, context = null) { 39 | return this.log(this.LEVEL_WARNING, message, context); 40 | } 41 | 42 | /** 43 | * Log info 44 | * @param {string} message 45 | * @param {any} context 46 | * @returns {object} 47 | */ 48 | info(message, context = null) { 49 | return this.log(this.LEVEL_INFO, message, context); 50 | } 51 | 52 | /** 53 | * Log debug info 54 | * @param {string} message 55 | * @param {any} context 56 | * @returns {object} 57 | */ 58 | debug(message, context = null) { 59 | return this.log(this.LEVEL_DEBUG, message, context); 60 | } 61 | 62 | /** 63 | * Log message 64 | * @param {number} level 65 | * @param {string} message 66 | * @param {any} context 67 | * @returns {object} 68 | */ 69 | log(level, message, context = null) { 70 | if (this._level > level) return; 71 | let levelName; 72 | switch (level) { 73 | case this.LEVEL_DEBUG: 74 | levelName = 'DEBUG'; 75 | break; 76 | case this.LEVEL_INFO: 77 | levelName = 'INFO'; 78 | break; 79 | case this.LEVEL_WARNING: 80 | levelName = 'WARNING'; 81 | break; 82 | case this.LEVEL_ERROR: 83 | levelName = 'ERROR'; 84 | break; 85 | case this.LEVEL_CRITICAL: 86 | levelName = 'CRITICAL'; 87 | break; 88 | default: 89 | levelName = 'UNKNOWN'; 90 | } 91 | let entry = { 92 | time: Date.now(), 93 | level: level, 94 | levelName: levelName, 95 | message: message, 96 | context: context, 97 | }; 98 | let cancelled = !this.dispatchEvent(new CustomEvent('log', {detail: entry})); 99 | if (cancelled) { 100 | return entry; 101 | } 102 | return this.entries.push(entry); 103 | } 104 | 105 | /** 106 | * Get default level 107 | * @returns {number} 108 | */ 109 | get level() { 110 | return this._level; 111 | } 112 | 113 | /** 114 | * Set default level 115 | * @param {number} value 116 | */ 117 | set level(value) { 118 | this._level = value; 119 | } 120 | } -------------------------------------------------------------------------------- /extension/services/StateChangeEvent.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Class StateChangeEvent 3 | */ 4 | export default class StateChangeEvent extends Event { 5 | constructor(originalState, currentState) { 6 | super('statechange'); 7 | this.originalState = originalState; 8 | this.currentState = currentState; 9 | } 10 | } -------------------------------------------------------------------------------- /extension/services/Task.js: -------------------------------------------------------------------------------- 1 | import TaskError from "./TaskError.js"; 2 | import Storage from "../storage.js"; 3 | 4 | export default class Task { 5 | /** 6 | * Parse HTML 7 | * @param {string} html 8 | * @param {string} url 9 | * @returns {Document} 10 | */ 11 | static parseHTML(html, url) { 12 | let context = document.implementation.createHTMLDocument(''); 13 | context.documentElement.innerHTML = html; 14 | let base = context.createElement('base'); 15 | base.href = url; 16 | context.head.appendChild(base); 17 | return context; 18 | } 19 | 20 | /** 21 | * Initialize the task 22 | * @param {callback} fetch 23 | * @param {Logger} logger 24 | * @param {number} jobId 25 | * @param {Object} session 26 | * @param {Storage} localStorage 27 | * @param {Object} targetUser 28 | * @param {boolean} isOtherUser 29 | */ 30 | init(fetch, logger, jobId, session, localStorage, targetUser, isOtherUser) { 31 | this.fetch = fetch; 32 | this.logger = logger; 33 | this.jobId = jobId; 34 | this.session = session; 35 | this.storage = localStorage; 36 | this.targetUser = targetUser; 37 | this.isOtherUser = isOtherUser; 38 | this.total = 1; 39 | this.completion = 0; 40 | this.parseHTML = Task.parseHTML 41 | } 42 | 43 | /** 44 | * Run task 45 | */ 46 | async run() { 47 | throw new TaskError('Not implemented.'); 48 | } 49 | 50 | toJSON() { 51 | return { 52 | taskType: this.constructor.name, // 存储类名 53 | jobId: this.jobId, 54 | session: this.session, 55 | targetUser: this.targetUser, 56 | isOtherUser: this.isOtherUser, 57 | total: this.total, 58 | completion: this.completion, 59 | // 忽略不可序列化的成员变量：fetch、logger、parseHTML、storage 60 | }; 61 | } 62 | 63 | /** 64 | * Get task name 65 | * @returns {string} 66 | */ 67 | get name() { 68 | throw new TaskError('Not implemented.'); 69 | } 70 | 71 | /** 72 | * Task completed 73 | */ 74 | complete() { 75 | this.completion = this.total; 76 | } 77 | 78 | /** 79 | * Progress step 80 | */ 81 | step() { 82 | this.completion += 1; 83 | } 84 | } -------------------------------------------------------------------------------- /extension/services/TaskError.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Class TaskError 3 | */ 4 | export default class TaskError extends Error { 5 | constructor(message) { 6 | super(message); 7 | } 8 | } -------------------------------------------------------------------------------- /extension/services/task_deserialize.js: -------------------------------------------------------------------------------- 1 | import Task from "./Task.js"; 2 | 3 | // 确保正确导入所有子类 4 | import Annotation from "../tasks/annotation.js"; 5 | import Blacklist from "../tasks/blacklist.js"; 6 | import Board from "../tasks/board.js"; 7 | import Doulist from "../tasks/doulist.js"; 8 | import Doumail from "../tasks/doumail.js"; 9 | import Files from "../tasks/files.js"; 10 | import Follower from "../tasks/follower.js"; 11 | import Following from "../tasks/following.js"; 12 | import Interest from "../tasks/interest.js"; 13 | import Mock from "../tasks/mock.js"; 14 | import Note from "../tasks/note.js"; 15 | import Photo from "../tasks/photo.js"; 16 | import Review from "../tasks/review.js"; 17 | import Status from "../tasks/status.js"; 18 | 19 | import MigrateAnnotation from "../tasks/migrate/annotation.js"; 20 | import MigrateBlacklist from "../tasks/migrate/blacklist.js"; 21 | import MigrateInterest from "../tasks/migrate/interest.js"; 22 | import MigrateNote from "../tasks/migrate/note.js"; 23 | import MigrateReview from "../tasks/migrate/review.js"; 24 | import Follow from "../tasks/migrate/follow.js"; 25 | 26 | export function taskFromJSON(json, fetch, logger, storage) { 27 | // 维护一个子类映射 28 | const taskClasses = { 29 | Annotation: Annotation, 30 | Blacklist: Blacklist, 31 | Board: Board, 32 | Doulist: Doulist, 33 | Doumail: Doumail, 34 | Files: Files, 35 | Follower: Follower, 36 | Following: Following, 37 | Interest: Interest, 38 | Mock: Mock, 39 | Note: Note, 40 | Photo: Photo, 41 | Review: Review, 42 | Status: Status, 43 | MigrateAnnotation: MigrateAnnotation, 44 | MigrateBlacklist: MigrateBlacklist, 45 | MigrateInterest: MigrateInterest, 46 | MigrateNote: MigrateNote, 47 | MigrateReview: MigrateReview, 48 | Follow: Follow 49 | }; 50 | 51 | const TaskClass = taskClasses[json.taskType] || Task; // 找到正确的类，默认是 Task 52 | const task = new TaskClass(); // 用子类实例化 53 | task.jobId = json.jobId; 54 | task.session = json.session; 55 | task.targetUser = json.targetUser; 56 | task.isOtherUser = json.isOtherUser; 57 | task.total = json.total; 58 | task.completion = json.completion; 59 | 60 | // 重新初始化不可序列化的成员变量 61 | task.fetch = fetch; 62 | task.logger = logger; 63 | task.parseHTML = Task.parseHTML; 64 | task.storage = storage; 65 | 66 | return task; 67 | } -------------------------------------------------------------------------------- /extension/settings.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Class Settings 3 | */ 4 | export default class Settings { 5 | static apply(target, settings) { 6 | for (let key in settings) { 7 | try { 8 | let keyPath = key.split('.'); 9 | if (keyPath.shift() !== target.name) { 10 | continue; 11 | } 12 | let lastNode = keyPath.pop(); 13 | for (let node of keyPath) { 14 | target = target[node]; 15 | } 16 | target[lastNode] = settings[key]; 17 | } catch (e) {} 18 | } 19 | } 20 | 21 | static async load(...args) { 22 | args.unshift({}); 23 | let defaults = Object.assign.apply(null, args); 24 | let settings = await new Promise(resolve => { 25 | chrome.storage.sync.get(Object.keys(defaults), resolve); 26 | }); 27 | return Object.assign({}, defaults, settings); 28 | } 29 | 30 | static async save(settings) { 31 | return await new Promise(resolve => { 32 | chrome.storage.sync.set(settings, resolve); 33 | }); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /extension/shortcuts.css: -------------------------------------------------------------------------------- 1 | .is-clipped { 2 | overflow: hidden!important; 3 | } 4 | 5 | #tofu-shortcuts { 6 | z-index: 29999; 7 | position: fixed; 8 | left: 0; 9 | right: 0; 10 | top: 0; 11 | bottom: 0; 12 | box-sizing: border-box; 13 | align-items: center; 14 | justify-content: center; 15 | overflow: hidden; 16 | display: flex; 17 | flex-direction: column; 18 | } 19 | 20 | #tofu-shortcuts .modal-background { 21 | background-color: rgba(10, 10, 10, 0.8); 22 | left: 0; 23 | right: 0; 24 | top: 0; 25 | bottom: 0; 26 | position: absolute; 27 | display: block; 28 | } 29 | 30 | #tofu-shortcuts .modal-content { 31 | display: block; 32 | overflow: auto; 33 | width: calc(100vw - 150px); 34 | height: calc(100vh - 150px); 35 | max-width: 640px; 36 | max-height: 480px; 37 | background: white; 38 | border-radius: 6px; 39 | position: relative; 40 | margin: 0; 41 | padding: 20px; 42 | } 43 | 44 | #tofu-shortcuts .modal-content > h1 { 45 | text-align: center; 46 | font-size: 26px; 47 | font-weight: bold; 48 | color: #666; 49 | line-height: 1.2; 50 | margin: 0; 51 | padding: 0 0 16px 0; 52 | } 53 | 54 | #tofu-shortcuts .modal-close { 55 | position: fixed; 56 | right: 20px; 57 | top: 20px; 58 | width: 32px; 59 | height: 32px; 60 | display: block; 61 | cursor: pointer; 62 | border: none; 63 | padding: 0; 64 | margin: 0; 65 | border-radius: 32px; 66 | background-image: url('data:image/svg+xml;utf8,'); 67 | background-repeat: no-repeat; 68 | background-position: 9px 6px; 69 | background-size: 14px; 70 | outline: none; 71 | background-color: transparent; 72 | } 73 | 74 | #tofu-shortcuts .modal-close:hover { 75 | background-color: black; 76 | } 77 | -------------------------------------------------------------------------------- /extension/shortcuts.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | 4 | const SHORTCUTS_TEMPLATE = `\ 5 | 6 | 10 | 11 | `; 12 | 13 | 14 | /** 15 | * Class Shortcuts 16 | */ 17 | class Shortcuts { 18 | constructor() { 19 | let shortcuts = this._elementRoot = document.createElement('DIV'); 20 | shortcuts.id = 'tofu-shortcuts'; 21 | shortcuts.innerHTML = SHORTCUTS_TEMPLATE; 22 | 23 | this._closed = true; 24 | shortcuts.querySelector('.modal-close').addEventListener('click', e => { 25 | this.close(); 26 | }); 27 | document.addEventListener('keydown', event => { 28 | if (event.code == 27) { 29 | this.close(); 30 | } 31 | }); 32 | } 33 | 34 | /** 35 | * Close shortcuts 36 | * @returns {Shortcuts} 37 | */ 38 | close() { 39 | this._closed = true; 40 | try { 41 | document.body.removeChild(this._elementRoot); 42 | } catch (e) {} 43 | document.documentElement.classList.remove('is-clipped'); 44 | return this; 45 | } 46 | 47 | /** 48 | * Open shortcuts 49 | * @returns {Shortcuts} 50 | */ 51 | open() { 52 | this._closed = false; 53 | document.body.appendChild(this._elementRoot); 54 | document.documentElement.classList.add('is-clipped'); 55 | return this; 56 | } 57 | 58 | /** 59 | * Setup shortcuts 60 | * @returns {Shortcuts} 61 | */ 62 | static setup() { 63 | if (!Shortcuts.instance) { 64 | Shortcuts.instance = new Shortcuts(); 65 | } 66 | return Shortcuts.instance; 67 | } 68 | } 69 | 70 | window.shortcuts = Shortcuts.setup(); 71 | -------------------------------------------------------------------------------- /extension/storage.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Dexie from './vendor/dexie.js'; 3 | import { importFromJsonString, exportToJsonString } from './vendor/IDBExportImport.js'; 4 | import * as fflate from './vendor/fflate.js'; 5 | 6 | const DB_NAME = 'tofu'; 7 | 8 | const SCHEMA_GLOBAL = [ 9 | null, 10 | { 11 | account: 'userId, userSymbol', 12 | job: '++id, userId, userSymbol', 13 | }, 14 | ]; 15 | 16 | const SCHEMA_LOCAL = [ 17 | null, 18 | { 19 | status: 'id', 20 | following: '++id, version', 21 | follower: '++id, version', 22 | blacklist: '++id, version', 23 | review: 'id, type, [type+version]', 24 | note: 'id, version', 25 | interest: 'id, &subject, [type+status], [type+status+version]', 26 | album: 'id, version', 27 | photo: 'id, album, [album+version]', 28 | doulist: 'id, type, [type+version]', 29 | doulistItem: 'id, doulist, [doulist+version]', 30 | doumail: 'id, contact', 31 | doumailContact: 'id, rank', 32 | version: 'table, version', 33 | }, 34 | { 35 | files: '++id, &url', 36 | }, 37 | { 38 | annotation: 'id, subject, [subject+version]', 39 | }, 40 | { 41 | board: 'id', 42 | }, 43 | ]; 44 | 45 | 46 | /** 47 | * Class Storage 48 | */ 49 | export default class Storage { 50 | static isDumping = false; 51 | static isRestoring = false; 52 | 53 | constructor(userId = null) { 54 | this.userId = userId; 55 | } 56 | 57 | get global() { 58 | if (!this._global) { 59 | let db = this._global = new Dexie(DB_NAME); 60 | for (let i = 1; i < SCHEMA_GLOBAL.length; i ++) { 61 | db.version(i).stores(SCHEMA_GLOBAL[i]); 62 | } 63 | } 64 | return this._global; 65 | } 66 | 67 | get local() { 68 | if (!this._local) { 69 | if (!this.userId) { 70 | throw new Error('No local storage'); 71 | } 72 | this._local = this.getLocalDb(this.userId); 73 | } 74 | return this._local; 75 | } 76 | 77 | getLocalDb(userId) { 78 | let db = new Dexie(`${DB_NAME}[${userId}]`); 79 | for (let i = 1; i < SCHEMA_LOCAL.length; i ++) { 80 | db.version(i).stores(SCHEMA_LOCAL[i]); 81 | } 82 | return db; 83 | } 84 | 85 | async drop(userId) { 86 | let localDbName = `${DB_NAME}[${userId}]`; 87 | if (await Dexie.exists(localDbName)) { 88 | try { 89 | await Dexie.delete(localDbName); 90 | } catch (e) { 91 | return false; 92 | } 93 | } 94 | return await this.global.account.where({ 95 | userId: parseInt(userId) 96 | }).delete() > 0; 97 | } 98 | 99 | async dump(onProgress) { 100 | if (this.constructor.isRestoring) { 101 | throw '正在恢复数据库'; 102 | } 103 | if (this.constructor.isDumping) { 104 | throw '正在备份数据库'; 105 | } 106 | this.constructor.isDumping = true; 107 | 108 | try { 109 | var backupData = {}; 110 | var dbFiles = []; 111 | 112 | const databases = await Dexie.getDatabaseNames(); 113 | const total = databases.length; 114 | var completed = 1; 115 | for (let database of databases) { 116 | if (database != DB_NAME) { 117 | dbFiles.push(database); 118 | } 119 | let db = new Dexie(database); 120 | await db.open(); 121 | let dbJson = await new Promise((resolve, reject) => { 122 | exportToJsonString(db.backendDB(), (error, jsonString) => { 123 | if (error) { 124 | reject(error); 125 | } else { 126 | resolve(jsonString); 127 | } 128 | }) 129 | }); 130 | db.close(); 131 | backupData[database + '.json'] = fflate.strToU8(dbJson); 132 | onProgress(completed ++ / total); 133 | } 134 | backupData['database.json'] = fflate.strToU8(JSON.stringify({ 135 | 'global': { 'version': SCHEMA_GLOBAL.length }, 136 | 'local': { 137 | 'version': SCHEMA_LOCAL.length, 138 | 'files': dbFiles 139 | } 140 | })); 141 | 142 | return await new Promise((resolve, reject) => { 143 | fflate.zip(backupData, (error, data) => { 144 | if (error) { 145 | reject(error); 146 | } else { 147 | resolve(data); 148 | } 149 | }) 150 | }); 151 | } finally { 152 | this.constructor.isDumping = false; 153 | } 154 | } 155 | 156 | async restore(zipBuffer, onProgress) { 157 | if (this.constructor.isRestoring) { 158 | throw '正在恢复数据库'; 159 | } 160 | if (this.constructor.isDumping) { 161 | throw '正在备份数据库'; 162 | } 163 | this.constructor.isRestoring = true; 164 | 165 | const unzipFile = function (filename) { 166 | return new Promise((resolve, reject) => { 167 | fflate.unzip(zipBuffer, { 168 | filter(file) { 169 | return file.name == filename 170 | } 171 | }, (error, data) => { 172 | if (error) { 173 | reject(error); 174 | } else { 175 | resolve(data[filename]); 176 | } 177 | }) 178 | }); 179 | }; 180 | 181 | try { 182 | var successes = []; 183 | var failures = []; 184 | var unzippedFile = await unzipFile('database.json'); 185 | if (!unzippedFile) throw '压缩包内未找到数据库信息'; 186 | 187 | let dbMeta = JSON.parse(fflate.strFromU8(unzippedFile)); 188 | if (dbMeta.global.version != SCHEMA_GLOBAL.length || dbMeta.local.version != SCHEMA_LOCAL.length) { 189 | throw '数据库版本不一致'; 190 | }; 191 | 192 | var completed = 1; 193 | const total = dbMeta.local.files.length + 2; 194 | onProgress(completed ++ / total); 195 | 196 | let globalDbFilename = `${DB_NAME}.json`; 197 | unzippedFile = await unzipFile(globalDbFilename); 198 | if (!unzippedFile) throw `压缩包缺失文件："${globalDbFilename}"`; 199 | let globalDb = JSON.parse(fflate.strFromU8(unzippedFile)); 200 | onProgress(completed ++ / total); 201 | 202 | await this.global.open(); 203 | try { 204 | for (let account of globalDb.account) { 205 | let dbName = `${DB_NAME}[${account.userId}]`; 206 | let dbFilename = dbName + '.json'; 207 | console.log(`importing ${dbFilename}`); 208 | if (await this.global.account.get({userId: account.userId})) { 209 | failures.push({ 210 | 'database': dbName, 211 | 'filename': dbFilename, 212 | 'error': '数据库已存在' 213 | }); 214 | onProgress(completed ++ / total); 215 | continue; 216 | } 217 | let dbFile = await unzipFile(dbFilename); 218 | if (!dbFile) { 219 | failures.push({ 220 | 'database': dbName, 221 | 'filename': dbFilename, 222 | 'error': '数据库文件缺失' 223 | }); 224 | continue; 225 | } 226 | 227 | let localDb = this.getLocalDb(account.userId); 228 | await localDb.open(); 229 | try { 230 | await new Promise((resolve, reject) => { 231 | importFromJsonString(localDb.backendDB(), fflate.strFromU8(dbFile), (error) => { 232 | if (error) { 233 | reject(error); 234 | } else { 235 | resolve(); 236 | } 237 | }); 238 | }); 239 | await this.global.account.put(account); 240 | successes.push({ 241 | 'database': dbName, 242 | 'filename': dbFilename 243 | }); 244 | } catch (error) { 245 | failures.push({ 246 | 'database': dbName, 247 | 'filename': dbFilename, 248 | 'error': error 249 | }); 250 | } finally { 251 | localDb.close(); 252 | } 253 | onProgress(completed ++ / total); 254 | } 255 | } finally { 256 | this.global.close(); 257 | } 258 | 259 | return { 260 | 'successes': successes, 261 | 'failures': failures 262 | }; 263 | } finally { 264 | this.constructor.isRestoring = false; 265 | } 266 | } 267 | } 268 | -------------------------------------------------------------------------------- /extension/tasks/annotation.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 50; 7 | const URL_ANNOTATIONS = 'https://m.douban.com/rexxar/api/v2/user/{uid}/annotations?start={start}&count=50&ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Annotation extends Task { 11 | async fetchAnnotation(url) { 12 | let fetch = await this.fetch 13 | let response = await fetch(url); 14 | if (response.status !== 200) { 15 | return; 16 | } 17 | let html = this.parseHTML(await response.text()); 18 | return html.querySelector('#link-report').innerHTML; 19 | } 20 | 21 | async run() { 22 | let version = this.jobId; 23 | await this.storage.table('version').put({table: 'annotation', version: version, updated: Date.now()}); 24 | 25 | let baseURL = URL_ANNOTATIONS 26 | .replace('{ck}', this.session.cookies.ck) 27 | .replace('{uid}', this.targetUser.id); 28 | 29 | let pageCount = 1; 30 | for (let i = 0; i < pageCount; i ++) { 31 | let fetch = await this.fetch 32 | let response = await fetch(baseURL.replace('{start}', i * PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/'}}); 33 | if (response.status !== 200) { 34 | throw new TaskError('豆瓣服务器返回错误'); 35 | } 36 | let json = await response.json(); 37 | this.total = parseInt(json.total); 38 | pageCount = Math.ceil(json.total / PAGE_SIZE); 39 | for (let collection of json.collections) { 40 | let subject = collection.subject; 41 | for (let annotation of collection.annotations) { 42 | let row = await this.storage.annotation.get(parseInt(annotation.id)); 43 | let fulltext = await this.fetchAnnotation(annotation.url); 44 | if (row) { 45 | let lastVersion = row.version; 46 | row.version = version; 47 | if (fulltext !== row.annotation.fulltext) { 48 | !row.history && (row.history = {}); 49 | row.history[lastVersion] = row.annotation; 50 | annotation.fulltext = fulltext; 51 | row.annotation = annotation; 52 | } 53 | } else { 54 | annotation.fulltext = fulltext; 55 | annotation.subject = subject; 56 | row = { 57 | id: parseInt(annotation.id), 58 | version: version, 59 | subject: parseInt(subject.id), 60 | annotation: annotation, 61 | } 62 | } 63 | await this.storage.annotation.put(row); 64 | } 65 | this.step(); 66 | } 67 | } 68 | this.complete(); 69 | } 70 | 71 | get name() { 72 | return '笔记'; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /extension/tasks/blacklist.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | const URL_BLACKLIST = 'https://www.douban.com/contacts/blacklist?start={start}'; 6 | const URL_USER_INFO = 'https://m.douban.com/rexxar/api/v2/user/{uid}?ck={ck}&for_mobile=1'; 7 | const PAGE_SIZE = 72; 8 | 9 | 10 | export default class Following extends Task { 11 | 12 | async run() { 13 | if (this.isOtherUser) { 14 | throw TaskError('不能备份其他用户的黑名单'); 15 | } 16 | 17 | await this.storage.table('version').put({table: 'blacklist', version: this.jobId, updated: Date.now()}); 18 | 19 | let totalPage = this.total = 1; 20 | 21 | for (let i = 0; i < totalPage; i ++) { 22 | let fetch = await this.fetch 23 | let response = await fetch(URL_BLACKLIST.replace('{start}', i * PAGE_SIZE)); 24 | if (response.status !== 200) { 25 | throw new TaskError('豆瓣服务器返回错误'); 26 | } 27 | let html = this.parseHTML(await response.text()); 28 | try { 29 | this.total = totalPage = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 30 | } catch (e) {} 31 | for (let dl of html.querySelectorAll('.obss.namel>dl')) { 32 | let avatar = dl.querySelector('.imgg'); 33 | let idMatch = avatar.src.match(/\/icon\/u(\d+)\-(\d+)\.jpg$/), idText; 34 | let userLink = dl.querySelector('.nbg').href; 35 | let uid = userLink.match(/https:\/\/www\.douban\.com\/people\/(.+)\//)[1]; 36 | if (idMatch) { 37 | idText = idMatch[1]; 38 | } else { 39 | let url = URL_USER_INFO 40 | .replace('{ck}', this.session.cookies.ck) 41 | .replace('{uid}', uid); 42 | let fetch = await this.fetch 43 | let response = await fetch(url, {headers: {'X-Override-Referer': 'https://www.douban.com/'}}); 44 | if (response.status != 200) { 45 | idText = null; 46 | } else { 47 | let json = await response.json(); 48 | idText = json.id; 49 | } 50 | } 51 | let row = { 52 | version: this.jobId, 53 | user: { 54 | avatar: avatar.src, 55 | id: idText, 56 | name: avatar.alt, 57 | uid: uid, 58 | uri: 'douban://douban.com/user/' + idText, 59 | url: userLink, 60 | } 61 | }; 62 | await this.storage.blacklist.put(row); 63 | } 64 | this.step(); 65 | } 66 | this.complete(); 67 | } 68 | 69 | get name() { 70 | return '黑名单'; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /extension/tasks/board.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 20; 7 | const URL_MESSAGE_BOARD = 'https://www.douban.com/people/{uid}/board?start={start}'; 8 | 9 | 10 | export default class Board extends Task { 11 | async run() { 12 | let version = this.jobId; 13 | let lastMessageId = ''; 14 | let maxMessageId = 0; 15 | await this.storage.transaction('rw', this.storage.table('version'), async () => { 16 | let verTable = this.storage.table('version'); 17 | let row = await verTable.get('board'); 18 | if (row) { 19 | lastMessageId = row.lastId; 20 | await verTable.update('board', {version: version, updated: Date.now()}); 21 | } else { 22 | await verTable.add({table: 'board', version: version, updated: Date.now()}); 23 | } 24 | }); 25 | 26 | let baseURL = URL_MESSAGE_BOARD 27 | .replace('{uid}', this.targetUser.id); 28 | 29 | let totalPage = this.total = 1; 30 | 31 | for (let i = 0; i < totalPage; i ++) { 32 | let fetch = await this.fetch 33 | let response = await fetch(baseURL.replace('{start}', i * PAGE_SIZE)); 34 | if (response.status !== 200) { 35 | throw new TaskError('豆瓣服务器返回错误'); 36 | } 37 | let html = this.parseHTML(await response.text()); 38 | try { 39 | this.total = totalPage = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 40 | } catch (e) {} 41 | 42 | for (let li of html.querySelectorAll('#comments>.comment-item')) { 43 | let messageId = parseInt(li.dataset.cid); 44 | if (messageId <= lastMessageId) { 45 | totalPage = 0; 46 | break; 47 | } 48 | if (messageId > maxMessageId) { 49 | maxMessageId = messageId; 50 | } 51 | let sendTime = li.querySelector('.pl').textContent; 52 | if (sendTime.length < 6) { 53 | let datetime = new Date(); 54 | let year = datetime.getFullYear().toString(); 55 | let month = datetime.getMonth() + 1; 56 | month = month < 10 ? '0' + month.toString() : month.toString(); 57 | let date = datetime.getDate(); 58 | date = date < 10 ? '0' + date.toString() : date.toString(); 59 | sendTime = year + ' ' + month + '-' + date + ' ' + sendTime; 60 | } else if (sendTime.length < 12) { 61 | sendTime = (new Date()).getFullYear().toString() + ' ' + sendTime; 62 | } 63 | let row = { 64 | id: messageId, 65 | sender: { 66 | avatar: li.previousElementSibling.querySelector('img').src, 67 | name: li.childNodes[0].text, 68 | url: li.childNodes[0].href, 69 | }, 70 | created: Date.now(), 71 | message: this.getMessage(li), 72 | sendTime: sendTime 73 | }; 74 | try { 75 | await this.storage.board.add(row); 76 | } catch (e) {} 77 | } 78 | this.step(); 79 | } 80 | await this.storage.table('version').update('board', { lastId: maxMessageId }); 81 | this.complete(); 82 | } 83 | 84 | getMessage(element) { 85 | let message = element.childNodes[1].textContent.substr(4); 86 | for (let i = 2; i < element.childNodes.length; i ++) { 87 | let childNode = element.childNodes[i]; 88 | if (childNode.className == 'pl') break; 89 | 90 | message += childNode.textContent; 91 | } 92 | 93 | return message; 94 | } 95 | 96 | get name() { 97 | return '留言板'; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /extension/tasks/doulist.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 50; 7 | const URL_DOULIST = 'https://m.douban.com/rexxar/api/v2/user/{uid}/{type}_doulists?start={start}&count=50&ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Doulist extends Task { 11 | compareDoulist(l, r) { 12 | if (l.desc != r.desc) return false; 13 | if (l.title != r.title) return false; 14 | if (l.tags.sort().toString() != r.tags.sort().toString()) return false; 15 | return true; 16 | } 17 | 18 | compareDoulistItem(l, r) { 19 | if (l.comment != r.comment) return false; 20 | return true; 21 | } 22 | 23 | async run() { 24 | let version = this.jobId; 25 | this.total = this.targetUser.following_doulist_count + this.targetUser.owned_doulist_count; 26 | if (this.total == 0) { 27 | return; 28 | } 29 | await this.storage.table('version').put({table: 'doulist', version: version, updated: Date.now()}); 30 | 31 | let baseURL = URL_DOULIST 32 | .replace('{ck}', this.session.cookies.ck) 33 | .replace('{uid}', this.targetUser.id); 34 | 35 | for (let type of ['owned', 'following']) { 36 | let urlWithType = baseURL.replace('{type}', type); 37 | let pageCount = 1; 38 | for (let i = 0; i < pageCount; i ++) { 39 | let fetch = await this.fetch 40 | let response = await fetch(urlWithType.replace('{start}', i * PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/doulist'}}); 41 | if (response.status != 200) { 42 | throw new TaskError('豆瓣服务器返回错误'); 43 | } 44 | let json = await response.json(); 45 | pageCount = Math.ceil(json.total / PAGE_SIZE); 46 | for (let doulist of json.doulists) { 47 | let doulistId = parseInt(doulist.id); 48 | let row = await this.storage.doulist.get(doulistId); 49 | if (row) { 50 | let lastVersion = row.version; 51 | row.version = version; 52 | if (!this.compareDoulist(doulist, row.doulist)) { 53 | !row.history && (row.history = {}); 54 | row.history[lastVersion] = row.doulist; 55 | row.doulist = doulist; 56 | } 57 | } else { 58 | row = { 59 | id: doulistId, 60 | type: type, 61 | version: version, 62 | doulist: doulist, 63 | }; 64 | } 65 | const DOULIST_PAGE_SIZE = 25; 66 | let doulistTotalPage = 1; 67 | for (let i = 0; i < doulistTotalPage; i ++) { 68 | let fetch = await this.fetch 69 | let response = await fetch(doulist.url + '?start=' + i * DOULIST_PAGE_SIZE); 70 | if (response.status != 200) { 71 | if (response.status < 500) continue; 72 | throw new TaskError('豆瓣服务器返回错误'); 73 | } 74 | let html = this.parseHTML(await response.text()); 75 | try { 76 | doulistTotalPage = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 77 | } catch (e) {} 78 | for (let item of html.querySelectorAll('.doulist-item')) { 79 | let addBtn = item.querySelector('.lnk-doulist-add'); 80 | if (!addBtn) continue; 81 | let itemId = parseInt(item.id.substr(4)); 82 | let itemBody = item.querySelector('.bd'); 83 | let itemTypes = []; 84 | for (let itemType of itemBody.classList) { 85 | if (itemType.startsWith('doulist-')) { 86 | itemTypes.push(itemType.substr(8)); 87 | } 88 | } 89 | let itemSource = item.querySelector('.source').innerText.trim().substr(3); 90 | let itemAbstract = item.querySelector('.abstract'); 91 | let commentBlockquote = item.querySelector('.comment-item>.comment'); 92 | let extra = {}; 93 | let itemCategory = parseInt(addBtn.dataset.cate); 94 | if (itemCategory == 3055) { 95 | // 广播 96 | try { 97 | let statusText = item.querySelector('.status-text'); 98 | let statusImages = []; 99 | for (let statusImage of item.querySelectorAll('.status-images>a')) { 100 | statusImages.push(statusImage.style.backgroundImage.slice(5,-2)); 101 | } 102 | let status = { 103 | text: statusText.innerText.trim(), 104 | images: statusImages, 105 | }; 106 | extra.status = status; 107 | } catch (e) { } 108 | } 109 | let itemEntity = { 110 | id: parseInt(addBtn.dataset.id), 111 | type: itemTypes, 112 | category: itemCategory, 113 | category_name: addBtn.dataset.catename, 114 | url: addBtn.dataset.url, 115 | title: addBtn.dataset.title, 116 | can_view: addBtn.dataset.canview == 'True', 117 | is_url_subject: addBtn.dataset.isurlsubject == 'true', 118 | picture: addBtn.dataset.picture, 119 | abstract: itemAbstract ? itemAbstract.innerText : null, 120 | source: itemSource, 121 | comment: commentBlockquote ? commentBlockquote.innerText : null, 122 | extra: extra, 123 | }; 124 | let row = await this.storage.doulistItem.get(itemId); 125 | if (row) { 126 | let lastVersion = row.version; 127 | row.version = version; 128 | if (!this.compareDoulistItem(item, row.item)) { 129 | !row.history && (row.history = {}); 130 | row.history[lastVersion] = row.item; 131 | row.item = itemEntity; 132 | } 133 | } else { 134 | row = { 135 | id: itemId, 136 | doulist: doulistId, 137 | version: version, 138 | item: itemEntity, 139 | } 140 | } 141 | await this.storage.doulistItem.put(row); 142 | } 143 | } 144 | await this.storage.doulist.put(row); 145 | this.step(); 146 | } 147 | } 148 | } 149 | this.complete(); 150 | } 151 | 152 | get name() { 153 | return '豆列'; 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /extension/tasks/doumail.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 20; 7 | const URL_DOUMAIL = 'https://www.douban.com/doumail/?start={start}'; 8 | const URL_DOUMAIL_LOAD_MORE = 'https://www.douban.com/j/doumail/loadmore'; 9 | 10 | 11 | export default class Doumail extends Task { 12 | async run() { 13 | if (this.isOtherUser) { 14 | throw TaskError('不能备份其他用户的豆邮'); 15 | } 16 | let pageCount = 1; 17 | for (let i = 0; i < pageCount; i ++) { 18 | let fetch = await this.fetch 19 | let response = await fetch(URL_DOUMAIL.replace('{start}', i * PAGE_SIZE)); 20 | if (response.status != 200) { 21 | throw new TaskError('豆瓣服务器返回错误'); 22 | } 23 | let html = this.parseHTML(await response.text()); 24 | try { 25 | pageCount = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 26 | } catch (e) {} 27 | this.total = pageCount * PAGE_SIZE; 28 | for (let contact of html.querySelectorAll('.doumail-list>ul>li')) { 29 | let operationAnchor = contact.querySelector('.operations>.post_link.report'); 30 | let userId = parseInt(operationAnchor.dataset.id); 31 | isNaN(userId) && (userId = 0); 32 | let contactName = operationAnchor.dataset.sname; 33 | let contactUrl = operationAnchor.dataset.slink; 34 | let contactAvatarImg = contact.querySelector('.pic img'); 35 | let contactAvatar = contactAvatarImg ? contactAvatarImg.src : null; 36 | let time = contact.querySelector('.title>.sender>.time').innerText; 37 | let abstract = contact.querySelector('.title>p').innerText; 38 | let doumailUrl = contact.querySelector('.title .url').href; 39 | let doumailContact = { 40 | id: userId, 41 | contact: { 42 | id: userId, 43 | name: contactName, 44 | url: contactUrl, 45 | avatar: contactAvatar, 46 | }, 47 | time: time, 48 | url: doumailUrl, 49 | abstract: abstract, 50 | rank: new Date(time).getTime(), 51 | }; 52 | let readMore = true; 53 | for (let start = 0; readMore; start += PAGE_SIZE) { 54 | let postData = new URLSearchParams(); 55 | postData.append('start', start); 56 | postData.append('target_id', userId); 57 | postData.append('ck', this.session.cookies.ck); 58 | let fetch = await this.fetch 59 | let response = await fetch(URL_DOUMAIL_LOAD_MORE, { 60 | headers: {'X-Override-Referer': doumailUrl}, 61 | method: 'POST', 62 | body: postData, 63 | }); 64 | if (response.status != 200) { 65 | throw new TaskError('豆瓣服务器返回错误'); 66 | } 67 | let json = await response.json(); 68 | readMore = json.more; 69 | if (json.err) { 70 | this.logger.warning(json.err); 71 | } 72 | let doumailList = document.createElement('DIV'); 73 | doumailList.innerHTML = json.html; 74 | let lastDate = null; 75 | for (let div of doumailList.children) { 76 | if (div.className == 'split-line') { 77 | lastDate = div.innerText.trim(); 78 | } else if (div.className == 'chat') { 79 | let chatId = parseInt(div.getAttribute('data')); 80 | let time = div.querySelector('.info>.time').innerText; 81 | let datetime = `${lastDate} ${time}`; 82 | let senderAvatarImg = div.querySelector('.pic img'); 83 | let senderAvatar = senderAvatarImg.src; 84 | let senderName = senderAvatarImg.alt; 85 | let senderAnchor = div.querySelector('.pic>a'); 86 | let senderUrl = senderAnchor ? senderAnchor.href : null; 87 | let content = div.querySelector('.content'); 88 | let contentSender = content.querySelector('div.sender'); 89 | contentSender && contentSender.remove(); 90 | let doumail = { 91 | id: chatId, 92 | contact: userId, 93 | sender: { 94 | avatar: senderAvatar, 95 | name: senderName, 96 | url: senderUrl, 97 | }, 98 | datetime: datetime, 99 | content: content.innerHTML, 100 | }; 101 | await this.storage.doumail.put(doumail); 102 | } 103 | } 104 | } 105 | await this.storage.doumailContact.put(doumailContact); 106 | this.step(); 107 | } 108 | } 109 | this.complete(); 110 | } 111 | 112 | get name() { 113 | return '豆邮'; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /extension/tasks/files.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Settings from '../settings.js'; 3 | import Task from '../services/Task.js'; 4 | import TaskError from '../services/TaskError.js'; 5 | 6 | export const TASK_FILES_SETTINGS = { 7 | '同步图片.cloudName': '', 8 | }; 9 | 10 | const UPLOAD_URL = 'https://api.cloudinary.com/v1_1/{cloud}/image/upload'; 11 | const PAGE_SIZE = 100; 12 | 13 | 14 | function encodeContext(context) { 15 | let contextArray = []; 16 | for (let key in context) { 17 | let value = context[key]; 18 | if (value.length > 100) { 19 | value = value.substring(0, 100); 20 | } 21 | key = key.replaceAll('|', '\|').replaceAll('=', '\='); 22 | value = value.replaceAll('|', '\|').replaceAll('=', '\='); 23 | contextArray.push(`${key}=${value}`); 24 | } 25 | return contextArray.join('|'); 26 | } 27 | 28 | 29 | export default class Files extends Task { 30 | async addFile(url, tags, meta, path) { 31 | if (!url) { 32 | return; 33 | } 34 | try { 35 | await this.storage.files.add({ 36 | url: url, 37 | tags: tags, 38 | meta: meta, 39 | path: path, 40 | }); 41 | } catch (e) { 42 | if (e.name != 'ConstraintError') { 43 | this.logger.warning(e.message); 44 | } 45 | } 46 | } 47 | 48 | async extractImages() { 49 | let escapeFolderName = name => { 50 | name = name.replaceAll('?', '？') 51 | .replaceAll('&', '＆') 52 | .replaceAll('#', '＃') 53 | .replaceAll('\\', '＼') 54 | .replaceAll('%', '％') 55 | .replaceAll('<', '＜') 56 | .replaceAll('>', '＞') 57 | .replaceAll('/', '／'); 58 | return name; 59 | }; 60 | 61 | await this.storage.transaction('rw', this.storage.album, this.storage.files, async () => { 62 | await this.storage.album.each(async item => { 63 | await this.addFile( 64 | item.album.cover_url, 65 | ['相册'], 66 | { 67 | caption: item.album.title, 68 | alt: item.album.description, 69 | from: item.album.url, 70 | }, 71 | 'thumbnail' 72 | ); 73 | }); 74 | }); 75 | await this.storage.transaction('rw', this.storage.album, this.storage.photo, this.storage.files, async () => { 76 | await this.storage.photo.each(async item => { 77 | let {album} = await this.storage.album.get(item.album); 78 | let meta = { 79 | caption: album.title, 80 | alt: item.photo.description, 81 | from: item.photo.url, 82 | }; 83 | await this.addFile( 84 | item.photo.cover, 85 | ['照片'], 86 | meta, 87 | 'thumbnail' 88 | ); 89 | await this.addFile( 90 | item.photo.raw, 91 | ['照片'], 92 | meta, 93 | '相册/' + escapeFolderName(album.title) 94 | ); 95 | }); 96 | }); 97 | await this.storage.transaction('rw', this.storage.note, this.storage.files, async () => { 98 | await this.storage.note.each(async item => { 99 | let images = this.parseHTML(item.note.fulltext).querySelectorAll('img'); 100 | for (let image of images) { 101 | await this.addFile( 102 | image.src, 103 | ['日记'], 104 | { 105 | caption: item.note.title, 106 | alt: item.note.abstract, 107 | from: item.note.url, 108 | }, 109 | '日记/' + escapeFolderName(item.note.title) 110 | ); 111 | } 112 | }); 113 | }); 114 | await this.storage.transaction('rw', this.storage.review, this.storage.files, async () => { 115 | await this.storage.review.each(async item => { 116 | let images = this.parseHTML(item.review.fulltext).querySelectorAll('img'); 117 | for (let image of images) { 118 | await this.addFile( 119 | image.src, 120 | ['评论'], 121 | { 122 | caption: item.review.title, 123 | alt: item.review.abstract, 124 | from: item.review.url, 125 | }, 126 | '评论/' + escapeFolderName(item.review.title) 127 | ); 128 | } 129 | }); 130 | }); 131 | await this.storage.transaction('rw', this.storage.annotation, this.storage.files, async () => { 132 | await this.storage.annotation.each(async item => { 133 | let images = this.parseHTML(item.annotation.fulltext).querySelectorAll('img'); 134 | for (let image of images) { 135 | await this.addFile( 136 | image.src, 137 | ['笔记'], 138 | { 139 | caption: item.annotation.title, 140 | alt: item.annotation.abstract, 141 | from: item.annotation.url, 142 | }, 143 | '笔记/' + escapeFolderName(item.annotation.title) 144 | ); 145 | } 146 | }); 147 | }); 148 | await this.storage.transaction('rw', this.storage.status, this.storage.files, async () => { 149 | await this.storage.status.each(async item => { 150 | if (item.status.images) { 151 | let statusUrl = item.status.sharing_url; 152 | for (let image of item.status.images) { 153 | await this.addFile(image.large.url, ['广播'], { from: statusUrl }, '广播'); 154 | await this.addFile(image.normal.url, ['广播'], { from: statusUrl }, 'thumbnail'); 155 | } 156 | } 157 | if (item.status.reshared_status && item.status.reshared_status.images) { 158 | let statusUrl = item.status.reshared_status.sharing_url; 159 | for (let image of item.status.reshared_status.images) { 160 | await this.addFile(image.large.url, ['广播'], { from: statusUrl }, '广播'); 161 | await this.addFile(image.normal.url, ['广播'], { from: statusUrl }, 'thumbnail'); 162 | } 163 | } 164 | }); 165 | }); 166 | } 167 | 168 | async run() { 169 | let settings = await Settings.load(TASK_FILES_SETTINGS); 170 | Settings.apply(this, settings); 171 | if (!this.cloudName) { 172 | this.logger.warning('Missing setting of cloudinary cloud name.'); 173 | return; 174 | } 175 | 176 | await this.extractImages(); 177 | 178 | this.total = await this.storage.files.filter(row => { 179 | return !(row.save); 180 | }).count(); 181 | if (this.total == 0) { 182 | return; 183 | } 184 | 185 | let escapePath = path => { 186 | let dirs = path.split('/'); 187 | let folderName = dirs.pop(); 188 | if (folderName.trim() == '') { 189 | folderName = folderName.replaceAll(' ', '␣'); 190 | } else if (folderName.startsWith(' ')) { 191 | folderName = folderName.replace('.', '␣'); 192 | } else if (folderName.replaceAll('.', '') == '') { 193 | folderName = folderName.replaceAll('.', '·'); 194 | } else if (folderName.startsWith('.')) { 195 | folderName = folderName.replace('.', '·'); 196 | } 197 | dirs.push(folderName); 198 | return dirs.join('/'); 199 | }; 200 | 201 | let uploadURL = UPLOAD_URL.replace('{cloud}', this.cloudName); 202 | 203 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 204 | for (let i = 0; i < pageCount; i ++) { 205 | let rows = await this.storage.files.filter(row => { 206 | return !(row.save); 207 | }).limit(PAGE_SIZE).toArray(); 208 | 209 | for (let row of rows) { 210 | if (!row.url) { 211 | this.step(); 212 | continue; 213 | } 214 | let postData = new URLSearchParams(); 215 | postData.append('file', row.url); 216 | postData.append('upload_preset', 'douban'); 217 | postData.append('tags', row.tags); 218 | postData.append('context', encodeContext(row.meta)); 219 | postData.append('folder', `${this.targetUser.uid}/${escapePath(row.path)}`); 220 | 221 | let fetch = await this.fetch 222 | let response = await fetch(uploadURL, { 223 | method: 'POST', 224 | body: postData, 225 | }, true); 226 | if (response.status >= 500) { 227 | throw new TaskError('Cloudinary 接口异常'); 228 | } 229 | let savedData = await response.json(); 230 | if (response.status == 400 && !savedData['error']['message'].startsWith('Error in loading http')) { 231 | throw new TaskError('Cloudinary 接口返回错误'); 232 | } 233 | await this.storage.files.update(row.id, { 234 | save: savedData, 235 | }) 236 | this.step(); 237 | } 238 | } 239 | 240 | this.complete(); 241 | } 242 | 243 | get name() { 244 | return '同步图片'; 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /extension/tasks/follower.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | const API_PAGE_SIZE = 50; 6 | const WEB_PAGE_SIZE = 20; 7 | const OTHER_USER_PAGE_SIZE = 70; 8 | const URL_FOLLOWERS = 'https://m.douban.com/rexxar/api/v2/user/{uid}/followers?start={start}&count=50&ck={ck}&for_mobile=1'; 9 | const URL_FOLLOWERS_WEBPAGE = 'https://www.douban.com/contacts/rlist?start={start}'; 10 | const URL_FOLLOWERS_OTHER_USER = 'https://www.douban.com/people/{uid}/rev_contacts?start={start}'; 11 | 12 | 13 | export default class Follower extends Task { 14 | async crawlByApi() { 15 | let baseURL = URL_FOLLOWERS 16 | .replace('{ck}', this.session.cookies.ck) 17 | .replace('{uid}', this.targetUser.id); 18 | 19 | let pageCount = 1; 20 | for (let i = 0; i < pageCount; i ++) { 21 | let fetch = await this.fetch 22 | let response = await fetch(baseURL.replace('{start}', i * API_PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/follower'}}); 23 | if (response.status !== 200) { 24 | throw new TaskError('豆瓣服务器返回错误'); 25 | } 26 | let json = await response.json(); 27 | pageCount = Math.ceil(json.total / API_PAGE_SIZE); 28 | for (let user of json.users) { 29 | let row = { 30 | version: this.jobId, 31 | user: user, 32 | }; 33 | await this.storage.follower.put(row); 34 | this.step(); 35 | } 36 | } 37 | } 38 | 39 | async crawlByWebpage() { 40 | let totalPage = 1; 41 | for (let i = 0; i < totalPage; i ++) { 42 | let fetch = await this.fetch 43 | let response = await fetch(URL_FOLLOWERS_WEBPAGE.replace('{start}', i * WEB_PAGE_SIZE)); 44 | if (response.status !== 200) { 45 | throw new TaskError('豆瓣服务器返回错误'); 46 | } 47 | let html = this.parseHTML(await response.text()); 48 | try { 49 | totalPage = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 50 | } catch (e) {} 51 | for (let li of html.querySelectorAll('.user-list>li')) { 52 | let idText = li.id.substr(1); 53 | let avatar = li.querySelector('.face'); 54 | let userLink = li.querySelector('.info>h3>a').href; 55 | let loc = null; 56 | let userInfo = li.querySelector('.info>p'); 57 | if (userInfo.childElementCount === 3) { 58 | loc = { name: userInfo.firstChild.textContent.trim() }; 59 | } 60 | let followInfo = userInfo.querySelectorAll('b'); 61 | let followers = followInfo[0].innerText; 62 | let following = followInfo[1].innerText; 63 | 64 | let row = { 65 | version: this.jobId, 66 | user: { 67 | avatar: avatar.src, 68 | id: idText, 69 | loc: loc, 70 | name: avatar.alt, 71 | uid: userLink.match(/https:\/\/www\.douban\.com\/people\/(.+)\//)[1], 72 | uri: 'douban://douban.com/user/' + idText, 73 | url: userLink, 74 | followers_count: followers, 75 | following_count: following, 76 | } 77 | }; 78 | await this.storage.follower.put(row); 79 | this.step(); 80 | } 81 | } 82 | } 83 | 84 | async crawlOtherUserByWebpage() { 85 | let totalPage = 1; 86 | for (let i = 0; i < totalPage; i ++) { 87 | let fetch = await this.fetch 88 | let response = await fetch( 89 | URL_FOLLOWERS_OTHER_USER 90 | .replace('{uid}', this.targetUser.id) 91 | .replace('{start}', i * OTHER_USER_PAGE_SIZE) 92 | ); 93 | if (response.status !== 200) { 94 | throw new TaskError('豆瓣服务器返回错误'); 95 | } 96 | let html = this.parseHTML(await response.text()); 97 | try { 98 | totalPage = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 99 | } catch (e) {} 100 | for (let anchor of html.querySelectorAll('.obu .nbg')) { 101 | let avatar = anchor.querySelector('img'); 102 | let userLink = anchor.href; 103 | let matches = avatar.src.match(/\/icon\/u(\d+)-\d+.jpg$/); 104 | let idText = matches ? matches[1] : null; 105 | let uid = userLink.match(/https:\/\/www\.douban\.com\/people\/(.+)\//)[1]; 106 | 107 | let row = { 108 | version: this.jobId, 109 | user: { 110 | avatar: avatar.src, 111 | id: idText, 112 | name: avatar.alt, 113 | uid: uid, 114 | uri: 'douban://douban.com/user/' + (idText || uid), 115 | url: userLink, 116 | } 117 | }; 118 | await this.storage.follower.put(row); 119 | this.step(); 120 | } 121 | } 122 | } 123 | 124 | async run() { 125 | this.total = this.targetUser.followers_count; 126 | if (this.total === 0) { 127 | return; 128 | } 129 | await this.storage.table('version').put({table: 'follower', version: this.jobId, updated: Date.now()}); 130 | if (this.total > 5000) { 131 | this.isOtherUser ? 132 | await this.crawlOtherUserByWebpage() : 133 | await this.crawlByWebpage(); 134 | } else { 135 | await this.crawlByApi(); 136 | } 137 | this.complete(); 138 | } 139 | 140 | get name() { 141 | return '被关注'; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /extension/tasks/following.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const API_PAGE_SIZE = 50; 7 | const URL_FOLLOWING_API = 'https://m.douban.com/rexxar/api/v2/user/{uid}/following?start={start}&count=50&ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Following extends Task { 11 | async run() { 12 | this.total = this.targetUser.following_count; 13 | if (this.total == 0) { 14 | return; 15 | } 16 | await this.storage.table('version').put({table: 'following', version: this.jobId, updated: Date.now()}); 17 | 18 | let baseURL = URL_FOLLOWING_API 19 | .replace('{ck}', this.session.cookies.ck) 20 | .replace('{uid}', this.targetUser.id); 21 | 22 | let pageCount = 1; 23 | for (let i = 0; i < pageCount; i ++) { 24 | let fetch = await this.fetch 25 | let response = await fetch(baseURL.replace('{start}', i * API_PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/followed'}}); 26 | if (response.status != 200) { 27 | throw new TaskError('豆瓣服务器返回错误'); 28 | } 29 | let json = await response.json(); 30 | pageCount = Math.ceil(json.total / API_PAGE_SIZE); 31 | for (let user of json.users) { 32 | let row = { 33 | version: this.jobId, 34 | user: user, 35 | }; 36 | await this.storage.following.put(row); 37 | this.step(); 38 | } 39 | } 40 | 41 | this.complete(); 42 | } 43 | 44 | get name() { 45 | return '关注'; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /extension/tasks/interest.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 50; 7 | const URL_INTERESTS = 'https://m.douban.com/rexxar/api/v2/user/{uid}/interests?type={type}&status={status}&start={start}&count={count}&ck={ck}&for_mobile=1'; 8 | const URL_TOTAL = 'https://m.douban.com/rexxar/api/v2/user/{uid}/interests?ck={ck}&count=1&for_mobile=1'; 9 | 10 | 11 | export default class Interest extends Task { 12 | async getTotal() { 13 | let totalURL = URL_TOTAL 14 | .replace('{ck}', this.session.cookies.ck) 15 | .replace('{uid}', this.targetUser.id); 16 | let fetch = await this.fetch 17 | let response = await fetch(totalURL, {headers: {'X-Override-Referer': 'https://m.douban.com/mine/'}}); 18 | if (response.status !== 200) { 19 | throw new TaskError('豆瓣服务器返回错误'); 20 | } 21 | let json = await response.json(); 22 | return parseInt(json.total); 23 | } 24 | 25 | compareInterest(l, r) { 26 | if (l.status !== r.status) return false; 27 | if (l.comment !== r.comment) return false; 28 | if (l.rating !== r.rating) { 29 | if (l.rating && r.rating) { 30 | if (l.rating.value !== r.rating.value) { 31 | return false; 32 | } 33 | } else { 34 | return false; 35 | } 36 | } 37 | if (l.tags.sort().toString() !== r.tags.sort().toString()) return false; 38 | return true; 39 | } 40 | 41 | async processInterest(interest, version, type) 42 | { 43 | if (!interest || Object.keys(interest).length === 0) { 44 | console.warn("Empty interest object received."); 45 | return; 46 | } 47 | let subjectId = parseInt(interest.subject.id) 48 | let interestId = parseInt(interest.id); 49 | let row = await this.storage.interest.get({ subject: subjectId }); 50 | if (row) { 51 | let lastVersion = row.version; 52 | let changed = false; 53 | row.version = version; 54 | if (row.id !== interestId) { 55 | await this.storage.interest.delete(row.id); 56 | row.id = interestId; 57 | changed = true; 58 | } else { 59 | changed = !this.compareInterest(row.interest, interest); 60 | } 61 | if (changed) { 62 | !row.history && (row.history = {}); 63 | row.history[lastVersion] = row.interest; 64 | row.status = interest.status; 65 | row.interest = interest; 66 | } 67 | } else { 68 | row = { 69 | id: interestId, 70 | subject: subjectId, 71 | version: version, 72 | type: type, 73 | status: interest.status, 74 | interest: interest, 75 | }; 76 | } 77 | await this.storage.interest.put(row); 78 | this.step(); 79 | } 80 | 81 | async run() { 82 | let version = this.jobId; 83 | this.total = await this.getTotal(); 84 | if (this.total === 0) { 85 | return; 86 | } 87 | await this.storage.table('version').put({table: 'interest', version: version, updated: Date.now()}); 88 | 89 | let baseURL = URL_INTERESTS 90 | .replace('{ck}', this.session.cookies.ck) 91 | .replace('{uid}', this.targetUser.id); 92 | 93 | // for (let type of ['movie']) { 94 | for (let type of ['game', 'music', 'book', 'movie', 'drama']) { 95 | let urlWithType = baseURL.replace('{type}', type); 96 | 97 | for (let status of ['mark', 'doing', 'done']) { 98 | let urlWithStatus = urlWithType.replace('{status}', status); 99 | let pageCount = 1; 100 | for (let i = 0; i < pageCount; i ++) { 101 | let start = i * PAGE_SIZE; 102 | let urlToFetch = urlWithStatus 103 | .replace('{start}', start) 104 | .replace('{count}', PAGE_SIZE); 105 | let fetch = await this.fetch 106 | let response = await fetch(urlToFetch, {headers: {'X-Override-Referer': 'https://m.douban.com/mine/' + type}}); 107 | if (response.status !== 200) { 108 | if (response.status === 500) { 109 | // try to fetch this page sliced 110 | for (let j = 0; j < PAGE_SIZE; j ++) { 111 | let startSliced = start + j; 112 | let urlToFetchSliced = urlWithStatus 113 | .replace('{start}', startSliced) 114 | .replace('{count}', 1); 115 | let fetch = await this.fetch 116 | let response = await fetch(urlToFetchSliced, {headers: {'X-Override-Referer': 'https://m.douban.com/mine/' + type}}); 117 | if (response.status !== 200) { 118 | // skip douban server error 119 | continue; 120 | } 121 | let json = await response.json(); 122 | for (let interest of json.interests) { 123 | await this.processInterest(interest, version, type); 124 | } 125 | if (json.total === startSliced) { 126 | continue; 127 | } 128 | } 129 | } else { 130 | throw new TaskError('豆瓣服务器返回错误: ' + response.status); 131 | } 132 | continue; 133 | } 134 | let json = await response.json(); 135 | pageCount = Math.ceil(json.total / PAGE_SIZE); 136 | for (let interest of json.interests) { 137 | await this.processInterest(interest, version, type); 138 | } 139 | } 140 | } 141 | } 142 | this.complete(); 143 | } 144 | 145 | get name() { 146 | return '书/影/音/游'; 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /extension/tasks/migrate/annotation.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../../services/Task.js'; 3 | import TaskError from '../../services/TaskError.js'; 4 | import Draft from '../../vendor/draft.js'; 5 | 6 | 7 | const URL_ANNOTATION_PUBLISH = 'https://book.douban.com/j/annotation/{nid}/publish'; 8 | const URL_ANNOTATION_CREATE_REFERER = 'https://book.douban.com/annotation/write?sid={subject}'; 9 | const URL_ANNOTATION_IMAGE_UPLOAD = 'https://book.douban.com/j/annotation/{nid}/upload'; 10 | const PAGE_SIZE = 100; 11 | const WORD_COUNT_LIMIT = 10; 12 | 13 | 14 | export default class Annotation extends Task { 15 | async getInitialData(createURL) { 16 | let fetch = await this.fetch 17 | let response = await fetch(createURL); 18 | let html = this.parseHTML(await response.text()); 19 | let input = html.querySelector('#review-editor-form>input[name="nid"]'); 20 | if (!input) { 21 | throw new TaskError('Cannot find "nid" value.'); 22 | } 23 | 24 | let script = html.querySelectorAll('script')[2]; 25 | let match = script.text.match(/name: 'upload_auth_token',\s value: '([\s\S]*?)'/m); 26 | if (!match) { 27 | throw new TaskError('Cannot find upload auth token.'); 28 | } 29 | 30 | return { 31 | nid: input.value, 32 | uploadAuthToken: match[1], 33 | }; 34 | } 35 | 36 | async uploadImages(draft, uploadAuthToken, uploadURL) { 37 | let uploadForm = new FormData(); 38 | uploadForm.append('ck', this.session.cookies.ck); 39 | uploadForm.append('upload_auth_token', uploadAuthToken); 40 | 41 | for (let i in draft.entities) { 42 | let entity = draft.entities[i]; 43 | if (entity.type != 'IMAGE') continue; 44 | let entityData = entity.data; 45 | let fetch = await this.fetch 46 | let imageResponse = await fetch(entityData.src, { 47 | 'X-Override-Referer': 'https://www.douban.com/', 48 | }, true); 49 | let imageBlob = await imageResponse.blob(); 50 | let imageURL = new URL(entityData.src); 51 | let filename = imageURL.pathname.split('/').pop(); 52 | uploadForm.set('image', imageBlob, filename); 53 | let uploadResponse = await fetch(uploadURL, {method: 'POST', body: uploadForm}, true); 54 | let uploadedImage = await uploadResponse.json(); 55 | entity.data = uploadedImage.photo; 56 | entity.data.src = entity.data.url; 57 | } 58 | } 59 | 60 | async run() { 61 | this.total = await this.storage.annotation.count(); 62 | if (this.total == 0) { 63 | return; 64 | } 65 | 66 | let postData = new URLSearchParams(); 67 | postData.append('ck', this.session.cookies.ck); 68 | postData.append('is_rich', '1'); 69 | 70 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 71 | for (let i = 0; i < pageCount; i ++) { 72 | let rows = await this.storage.annotation 73 | .offset(PAGE_SIZE * i).limit(PAGE_SIZE) 74 | .toArray(); 75 | for (let row of rows) { 76 | let createURL = URL_ANNOTATION_CREATE_REFERER.replace('{subject}', row.subject); 77 | let {nid, uploadAuthToken} = await this.getInitialData(createURL); 78 | let annotation = row.annotation; 79 | let html = this.parseHTML(annotation.fulltext); 80 | let body = html.querySelector('body'); 81 | 82 | let draft = new Draft(); 83 | draft.feed(body); 84 | let wordPadding = WORD_COUNT_LIMIT - draft.count(); 85 | if (wordPadding > 0) { 86 | draft.addBlock('unstyled').write(''.padEnd(wordPadding, '=')).end(); 87 | } 88 | await this.uploadImages(draft, uploadAuthToken, URL_ANNOTATION_IMAGE_UPLOAD.replace('{nid}', nid)); 89 | 90 | postData.set('chapter', annotation.chapter); 91 | postData.set('page', annotation.page || 1); 92 | postData.set('content', JSON.stringify(draft.toArray())); 93 | let fetch = await this.fetch 94 | let response = await fetch(URL_ANNOTATION_PUBLISH.replace('{nid}', nid), { 95 | headers: { 96 | 'X-Override-Referer': createURL, 97 | 'X-Requested-With': 'XMLHttpRequest', 98 | 'X-Override-Origin': 'https://www.douban.com', 99 | }, 100 | method: 'POST', 101 | body: postData, 102 | }); 103 | let result = await response.json(); 104 | if (result.r == 0) { 105 | this.logger.info('Success to publish annotation:' + annotation.title); 106 | } else { 107 | this.logger.warning('Fail to publish annotation:' + annotation.title); 108 | } 109 | this.step(); 110 | } 111 | } 112 | this.complete(); 113 | } 114 | 115 | get name() { 116 | return '发布笔记'; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /extension/tasks/migrate/blacklist.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../../services/Task.js'; 3 | import TaskError from '../../services/TaskError.js'; 4 | 5 | 6 | const URL_FORBID = 'https://www.douban.com/j/contact/addtoblacklist'; 7 | const PAGE_SIZE = 100; 8 | 9 | 10 | export default class Blacklist extends Task { 11 | async run() { 12 | this.total = await this.storage.blacklist.count(); 13 | if (this.total == 0) { 14 | return; 15 | } 16 | 17 | let postData = new URLSearchParams(); 18 | postData.append('ck', this.session.cookies.ck); 19 | 20 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 21 | for (let i = 0; i < pageCount; i ++) { 22 | let rows = await this.storage.blacklist 23 | .offset(PAGE_SIZE * i).limit(PAGE_SIZE) 24 | .reverse().toArray(); 25 | for (let row of rows) { 26 | let uid = row.user.id || row.user.uid; 27 | postData.set('people', uid); 28 | let fetch = await this.fetch 29 | let response = await fetch(URL_FORBID, { 30 | headers: { 31 | 'X-Override-Referer': 'https://www.douban.com/', 32 | 'X-Requested-With': 'XMLHttpRequest', 33 | 'X-Override-Origin': 'https://www.douban.com', 34 | }, 35 | method: 'POST', 36 | body: postData, 37 | }); 38 | let result = await response.json(); 39 | if (result.result) { 40 | this.logger.info('Success to forbid user:' + uid); 41 | } else { 42 | this.logger.warning('Fail to forbid user:' + uid); 43 | } 44 | this.step(); 45 | } 46 | } 47 | this.complete(); 48 | } 49 | 50 | get name() { 51 | return '加黑名单'; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /extension/tasks/migrate/follow.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../../services/Task.js'; 3 | import TaskError from '../../services/TaskError.js'; 4 | 5 | 6 | const URL_FOLLOW = 'https://www.douban.com/j/contact/addcontact'; 7 | const PAGE_SIZE = 100; 8 | 9 | 10 | export default class Follow extends Task { 11 | async run() { 12 | this.total = await this.storage.following.count(); 13 | if (this.total == 0) { 14 | return; 15 | } 16 | 17 | let postData = new URLSearchParams(); 18 | postData.append('ck', this.session.cookies.ck); 19 | 20 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 21 | for (let i = 0; i < pageCount; i ++) { 22 | let rows = await this.storage.following 23 | .offset(PAGE_SIZE * i).limit(PAGE_SIZE) 24 | .reverse().toArray(); 25 | for (let row of rows) { 26 | let uid = row.user.id || row.user.uid; 27 | postData.set('people', uid); 28 | let fetch = await this.fetch 29 | let response = await fetch(URL_FOLLOW, { 30 | headers: { 31 | 'X-Override-Referer': 'https://www.douban.com/', 32 | 'X-Requested-With': 'XMLHttpRequest', 33 | 'X-Override-Origin': 'https://www.douban.com', 34 | }, 35 | method: 'POST', 36 | body: postData, 37 | }); 38 | let result = await response.json(); 39 | if (result.result) { 40 | this.logger.info('Success to follow user:' + uid); 41 | } else { 42 | this.logger.warning('Fail to follow user:' + uid); 43 | } 44 | this.step(); 45 | } 46 | } 47 | this.complete(); 48 | } 49 | 50 | get name() { 51 | return '加关注'; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /extension/tasks/migrate/interest.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../../services/Task.js'; 3 | import TaskError from '../../services/TaskError.js'; 4 | 5 | 6 | const URL_INTEREST = { 7 | movie: 'https://movie.douban.com/j/subject/{subject_id}/interest', 8 | music: 'https://music.douban.com/j/subject/{subject_id}/interest', 9 | book: 'https://book.douban.com/j/subject/{subject_id}/interest', 10 | game: 'https://www.douban.com/j/ilmen/thing/{subject_id}/interest', 11 | drama: 'https://www.douban.com/j/location/drama/{subject_id}/interest', 12 | }; 13 | 14 | const MARKS = { 15 | mark: 'wish', 16 | doing: 'do', 17 | done: 'collect', 18 | }; 19 | 20 | const PAGE_SIZE = 100; 21 | 22 | 23 | export default class Note extends Task { 24 | async run() { 25 | this.total = await this.storage.interest.count(); 26 | if (this.total == 0) { 27 | return; 28 | } 29 | 30 | let postData = new URLSearchParams(); 31 | postData.append('ck', this.session.cookies.ck); 32 | postData.append('foldcollect', 'F'); 33 | 34 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 35 | for (let i = 0; i < pageCount; i ++) { 36 | let rows = await this.storage.interest 37 | .offset(PAGE_SIZE * i).limit(PAGE_SIZE) 38 | .toArray(); 39 | for (let row of rows) { 40 | let interest = row.interest; 41 | postData.set('rating', interest.rating ? interest.rating.value : ''); 42 | postData.set('interest', MARKS[row.status]); 43 | postData.set('tags', interest.tags ? interest.tags.join(' ') : ''); 44 | postData.set('comment', interest.comment + ' @' + interest.create_time); 45 | 46 | let fetch = await this.fetch 47 | let response = await fetch(URL_INTEREST[row.type].replace('{subject_id}', row.subject), { 48 | headers: { 49 | 'X-Override-Referer': 'https://www.douban.com/', 50 | 'X-Requested-With': 'XMLHttpRequest', 51 | 'X-Override-Origin': 'https://www.douban.com', 52 | }, 53 | method: 'POST', 54 | body: postData, 55 | }); 56 | let result = await response.json(); 57 | this.step(); 58 | } 59 | } 60 | this.complete(); 61 | } 62 | 63 | get name() { 64 | return '标记影音书游剧'; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /extension/tasks/migrate/note.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../../services/Task.js'; 3 | import TaskError from '../../services/TaskError.js'; 4 | import Draft from '../../vendor/draft.js'; 5 | 6 | 7 | const URL_NOTE_PUBLISH = 'https://www.douban.com/j/note/publish'; 8 | const URL_NOTE_CREATE_REFERER = 'https://www.douban.com/note/create'; 9 | const PAGE_SIZE = 100; 10 | 11 | 12 | export default class Note extends Task { 13 | getIntro(html) { 14 | let intro = html.querySelector('div.introduction'); 15 | if (intro) { 16 | let introText = intro.innerText; 17 | intro.remove(); 18 | return introText; 19 | } 20 | return ''; 21 | } 22 | 23 | async run() { 24 | this.total = await this.storage.note.count(); 25 | if (this.total == 0) { 26 | return; 27 | } 28 | 29 | let postData = new URLSearchParams(); 30 | postData.append('ck', this.session.cookies.ck); 31 | postData.append('is_rich', '1'); 32 | postData.append('note_id', ''); 33 | postData.append('note_privacy', 'X'); 34 | postData.append('action', 'new'); 35 | 36 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 37 | for (let i = 0; i < pageCount; i ++) { 38 | let rows = await this.storage.note 39 | .offset(PAGE_SIZE * i).limit(PAGE_SIZE) 40 | .toArray(); 41 | for (let row of rows) { 42 | let note = row.note; 43 | let html = this.parseHTML(note.fulltext).querySelector('body'); 44 | let intro = this.getIntro(html); 45 | 46 | let draft = new Draft(); 47 | draft.feed(html); 48 | 49 | postData.set('introduction', intro); 50 | postData.set('note_title', note.title); 51 | postData.set('note_text', JSON.stringify(draft.toArray())); 52 | 53 | let fetch = await this.fetch 54 | let response = await fetch(URL_NOTE_PUBLISH, { 55 | headers: { 56 | 'X-Override-Referer': URL_NOTE_CREATE_REFERER, 57 | 'X-Requested-With': 'XMLHttpRequest', 58 | 'X-Override-Origin': 'https://www.douban.com', 59 | }, 60 | method: 'POST', 61 | body: postData, 62 | }); 63 | let result = await response.json(); 64 | if (!result.error) { 65 | this.logger.info('Success to publish note:' + note.title); 66 | } else { 67 | this.logger.warning('Fail to publish note:' + note.title); 68 | } 69 | this.step(); 70 | } 71 | } 72 | this.complete(); 73 | } 74 | 75 | get name() { 76 | return '发布日记'; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /extension/tasks/migrate/review.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../../services/Task.js'; 3 | import TaskError from '../../services/TaskError.js'; 4 | import Draft from '../../vendor/draft.js'; 5 | 6 | 7 | const URL_REVIEW_PUBLISH = 'https://www.douban.com/j/review/create'; 8 | const URL_REVIEW_CREATE_REFERER = 'https://www.douban.com/subject/{subject}/new_review'; 9 | const PAGE_SIZE = 100; 10 | const WORD_COUNT_LIMIT = 140; 11 | 12 | 13 | export default class Review extends Task { 14 | getIntro(html) { 15 | let intro = html.querySelector('div.introduction'); 16 | if (intro) { 17 | let introText = intro.innerText; 18 | intro.remove(); 19 | return introText; 20 | } 21 | return ''; 22 | } 23 | 24 | async uploadImages() { 25 | 26 | } 27 | 28 | async run() { 29 | this.total = await this.storage.review.count(); 30 | if (this.total == 0) { 31 | return; 32 | } 33 | 34 | let postData = new URLSearchParams(); 35 | postData.append('ck', this.session.cookies.ck); 36 | postData.append('is_rich', '1'); 37 | postData.append('topic_id', ''); 38 | postData.set('review[rating]', ''); 39 | postData.set('review[spoiler]', ''); 40 | postData.set('review[donate]', ''); 41 | postData.set('review[original]', ''); 42 | 43 | let pageCount = Math.ceil(this.total / PAGE_SIZE); 44 | for (let i = 0; i < pageCount; i ++) { 45 | let rows = await this.storage.review 46 | .offset(PAGE_SIZE * i).limit(PAGE_SIZE) 47 | .toArray(); 48 | for (let row of rows) { 49 | let review = row.review; 50 | let html = this.parseHTML(review.fulltext).querySelector('body'); 51 | let intro = this.getIntro(html); 52 | 53 | let draft = new Draft(); 54 | draft.feed(html); 55 | 56 | let wordPadding = WORD_COUNT_LIMIT - draft.count(); 57 | if (wordPadding > 0) { 58 | draft.addBlock('unstyled').write(''.padEnd(wordPadding, '=')).end(); 59 | } 60 | 61 | postData.set('review[introduction]', intro); 62 | postData.set('review[subject_id]', row.subject); 63 | postData.set('review[title]', review.title); 64 | postData.set('review[text]', JSON.stringify(draft.toArray())); 65 | 66 | let fetch = await this.fetch 67 | let response = await fetch(URL_REVIEW_PUBLISH, { 68 | headers: { 69 | 'X-Override-Referer': URL_REVIEW_CREATE_REFERER.replace('{subject}', row.subject), 70 | 'X-Requested-With': 'XMLHttpRequest', 71 | 'X-Override-Origin': 'https://www.douban.com', 72 | }, 73 | method: 'POST', 74 | body: postData, 75 | }); 76 | let result = await response.json(); 77 | if (result.result) { 78 | this.logger.info('Success to publish review:' + review.title); 79 | } else { 80 | this.logger.warning('Fail to publish review:' + review.title); 81 | } 82 | this.step(); 83 | } 84 | } 85 | this.complete(); 86 | } 87 | 88 | get name() { 89 | return '发布评论'; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /extension/tasks/mock.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | import Drafter from '../vendor/draft.js'; 5 | 6 | 7 | const URL_MOCK = 'https://foo.bar/'; 8 | 9 | 10 | class Mock extends Task { 11 | async run() { 12 | let fetch = await this.fetch 13 | let response = await fetch(URL_MOCK); 14 | } 15 | 16 | get name() { 17 | return 'Mock'; 18 | } 19 | } 20 | 21 | 22 | export default class Test extends Task { 23 | async run() { 24 | let row = await this.storage.note.limit(1); 25 | let note = this.parseHTML(row.note.fulltext); 26 | let drafter = new Drafter(); 27 | drafter.feed(note); 28 | console.log(drafter.toArray()); 29 | } 30 | 31 | get name() { 32 | return 'Test'; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /extension/tasks/note.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 50; 7 | const URL_NOTES = 'https://m.douban.com/rexxar/api/v2/user/{uid}/notes?start={start}&count=50&ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Note extends Task { 11 | async fetchNote(url) { 12 | let fetch = await this.fetch 13 | let response = await fetch(url); 14 | if (response.status != 200) { 15 | return; 16 | } 17 | let html = this.parseHTML(await response.text()); 18 | return html.querySelector('#link-report>.note').innerHTML; 19 | } 20 | 21 | async run() { 22 | let version = this.jobId; 23 | this.total = this.targetUser.notes_count; 24 | if (this.total == 0) { 25 | return; 26 | } 27 | await this.storage.table('version').put({table: 'note', version: version, updated: Date.now()}); 28 | 29 | let baseURL = URL_NOTES 30 | .replace('{ck}', this.session.cookies.ck) 31 | .replace('{uid}', this.targetUser.id); 32 | 33 | let pageCount = 1; 34 | for (let i = 0; i < pageCount; i ++) { 35 | let fetch = await this.fetch 36 | let response = await fetch(baseURL.replace('{start}', i * PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/notes'}}); 37 | if (response.status != 200) { 38 | throw new TaskError('豆瓣服务器返回错误'); 39 | } 40 | let json = await response.json(); 41 | pageCount = Math.ceil(json.total / PAGE_SIZE); 42 | for (let note of json.notes) { 43 | let row = await this.storage.note.get(parseInt(note.id)); 44 | if (row) { 45 | let lastVersion = row.version; 46 | row.version = version; 47 | if (note.update_time != row.note.update_time || row.note.fulltext == undefined) { 48 | !row.history && (row.history = {}); 49 | row.history[lastVersion] = row.note; 50 | note.fulltext = await this.fetchNote(note.url); 51 | row.note = note; 52 | } 53 | } else { 54 | note.fulltext = await this.fetchNote(note.url); 55 | row = { 56 | id: parseInt(note.id), 57 | version: version, 58 | note: note, 59 | } 60 | } 61 | await this.storage.note.put(row); 62 | this.step(); 63 | } 64 | } 65 | this.complete(); 66 | } 67 | 68 | get name() { 69 | return '日记'; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /extension/tasks/photo.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const PAGE_SIZE = 50; 7 | const URL_PHOTOS = 'https://m.douban.com/rexxar/api/v2/user/{uid}/photo_albums?start={start}&count=50&ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Photo extends Task { 11 | compareAlbum(l, r) { 12 | if (l.description != r.description) return false; 13 | return true; 14 | } 15 | 16 | async fetchPhotoDetail(url) { 17 | let fetch = await this.fetch 18 | let response = await fetch(url); 19 | if (response.status != 200) { 20 | if (response.status < 500) { 21 | return false; 22 | } 23 | throw new TaskError('豆瓣服务器返回错误'); 24 | } 25 | let html = this.parseHTML(await response.text()); 26 | let photo = html.querySelector('.mainphoto>img'); 27 | return photo.src; 28 | } 29 | 30 | async run() { 31 | let version = this.jobId; 32 | this.total = this.targetUser.photo_albums_count; 33 | if (this.total == 0) { 34 | return; 35 | } 36 | await this.storage.table('version').put({table: 'photo', version: version, updated: Date.now()}); 37 | 38 | let baseURL = URL_PHOTOS 39 | .replace('{uid}', this.targetUser.id) 40 | .replace('{ck}', this.session.cookies.ck); 41 | 42 | let pageCount = 1; 43 | for (let i = 0; i < pageCount; i ++) { 44 | let fetch = await this.fetch 45 | let response = await fetch(baseURL.replace('{start}', i * PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/photos'}}); 46 | if (response.status != 200) { 47 | throw new TaskError('豆瓣服务器返回错误'); 48 | } 49 | let json = await response.json(); 50 | pageCount = Math.ceil(json.total / PAGE_SIZE); 51 | for (let album of json.photo_albums) { 52 | let albumId = parseInt(album.id); 53 | let albumPrivacy = album.privacy; 54 | if (isNaN(albumId)) continue; 55 | let row = await this.storage.album.get(albumId); 56 | if (row) { 57 | let lastVersion = row.version; 58 | row.version = version; 59 | if (!this.compareAlbum(album, row.album)) { 60 | !row.history && (row.history = {}); 61 | row.history[lastVersion] = row.album; 62 | row.album = album; 63 | } 64 | } else { 65 | row = { 66 | id: albumId, 67 | version: version, 68 | album: album, 69 | }; 70 | } 71 | const ALBUM_PAGE_SIZE = 18; 72 | let albumTotalPage = 1; 73 | for (let i = 0; i < albumTotalPage; i ++) { 74 | let fetch = await this.fetch 75 | let response = await fetch(album.url + '?m_start=' + i * ALBUM_PAGE_SIZE); 76 | if (response.status != 200) { 77 | if (response.status < 500) continue; 78 | throw new TaskError('豆瓣服务器返回错误'); 79 | } 80 | let html = this.parseHTML(await response.text()); 81 | try { 82 | albumTotalPage = parseInt(html.querySelector('.paginator .thispage').dataset.totalPage); 83 | } catch (e) {} 84 | for (let photoAnchor of html.querySelectorAll('.photolst_photo')) { 85 | let photoId = parseInt( 86 | photoAnchor.href.match(/https:\/\/www\.douban\.com\/photos\/photo\/(\d+)\//)[1] 87 | ); 88 | let photoImg = photoAnchor.querySelector('img'); 89 | let photoDescription = photoAnchor.title; 90 | let row = await this.storage.photo.get(photoId); 91 | if (row) { 92 | let lastVersion = row.version; 93 | row.version = version; 94 | if (row.photo.description != photoDescription || 95 | row.photo.cover != photoImg.src) { 96 | !row.history && (row.history = {}); 97 | row.history[lastVersion] = row.photo; 98 | row.photo.description = photoDescription; 99 | if (row.photo.cover != photoImg.src) { 100 | if (albumPrivacy != 'public') { 101 | let rawUrl = await this.fetchPhotoDetail(photoAnchor.href); 102 | row.photo.raw = rawUrl || photoImg.src; 103 | } else { 104 | row.photo.raw = photoImg.src.replace('/m/', '/l/'); 105 | } 106 | row.photo.cover = photoImg.src; 107 | } 108 | } 109 | } else { 110 | row = { 111 | id: photoId, 112 | album: albumId, 113 | version: version, 114 | photo: { 115 | url: photoAnchor.href, 116 | cover: photoImg.src, 117 | description: photoDescription, 118 | } 119 | } 120 | if (albumPrivacy != 'public') { 121 | let rawUrl = await this.fetchPhotoDetail(photoAnchor.href); 122 | row.photo.raw = rawUrl || photoImg.src; 123 | } else { 124 | row.photo.raw = photoImg.src.replace('/m/', '/l/'); 125 | } 126 | } 127 | await this.storage.photo.put(row); 128 | } 129 | } 130 | await this.storage.album.put(row); 131 | this.step(); 132 | } 133 | } 134 | this.complete(); 135 | } 136 | 137 | get name() { 138 | return '相册'; 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /extension/tasks/review.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | import Storage from "../storage.js"; 5 | 6 | const PAGE_SIZE = 50; 7 | const URL_REVIEWS = 'https://m.douban.com/rexxar/api/v2/user/{uid}/reviews?type={type}&start={start}&count=50&ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Review extends Task { 11 | async fetchReview(url) { 12 | let fetch = await this.fetch 13 | let response = await fetch(url); 14 | if (response.status !== 200) { 15 | return; 16 | } 17 | let html = this.parseHTML(await response.text()); 18 | return html.querySelector('.review-content').innerHTML; 19 | } 20 | 21 | async run() { 22 | let version = this.jobId; 23 | this.total = this.targetUser.reviews_count; 24 | if (this.total === 0) { 25 | return; 26 | } 27 | await this.storage.table('version').put({table: 'review', version: version, updated: Date.now()}); 28 | 29 | let baseURL = URL_REVIEWS 30 | .replace('{ck}', this.session.cookies.ck) 31 | .replace('{uid}', this.targetUser.id); 32 | 33 | for (let type of ['music', 'book', 'movie', 'drama', 'game']) { 34 | let fullURL = baseURL.replace('{type}', type); 35 | let pageCount = 1; 36 | for (let i = 0; i < pageCount; i ++) { 37 | let response = await (await this.fetch)(fullURL.replace('{start}', i * PAGE_SIZE), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/' + type}}); 38 | if (response.status !== 200) { 39 | throw new TaskError('豆瓣服务器返回错误'); 40 | } 41 | let json = await response.json(); 42 | pageCount = Math.ceil(json.total / PAGE_SIZE); 43 | for (let review of json.reviews) { 44 | let fulltext = await this.fetchReview(review.url); 45 | let row = await this.storage.review.get(parseInt(review.id)); 46 | if (row) { 47 | let lastVersion = row.version; 48 | row.version = version; 49 | if (fulltext !== row.review.fulltext) { 50 | !row.history && (row.history = {}); 51 | row.history[lastVersion] = row.review; 52 | review.fulltext = fulltext; 53 | row.review = review; 54 | } 55 | } else { 56 | review.fulltext = fulltext; 57 | row = { 58 | id: parseInt(review.id), 59 | version: version, 60 | type: type, 61 | review: review, 62 | } 63 | } 64 | await this.storage.review.put(row); 65 | this.step(); 66 | } 67 | } 68 | } 69 | this.complete(); 70 | } 71 | 72 | get name() { 73 | return '评论'; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /extension/tasks/status.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import Task from '../services/Task.js'; 3 | import TaskError from '../services/TaskError.js'; 4 | 5 | 6 | const URL_TIMELINE = 'https://m.douban.com/rexxar/api/v2/status/user_timeline/{uid}?max_id={maxId}&ck={ck}&for_mobile=1'; 7 | const URL_STATUS = 'https://m.douban.com/rexxar/api/v2/status/{id}?ck={ck}&for_mobile=1'; 8 | 9 | 10 | export default class Status extends Task { 11 | async fetchStatusFulltext(id) { 12 | let url = URL_STATUS 13 | .replace('{ck}', this.session.cookies.ck) 14 | .replace('{id}', id); 15 | let fetch = await this.fetch 16 | let response = await fetch(url, {headers: {'X-Override-Referer': 'https://m.douban.com/mine/statuses'}}); 17 | if (response.status !== 200) { 18 | throw new TaskError('豆瓣服务器返回错误'); 19 | } 20 | return await response.json(); 21 | } 22 | 23 | async run() { 24 | let version = this.jobId; 25 | this.total = this.targetUser.statuses_count; 26 | if (this.total === 0) { 27 | return; 28 | } 29 | let lastStatusId = ''; 30 | await this.storage.transaction('rw', this.storage.table('version'), async () => { 31 | let verTable = this.storage.table('version'); 32 | let row = await verTable.get('status'); 33 | if (row) { 34 | lastStatusId = row.lastId; 35 | await verTable.update('status', {version: version, updated: Date.now()}); 36 | } else { 37 | await verTable.add({table: 'status', version: version, updated: Date.now()}); 38 | } 39 | }) 40 | 41 | let baseURL = URL_TIMELINE 42 | .replace('{ck}', this.session.cookies.ck) 43 | .replace('{uid}', this.targetUser.id); 44 | 45 | let count, retried = false; 46 | do { 47 | let fetch = await this.fetch 48 | let response = await fetch(baseURL.replace('{maxId}', lastStatusId), {headers: {'X-Override-Referer': 'https://m.douban.com/mine/statuses'}}); 49 | if (response.status !== 200) { 50 | throw new TaskError('豆瓣服务器返回错误'); 51 | } 52 | let json = await response.json(); 53 | count = json.items.length; 54 | for (let item of json.items) { 55 | let status = item.status; 56 | item.id = parseInt(status.id); 57 | item.created = Date.now(); 58 | lastStatusId = status.id; 59 | if (status.text.length >= 140 && status.text.substr(-3, 3) === '...') { 60 | item.status = await this.fetchStatusFulltext(lastStatusId); 61 | } 62 | try { 63 | await this.storage.status.add(item); 64 | } catch (e) { 65 | if (retried) { 66 | if (e.name === 'ConstraintError') { 67 | this.logger.debug(e.message); 68 | this.complete(); 69 | return; 70 | } 71 | throw e; 72 | } else { 73 | retried = true; 74 | count = 0; 75 | break; 76 | } 77 | } 78 | await this.storage.table('version').update('status', { lastId: item.id }); 79 | this.step(); 80 | } 81 | } while (count > 0 || (lastStatusId = '') === ''); 82 | this.complete(); 83 | } 84 | 85 | get name() { 86 | return '广播'; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /extension/tests/draft.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |「豆坟」这个名字源自于本人去年开发的一款豆瓣账号备份工具（https://github.com/tabris17/doufen）。由于后来豆瓣关闭了API接口，导致该工具无法继续使用。因此，今年我又重新开发一款豆瓣账号备份工具：豆伴（https://blog.doufen.org/posts/tofu-user-guide/）。
想要使用「豆伴」备份豆瓣账号的朋友，请查看《豆伴：豆瓣账号备份工具》一文。本篇不再赘述安装和使用细节。
然而，「豆伴」这种单机版的备份工具有其先天缺陷：备份只能手动进行，无法实时备份；安装繁琐，对普通用户不友好，还有各种兼容性的BUG；必须在PC上使用，手机等移动设备用户对此无能为力。此外，鉴于目前豆瓣的局势，即便备份了自己的用户数据，也面临着无处容身的窘局。
作为一名在豆瓣摸鱼十多年的资深用户，觉得现在是时候做点什么了。
鉴于此，本人有意发起一个项目，仍以「豆坟」为名，以解决部分豆瓣用户的后顾之忧。
「豆坟」项目并非是豆瓣的替代品，而是和豆瓣互为表里，姑且将其称作「里豆瓣」。其主要目的就是作为豆瓣用户的数据保险柜。用户可以将其豆瓣的数据同步保存到豆坟，和豆瓣不同的是，这些数据的所有权完全属于用户本人。用户甚至可以将其豆瓣的友邻（互相关注）关系也同步到豆坟，并和友邻实现互动，且与豆瓣的开放式的关注不同，此类互动也仅限于友邻之间。
你可能经常会在豆瓣碰到类似的情况，比如标注的某本书或某部电影突然消失了，而你标记的信息也随之一起消失了；你的某条广播或日记莫名其妙地被删除了；你的某个友邻突然不说话了……
而豆坟的存在就是为了解决上面的那些问题。豆坟的宗旨是：UGC（用户原创内容）数据来自用户，所有权也应当且仅属于用户本人。
另外，为了规避一些风险，豆坟的数据将无法对外分享，用户也无法查看非友邻关系的用户信息，豆坟也不会提供类似论坛之类的开放交流板块。
因为豆坟项目部署需要采购云服务，预计每年会有数千元（保守估计）的运营成本。本人财力有限，如果你支持这个项目，请不吝打赏。本文已经开通了打赏，请点击下方的打赏按钮。
最后，如果你觉得这个项目还有那么点意思，那请多多转发推广，让跟多的人看到。谢谢！