├── images └── travel.png ├── css ├── content.css └── options.css ├── README.md ├── package.json ├── options.html ├── manifest.json ├── doc └── storageStructure.md └── js ├── cs.js └── background.js /images/travel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JimSunJing/NoBan/HEAD/images/travel.png -------------------------------------------------------------------------------- /css/content.css: -------------------------------------------------------------------------------- 1 | .NoBanBtn { 2 | background-color: black; 3 | color: white; 4 | margin-top: 10px; 5 | border-radius: 15px; 6 | padding: 4px; 7 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # No(tion)(Dou)Ban 2 | 3 | 一个豆瓣跑路（备份） notion 的chrome拓展应用 4 | 可能后期会开发一些新功能比如： 5 | - 小组批量拉黑 6 | - 其他网站内容导入Notion 7 | 8 | ## 说明 9 | 本拓展将会使用 [Notabase](https://github.com/mayneyao/notabase) -------------------------------------------------------------------------------- /css/options.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-color: black; 3 | } 4 | 5 | h1, label { 6 | color: white; 7 | } 8 | 9 | label { 10 | margin: 0 5px; 11 | } 12 | 13 | input { 14 | width: 150px; 15 | } 16 | 17 | .Line { 18 | margin-top: 10px; 19 | } 20 | 21 | .main{ 22 | text-align: center; /*让div内部文字居中*/ 23 | background-color: rgb(36, 38, 138); 24 | border-radius: 20px; 25 | width: 300px; 26 | height: 350px; 27 | margin: auto; 28 | position: absolute; 29 | top: 0; 30 | left: 0; 31 | right: 0; 32 | bottom: 0; 33 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "noban", 3 | "version": "0.0.1", 4 | "description": "a chrome extension for Douban and Notion Communicate", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/JimSunJing/NoBan.git" 12 | }, 13 | "author": "JimSun", 14 | "license": "ISC", 15 | "bugs": { 16 | "url": "https://github.com/JimSunJing/NoBan/issues" 17 | }, 18 | "homepage": "https://github.com/JimSunJing/NoBan#readme" 19 | } 20 | -------------------------------------------------------------------------------- /options.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 |

NoBan

10 |
11 |
12 | 13 | 14 |
15 |
16 | 17 | 18 |
19 |
20 | 21 | 22 |
23 | 24 | -------------------------------------------------------------------------------- /manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version": 2, 3 | "name": "NoBan", 4 | "description": "A Douban to Notion Escape. 一个备份豆瓣到 Notion 的工具", 5 | "version": "0.1", 6 | "permissions": ["storage"], 7 | "options_page": "options.html", 8 | "content_scripts": [ 9 | { 10 | "matches": [ 11 | "*://*.notion.so/*", 12 | "https://*.douban.com/*" 13 | ], 14 | "js": [ 15 | "js/cs.js" 16 | ], 17 | "css": ["css/content.css"], 18 | "run_at": "document_end" 19 | } 20 | ], 21 | "background": { 22 | "scripts": ["js/background.js"] 23 | }, 24 | "icons": { 25 | "16": "images/travel.png", 26 | "32": "images/travel.png", 27 | "48": "images/travel.png", 28 | "128": "images/travel.png" 29 | } 30 | } -------------------------------------------------------------------------------- /doc/storageStructure.md: -------------------------------------------------------------------------------- 1 | ## storage 内的存储结构 2 | 3 | 记一下我这个项目中存储到 `chrome.storage` 中的数据的结构。 4 | 5 | ### 豆瓣书影音条目数据的存储 6 | 7 | - `doubanUser_douId` [Object] 8 | - douId [String: "douban_user_ID"] 9 | - movieCounts 10 | - "done": 123 11 | - "wish": 123 12 | - "ing": 123 13 | - bookCounts 14 | - "done": 123 15 | - "wish": 123 16 | - "ing": 123 17 | - musicCounts 18 | - "done": 123 19 | - "wish": 123 20 | - "ing": 123 21 | - movies 22 | - done: [Array of {subjectid:xxx, marks: rating,comment...}] 23 | - wish: [Array of {subjectid:xxx, marks: rating,comment...}] 24 | - ing: [Array of {subjectid:xxx, marks: rating,comment...}] 25 | - books 26 | - done: [Array of {subjectid:xxx, marks: rating,comment...}] 27 | - wish: [Array of {subjectid:xxx, marks: rating,comment...}] 28 | - ing: [Array of {subjectid:xxx, marks: rating,comment...}] 29 | - musics 30 | - done: [Array of {subjectid:xxx, marks: rating,comment...}] 31 | - wish: [Array of {subjectid:xxx, marks: rating,comment...}] 32 | - ing: [Array of {subjectid:xxx, marks: rating,comment...}] 33 | - `MovieMap` [Map] 34 | - key：subjectId 35 | - val：movieItem 36 | - `BookMap` [Map] 37 | - key：subjectId 38 | - val：bookItem 39 | - `MusicMap` [Map] 40 | - key：subjectId 41 | - val：musicItem -------------------------------------------------------------------------------- /js/cs.js: -------------------------------------------------------------------------------- 1 | console.log("content script running...") 2 | 3 | const doubanBtn = document.createElement('button') 4 | doubanBtn.className = 'NoBanBtn' 5 | doubanBtn.textContent = "备份电影到 NoBan" 6 | 7 | const ItemCount = (counts) => { 8 | return Array.from(counts) 9 | .map((x)=>{ 10 | let i = parseInt(x.innerText.split("部")[0]) 11 | if (x.innerText.indexOf("在") !== -1){ 12 | return {"ing": i} 13 | }else if (x.innerText.indexOf("想") !== -1){ 14 | return {"wish": i} 15 | } 16 | return {"done": i} 17 | }) 18 | .reduce((acc,curr)=>{ 19 | return {...acc, ...curr} 20 | }) 21 | } 22 | 23 | doubanBtn.onclick = () => { 24 | console.log("可以开始爬了") 25 | // 获取豆瓣用户的 ID 26 | const DouId = document.querySelector(".user-info .pl") 27 | .innerHTML.split("
")[0].trim() 28 | console.log("douban id:",DouId) 29 | // 获取条目计数 30 | // 电影 31 | const movieCounts = ItemCount(document.querySelectorAll("#movie h2 span a")) 32 | // 图书 33 | const bookCounts = ItemCount(document.querySelectorAll("#book h2 span a")) 34 | // 音乐 35 | const musicCounts = ItemCount(document.querySelectorAll("#music h2 span a")) 36 | // 接下来应该把 id 发给 background.js 37 | chrome.runtime.sendMessage({ 38 | douId: DouId, 39 | movieCounts: movieCounts, 40 | bookCounts: bookCounts, 41 | musicCounts: musicCounts 42 | }, (response) => {console.log("response:",response)}) 43 | } 44 | 45 | window.onload = () => { 46 | let userInfo = document.querySelector(".user-info") 47 | // 首先自动获取用户id 48 | if (userInfo !== null){ 49 | userInfo.appendChild(doubanBtn) 50 | } 51 | } -------------------------------------------------------------------------------- /js/background.js: -------------------------------------------------------------------------------- 1 | console.log("background.js is running...") 2 | 3 | const UserKey = (douId) => { 4 | return "douban_user_" + douId 5 | } 6 | 7 | chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { 8 | console.log(message) 9 | if (sender.url.indexOf("www.douban.com/people") !== -1) { 10 | window.userInfo = message 11 | // 下面应该开始爬虫 12 | const Ukey = UserKey(message.douId) 13 | chrome.storage.local.set({ [Ukey]: message }, () => { 14 | console.log("user info saved", message) 15 | }) 16 | movieMainControl() 17 | } 18 | sendResponse("background got the message..") 19 | }) 20 | 21 | // 爬虫部分 22 | // 主控制程序，翻页之类的 23 | const movieMainControl = () => { 24 | // 电影看过爬取 25 | let page = 0 26 | const url = "https://movie.douban.com/people/" + userInfo.douId + 27 | "/collect?sort=time&start=" + page * 30 + 28 | "&filter=all&mode=list&tags_sort=count" 29 | if (!saveMoviePage(url)) { 30 | // 已有重复爬取条目，可退出 31 | return false 32 | } 33 | } 34 | 35 | // 异步请求页面 36 | const fetchText = async (url) => { 37 | let response = await fetch(url) 38 | return await response.text() 39 | } 40 | 41 | // 获取storage内的movie id 42 | const getUserMovies = (douId, type) => { 43 | chrome.storage.local.get(UserKey(douId), (data) => { 44 | // data["user_xxx"]["movies"] 应该是 {"done":Array,...} 45 | return data.movies[type] 46 | }) 47 | } 48 | 49 | const setUserMovies = async (douId, type, userMovies) => { 50 | const Ukey = UserKey(douId) 51 | chrome.storage.local.get([Ukey], (data)=>{ 52 | let newUser = { 53 | ...data, 54 | movies: { 55 | ...data.movies, 56 | [type]: userMovies 57 | } 58 | } 59 | chrome.storage.set({[Ukey]: newUser}, (newUser)=>{ 60 | console.log("user movies updated,",newUser) 61 | }) 62 | }) 63 | } 64 | 65 | const getStorage = (key) => { 66 | let res 67 | chrome.storage.local.get(key,(data) => { 68 | // 如果没有会返回 空 Object 69 | res = data 70 | }) 71 | return res 72 | } 73 | 74 | const setMovieMap = (map) => { 75 | chrome.storage.local.set({"movieMap": map},(map)=>{ 76 | console.log("movieMap stored...",map) 77 | }) 78 | } 79 | 80 | const getType = (url) => { 81 | if (url.indexOf("collect") !== -1) 82 | return "done" 83 | else if (url.indexOf("wish") !== -1) 84 | return "wish" 85 | return "ing" 86 | } 87 | 88 | // 获取dom 89 | const saveMoviePage = (url) => { 90 | fetchText(url) 91 | .then((text) => { 92 | // 爬取类别 93 | const Type = getType(url) 94 | // 标记是否已经爬到历史记录 95 | let finished = false 96 | // console.log(text) 97 | let dom = new DOMParser().parseFromString(text, "text/html"); 98 | // 获取 items 元素 99 | let items = Array.from(dom.querySelectorAll(".item")) 100 | // 先获取所有已经保存好的电影条目ID，避免多次爬取 101 | chrome.storage.local.get("movieMap",(mmap) => { 102 | console.log("mmap:", mmap) 103 | if (Object.keys(mmap).length === 0 && mmap.constructor === Object){ 104 | window.movieMap = new Map() 105 | }else{ 106 | window.movieMap = mmap 107 | } 108 | console.log("movieMap:", movieMap) 109 | const movieIds = Array.from(movieMap.keys()) 110 | chrome.storage.local.get(UserKey(userInfo.douId), (data) => { 111 | // data["user_xxx"]["movies"] 应该是 {"done":Array,...} 112 | let userMovies = data.hasOwnProperty("movies")? data.movies[Type] : newDoneWishIng() 113 | console.log("userMovies:",userMovies) 114 | userMovies = Array.from(userMovies).map((data)=>{return data.subjectId}) 115 | movieItemHelper(items, movieIds, userMovies, movieMap, userInfo.douId, Type) 116 | }) 117 | }) 118 | }).catch(e => { 119 | console.log("error when fetching:", url, "info:", e); 120 | }) 121 | } 122 | 123 | const newDoneWishIng = () => {return {"done": [], "wish": [], "ing": []}} 124 | 125 | const movieItemHelper = async (items, movieIds, userMovies, movieMap, douId, Type) => { 126 | // 对每个item进行处理 127 | items.map((item) => { 128 | const subjectId = item.querySelector("a").href.split("/")[4] 129 | // 判断是否之前存入用户的 movies 了 130 | if (userMovies.indexOf(subjectId) !== -1){ 131 | let marks = { 132 | "用户评分": item.querySelector(".date span").className.charAt(6), 133 | "标记时间": item.querySelector(".date").innerText.trim(), 134 | "短评": item.querySelector(".comment") !== null ? 135 | item.querySelector(".comment").innerText.trim() : "..." 136 | } 137 | userMovies.push({ 138 | "subjectId":movie.subjectId, 139 | "marks": movie.marks 140 | }) 141 | console.log("UserMovies updated:",userMovies) 142 | setUserMovies(douId,Type,userMovies) 143 | } 144 | }) 145 | } 146 | 147 | // 从items获取信息，存入chrome.storage 148 | const getMovieItem = async (item, movieIds, userMovies) => { 149 | const subjectId = item.querySelector("a").href.split("/")[4] 150 | // 判断是否之前存入用户的 movies 了 151 | if (userMovies.indexOf(subjectId) !== -1) return false 152 | 153 | // 判断是否已经爬取过详细信息了 154 | if (movieIds.indexOf(subjectId) !== -1){ 155 | return { 156 | "subjectId": subjectId, 157 | "item": null 158 | } 159 | } 160 | 161 | let movieItem = { 162 | "电影名": item.querySelector(".title a").innerText.trim(), 163 | } 164 | console.log("simple item:",movieItem) 165 | // 详细信息收集 166 | let details = movieDetail(subjectId) 167 | movieItem = { 168 | ...movieItem, 169 | ...details 170 | } 171 | console.log("movieItem in [getMovieItem]",movieItem) 172 | // 返回一个movie对象 173 | return { 174 | "subjectId": subjectId, 175 | "item": movieItem 176 | } 177 | } 178 | 179 | // 进入subject页面进行信息收集，存入 chrome.storage 180 | const movieDetail = (movieSubjectId) => { 181 | // 拼接url 182 | const url = 'https://movie.douban.com/subject/' + movieSubjectId 183 | fetchText(url) 184 | .then((text) => { 185 | const dom = new DOMParser().parseFromString(text, "text/html"); 186 | let details = { 187 | "年份": dom.querySelector(".year").innerText.replace("(","").replace(")",""), 188 | "封面": dom.querySelector("#mainpic img").src.replace("webp","jpg") 189 | } 190 | const infos = Array.from(dom.querySelector("#info").innerText.split(/\r?\n/)) 191 | // console.log("infos:",infos) 192 | details = infos.reduce((acc, curr)=>{ 193 | const k = curr.split(": ")[0] 194 | const v = curr.split(": ")[1] 195 | return { 196 | ...acc, 197 | ...{[k]: v} 198 | } 199 | } 200 | ,details) 201 | console.log("movie item detail:",details) 202 | // 直接返回detail Object 203 | return details 204 | }).catch(e => { 205 | console.log("error when fetching:", url, "info:", e); 206 | return null 207 | }) 208 | } 209 | 210 | // 检查条目的更新，将条目调入最新状态的数组 211 | 212 | // 213 | 214 | 215 | --------------------------------------------------------------------------------