Vite App

├── v2ex_scrapy ├── __init__.py ├── spiders │ ├── __init__.py │ ├── V2exSpider.py │ ├── V2exMemberSpider.py │ ├── V2exNodeTopicSpider.py │ └── CommonSpider.py ├── utils.py ├── insert_ignore.py ├── pipelines.py ├── DB.py ├── items.py ├── settings.py ├── v2ex_parser.py └── middlewares.py ├── requirements-analysis.txt ├── analysis ├── v2ex-analysis │ ├── env.d.ts │ ├── src │ │ ├── assets │ │ │ ├── base.css │ │ │ ├── logo.svg │ │ │ └── main.css │ │ ├── main.ts │ │ ├── components │ │ │ ├── base │ │ │ │ ├── Tag.vue │ │ │ │ ├── Node.vue │ │ │ │ ├── Member.vue │ │ │ │ ├── Topic.vue │ │ │ │ └── Comment.vue │ │ │ ├── TopTag.vue │ │ │ ├── TopComment.vue │ │ │ ├── TopMember.vue │ │ │ └── TopTopic.vue │ │ ├── types │ │ │ └── F.ts │ │ └── App.vue │ ├── public │ │ ├── favicon.ico │ │ ├── tag-usage-count.json │ │ ├── top-user-by-topic_count.json │ │ ├── top-user-by-comment_count.json │ │ ├── top-topic-by-clicks.json │ │ ├── top-topic-by-votes.json │ │ ├── top-topic-by-thank_count.json │ │ ├── top-topic-by-favorite_count.json │ │ ├── new-topic-every-month.json │ │ ├── new-comment-every-month.json │ │ ├── new-member-every-month.json │ │ └── top-comment.json │ ├── postcss.config.js │ ├── tsconfig.json │ ├── tailwind.config.js │ ├── tsconfig.node.json │ ├── vite.config.ts │ ├── index.html │ ├── tsconfig.app.json │ ├── .gitignore │ ├── package.json │ ├── README.md │ └── pnpm-lock.yaml └── main.py ├── requirements.txt ├── image └── t │ ├── 水深火热-每月新增帖子.png │ ├── 水深火热-每月新增评论.png │ ├── 1688469358680.png │ ├── 1688470374959.png │ └── 1688470738877.png ├── main.py ├── scrapy.cfg ├── .vscode └── launch.json ├── user-agents.txt ├── LICENSE ├── .github └── workflows │ └── gh-pages.yml ├── query.sql ├── .gitignore ├── README.md └── README-en.md /v2ex_scrapy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements-analysis.txt: -------------------------------------------------------------------------------- 1 | pandas==2.0.1 2 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | arrow==1.2.3 2 | httpx==0.24.1 3 | Scrapy==2.9.0 4 | SQLAlchemy==2.0.17 5 | -------------------------------------------------------------------------------- /image/t/水深火热-每月新增帖子.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldshensheep/v2ex_scrapy/HEAD/image/t/水深火热-每月新增帖子.png -------------------------------------------------------------------------------- /image/t/水深火热-每月新增评论.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldshensheep/v2ex_scrapy/HEAD/image/t/水深火热-每月新增评论.png -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/assets/base.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /image/t/1688469358680.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldshensheep/v2ex_scrapy/HEAD/image/t/1688469358680.png -------------------------------------------------------------------------------- /image/t/1688470374959.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldshensheep/v2ex_scrapy/HEAD/image/t/1688470374959.png -------------------------------------------------------------------------------- /image/t/1688470738877.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldshensheep/v2ex_scrapy/HEAD/image/t/1688470738877.png -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldshensheep/v2ex_scrapy/HEAD/analysis/v2ex-analysis/public/favicon.ico -------------------------------------------------------------------------------- /analysis/v2ex-analysis/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/main.ts: -------------------------------------------------------------------------------- 1 | import './assets/main.css' 2 | 3 | import { createApp } from 'vue' 4 | import App from './App.vue' 5 | 6 | createApp(App).mount('#app') 7 | -------------------------------------------------------------------------------- /v2ex_scrapy/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from scrapy.crawler import CrawlerProcess 2 | from scrapy.utils.project import get_project_settings 3 | 4 | process = CrawlerProcess(get_project_settings()) 5 | 6 | process.crawl("v2ex") 7 | process.start() 8 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { 5 | "path": "./tsconfig.node.json" 6 | }, 7 | { 8 | "path": "./tsconfig.app.json" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/base/Tag.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 11 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/base/Node.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/base/Member.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 11 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: [ 4 | "./index.html", 5 | "./src/**/*.{vue,js,ts,jsx,tsx}", 6 | ], 7 | theme: { 8 | extend: {}, 9 | }, 10 | plugins: [require("daisyui")], 11 | 12 | } 13 | 14 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/assets/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = v2ex_scrapy.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = v2ex_scrapy 12 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/base/Topic.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 15 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node18/tsconfig.json", 3 | "include": [ 4 | "vite.config.*", 5 | "vitest.config.*", 6 | "cypress.config.*", 7 | "nightwatch.conf.*", 8 | "playwright.config.*" 9 | ], 10 | "compilerOptions": { 11 | "composite": true, 12 | "module": "ESNext", 13 | "types": ["node"] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { fileURLToPath, URL } from 'node:url' 2 | 3 | import { defineConfig } from 'vite' 4 | import vue from '@vitejs/plugin-vue' 5 | 6 | // https://vitejs.dev/config/ 7 | export default defineConfig({ 8 | plugins: [ 9 | vue(), 10 | ], 11 | resolve: { 12 | alias: { 13 | '@': fileURLToPath(new URL('./src', import.meta.url)) 14 | } 15 | } 16 | }) 17 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Vite App 8 | 9 | 10 |

	Tag	使用次数
{{ i + 1 }}	32 \| 33 \|	{{ d.count }}

	评论	主题标题	感谢数
{{ i + 1 }}	33 \| 34 \|	36 \| 37 \|	{{ d.thank_count }}

	用户名	{{ a[selected] }}
{{ i + 1 }}	60 \| 61 \|	{{ d[selected] }}

	主题标题	{{ a[selected] }}
{{ i + 1 }}	68 \| 69 \|	{{ d[selected] }}

11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "configurations": [ 4 | { 5 | "name": "Python: Launch Scrapy Spider", 6 | "type": "python", 7 | "request": "launch", 8 | "module": "scrapy", 9 | "args": [ 10 | "runspider", 11 | "${file}" 12 | ], 13 | "console": "integratedTerminal" 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /analysis/v2ex-analysis/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@vue/tsconfig/tsconfig.dom.json", 3 | "include": [ 4 | "env.d.ts", 5 | "src/**/*", 6 | "src/**/*.vue", 7 | "src/**/*.ts" 8 | ], 9 | "exclude": [ 10 | "src/**/__tests__/*" 11 | ], 12 | "compilerOptions": { 13 | "composite": true, 14 | "baseUrl": ".", 15 | "paths": { 16 | "@/*": [ 17 | "./src/*" 18 | ] 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /analysis/v2ex-analysis/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | .DS_Store 12 | dist 13 | dist-ssr 14 | coverage 15 | *.local 16 | 17 | /cypress/videos/ 18 | /cypress/screenshots/ 19 | 20 | # Editor directories and files 21 | .vscode/* 22 | !.vscode/extensions.json 23 | .idea 24 | *.suo 25 | *.ntvs* 26 | *.njsproj 27 | *.sln 28 | *.sw? 29 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/tag-usage-count.json: -------------------------------------------------------------------------------- 1 | [{"tag":"程序员","count":225},{"tag":"华为","count":201},{"tag":"疫情","count":189},{"tag":"GitHub","count":172},{"tag":"微信","count":158},{"tag":"知乎","count":154},{"tag":"V2EX","count":154},{"tag":"App","count":146},{"tag":"核酸","count":119},{"tag":"帖子","count":119},{"tag":"Google","count":111},{"tag":"百度","count":110},{"tag":"icu","count":105},{"tag":"疫苗","count":98},{"tag":"开源","count":97},{"tag":"房价","count":95},{"tag":"域名","count":87},{"tag":"网站","count":81},{"tag":"代码","count":74},{"tag":"讨论","count":72}] -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/base/Comment.vue: -------------------------------------------------------------------------------- 1 | 13 | 14 | 20 | -------------------------------------------------------------------------------- /user-agents.txt: -------------------------------------------------------------------------------- 1 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 2 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9 3 | Mozilla/5.0 (Windows NT 10.0; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36 OPR/89.0.4447.51 4 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 5 | Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0 -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/assets/main.css: -------------------------------------------------------------------------------- 1 | @import './base.css'; 2 | 3 | #app { 4 | max-width: 1280px; 5 | margin: 0 auto; 6 | padding: 2rem; 7 | 8 | font-weight: normal; 9 | } 10 | 11 | a, 12 | .green { 13 | text-decoration: none; 14 | color: hsla(160, 100%, 37%, 1); 15 | transition: 0.4s; 16 | } 17 | 18 | @media (hover: hover) { 19 | a:hover { 20 | background-color: hsla(160, 100%, 37%, 0.2); 21 | } 22 | } 23 | /* 24 | @media (min-width: 1024px) { 25 | body { 26 | display: flex; 27 | place-items: center; 28 | } 29 | 30 | #app { 31 | display: grid; 32 | grid-template-columns: 1fr 1fr; 33 | padding: 0 2rem; 34 | } 35 | } */ 36 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-user-by-topic_count.json: -------------------------------------------------------------------------------- 1 | [{"username":"vitovan","topic_count":31},{"username":"est","topic_count":25},{"username":"taobibi","topic_count":20},{"username":"xuanwu","topic_count":15},{"username":"youmee","topic_count":14},{"username":"permaylau","topic_count":14},{"username":"weiruanniubi","topic_count":13},{"username":"ech0x","topic_count":13},{"username":"agagega","topic_count":13},{"username":"li24361","topic_count":12},{"username":"labulaka521","topic_count":12},{"username":"kisshere","topic_count":12},{"username":"whwq2012","topic_count":11},{"username":"statement","topic_count":11},{"username":"razios","topic_count":11},{"username":"flowfire","topic_count":11},{"username":"find456789","topic_count":11},{"username":"bclerdx","topic_count":11},{"username":"also24","topic_count":11},{"username":"CEBBCAT","topic_count":11}] -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/types/F.ts: -------------------------------------------------------------------------------- 1 | export interface TopComment { 2 | topic_id: number; 3 | id: number; 4 | content: string; 5 | thank_count: number; 6 | no: number; 7 | title: string; 8 | } 9 | 10 | export enum TopicBy { 11 | clicks = "clicks", 12 | favorite_count = "favorite_count", 13 | thank_count = "thank_count", 14 | votes = "votes", 15 | } 16 | export enum UserBy { 17 | comment_count = "comment_count", 18 | topic_count = "topic_count", 19 | } 20 | 21 | export interface TopTopic { 22 | id: number; 23 | title: string; 24 | clicks?: number; 25 | favorite_count?: number; 26 | thank_count?: number; 27 | votes?: number; 28 | } 29 | 30 | export interface TopMember { 31 | username: string; 32 | comment_count?: number; 33 | topic_count?: number; 34 | } 35 | 36 | export interface TopTag { 37 | tag: string; 38 | count?: number; 39 | } 40 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-user-by-comment_count.json: -------------------------------------------------------------------------------- 1 | [{"username":"murmur","comment_count":1276},{"username":"glfpes","comment_count":1093},{"username":"reus","comment_count":1055},{"username":"sagaxu","comment_count":1022},{"username":"cmdOptionKana","comment_count":876},{"username":"hoyixi","comment_count":843},{"username":"learningman","comment_count":804},{"username":"Jooooooooo","comment_count":787},{"username":"juded","comment_count":783},{"username":"imn1","comment_count":737},{"username":"x86","comment_count":734},{"username":"optional","comment_count":734},{"username":"TypeError","comment_count":706},{"username":"Livid","comment_count":703},{"username":"yaphets666","comment_count":670},{"username":"ericwoflskin","comment_count":668},{"username":"Leonard","comment_count":662},{"username":"coderluan","comment_count":653},{"username":"est","comment_count":604},{"username":"eason1874","comment_count":552}] -------------------------------------------------------------------------------- /analysis/v2ex-analysis/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "v2ex-analysis", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "vite", 7 | "build": "run-p type-check build-only", 8 | "preview": "vite preview", 9 | "build-only": "vite build", 10 | "type-check": "vue-tsc --noEmit -p tsconfig.app.json --composite false" 11 | }, 12 | "dependencies": { 13 | "plotly.js-dist-min": "^2.24.3", 14 | "vue": "^3.3.4" 15 | }, 16 | "devDependencies": { 17 | "@tsconfig/node18": "^2.0.1", 18 | "@types/node": "^18.16.17", 19 | "@vitejs/plugin-vue": "^4.2.3", 20 | "@vue/tsconfig": "^0.4.0", 21 | "autoprefixer": "^10.4.14", 22 | "daisyui": "^3.2.1", 23 | "npm-run-all": "^4.1.5", 24 | "postcss": "^8.4.25", 25 | "tailwindcss": "^3.3.2", 26 | "typescript": "~5.0.4", 27 | "vite": "^4.3.9", 28 | "vue-tsc": "^1.6.5" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/TopTag.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 40 | -------------------------------------------------------------------------------- /v2ex_scrapy/utils.py: -------------------------------------------------------------------------------- 1 | from http.cookies import SimpleCookie 2 | import json 3 | from typing import Union 4 | 5 | import arrow 6 | 7 | 8 | def time_to_timestamp(t: str) -> int: 9 | t = t.strip() 10 | a = None 11 | try: 12 | if t[-2:] == "00": 13 | a = arrow.get(t, "YYYY-MM-DD HH:mm:ss ZZ") 14 | else: 15 | a = arrow.utcnow().dehumanize(t.replace(" ", ""), "zh") 16 | return int(a.to("+08:00").timestamp()) 17 | except Exception: 18 | return 0 19 | 20 | 21 | def none_or_strip(s: Union[str, None]) -> Union[str, None]: 22 | if s is not None: 23 | return s.strip() 24 | return s 25 | 26 | 27 | def json_to_str(j): 28 | return json.dumps(j, ensure_ascii=False) 29 | 30 | 31 | def cookie_str2cookie_dict(cookie_str: str): 32 | simple_cookie = SimpleCookie() 33 | simple_cookie.load(cookie_str) 34 | return {k: v.value for k, v in simple_cookie.items()} 35 | 36 | 37 | if __name__ == "__main__": 38 | a = ["2022-04-28 13:24:38 +08:00", "287 天前", "1 小时前"] 39 | 40 | for i in a: 41 | print(time_to_timestamp(i)) 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 oldshensheep 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-topic-by-clicks.json: -------------------------------------------------------------------------------- 1 | [{"id":557538,"title":"Openbilibili？ B 站在 Github 上公开了自己的后端源代码。","clicks":81613},{"id":388344,"title":"一兆韦德健身房无耻套路(被坑经历)","clicks":55447},{"id":603138,"title":"Vultr 的 ip 是不是被重点关照了？","clicks":54856},{"id":130791,"title":"有没有办法抓到使用国外服务器办色情站的人员？","clicks":46713},{"id":937283,"title":"上海 7 年前端求捞","clicks":43220},{"id":299142,"title":"我想知道在中国如何邮寄一把菜刀","clicks":42597},{"id":315502,"title":"python 刷 V2EX 活跃度","clicks":40046},{"id":865906,"title":"感觉要出大事情了？","clicks":38956},{"id":329592,"title":"防盗版软件黑科技又一案例","clicks":35476},{"id":887715,"title":"国庆刚办完婚礼，现在感觉已经过不下去了","clicks":35418},{"id":670151,"title":"如何看待一个 MC 启动器作者被逼向他软件的破解者道歉？","clicks":34749},{"id":898448,"title":"疫情管控，层层加码，一刀切","clicks":33741},{"id":728186,"title":"8 亿 QQ 手机绑定数据泄露？请问作为用户发现自己手机号被泄露的情况下应该如何做？","clicks":32254},{"id":731552,"title":"有在工作后移民的吗？","clicks":31484},{"id":400557,"title":"北京电信启用了端口白名单","clicks":31299},{"id":554157,"title":"华为开源了个方舟编译器","clicks":30421},{"id":582042,"title":"我为什么说大疆是国产垃圾品牌中的战斗机","clicks":30074},{"id":416067,"title":"Steam 主动限制了中国地区的社区功能","clicks":30029},{"id":604675,"title":"阿里 IDC 真是牛逼。昨天的交换机事件。","clicks":29323},{"id":784789,"title":"原来 Linux 内核贡献第二是这么来的","clicks":29202}] -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-topic-by-votes.json: -------------------------------------------------------------------------------- 1 | [{"id":473163,"title":"我花了 14 个小时找了一下长春长生们究竟卖到了哪里去","votes":358},{"id":757489,"title":"猴年马月狗日，王伟疯了","votes":67},{"id":759427,"title":"王妈说：“伟宝，你去哪啊？”","votes":65},{"id":670151,"title":"如何看待一个 MC 启动器作者被逼向他软件的破解者道歉？","votes":58},{"id":482085,"title":"[帮转] 大家千万别租桃园地铁站 C 出口南景苑大厦 7M 的房子，那房子房东专门骗钱的！我跟多位租客已受害。。","votes":56},{"id":758647,"title":"王伟站在栅栏前，对面有条流浪狗","votes":49},{"id":606364,"title":"龙芯 & Golang！","votes":48},{"id":756365,"title":"小王，你这个方案还是不行啊","votes":43},{"id":548359,"title":"为何蔡徐坤每条微博转发量 100 万+？用大数据扒一扒他的真假流量粉","votes":39},{"id":784789,"title":"原来 Linux 内核贡献第二是这么来的","votes":39},{"id":624860,"title":"不再购买或推荐某为的任何消费级产品","votes":38},{"id":185123,"title":"我尊重@Livid 的抉择，但我不想让@Livid 以外的人支配我在 V2 的数据，能不能提供一个删档走人的选项？","votes":37},{"id":758236,"title":"王伟今年已经换了三份工作","votes":37},{"id":604675,"title":"阿里 IDC 真是牛逼。昨天的交换机事件。","votes":34},{"id":757833,"title":"王伟凌晨加班回来，感到一阵便意","votes":34},{"id":474773,"title":"我又花了 28 个小时分析了一下各省二类疫苗采购公示数据","votes":33},{"id":595854,"title":"请给出 V2EX 的封禁政策","votes":31},{"id":849140,"title":"给工信部写信了，要求开放苹果的 facetime audio 和 CallKit 功能。","votes":31},{"id":874223,"title":"中文互联网已死","votes":31},{"id":111204,"title":"声讨@Bob 的倒卖（诈骗）行为","votes":30}] -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-topic-by-thank_count.json: -------------------------------------------------------------------------------- 1 | [{"id":473163,"title":"我花了 14 个小时找了一下长春长生们究竟卖到了哪里去","thank_count":531},{"id":759427,"title":"王妈说：“伟宝，你去哪啊？”","thank_count":132},{"id":474773,"title":"我又花了 28 个小时分析了一下各省二类疫苗采购公示数据","thank_count":115},{"id":606364,"title":"龙芯 & Golang！","thank_count":71},{"id":757489,"title":"猴年马月狗日，王伟疯了","thank_count":65},{"id":758647,"title":"王伟站在栅栏前，对面有条流浪狗","thank_count":65},{"id":859497,"title":"正经脱单教学帖丨关于唐山、上外事件有感。","thank_count":57},{"id":756365,"title":"小王，你这个方案还是不行啊","thank_count":56},{"id":473916,"title":"疫苗批号反查生产企业","thank_count":45},{"id":755305,"title":"早晨，王伟被老婆的尖叫声惊醒","thank_count":39},{"id":757833,"title":"王伟凌晨加班回来，感到一阵便意","thank_count":38},{"id":640127,"title":"17 例新型冠状病毒感染的肺炎死亡病例病情简单分析","thank_count":33},{"id":905479,"title":"网易 2022 年度盘点：致敬每一个扛住了生活的平凡人","thank_count":33},{"id":935642,"title":"欧洲错过了互联网？","thank_count":33},{"id":849140,"title":"给工信部写信了，要求开放苹果的 facetime audio 和 CallKit 功能。","thank_count":32},{"id":950392,"title":"成都经济的支柱","thank_count":32},{"id":953346,"title":"补充补充一些不存在的记忆（政治不正确版）","thank_count":30},{"id":758236,"title":"王伟今年已经换了三份工作","thank_count":29},{"id":760530,"title":"王伟说：“老婆，我也爱你。”","thank_count":28},{"id":754936,"title":"早上睁开眼，王伟说，不想活了","thank_count":26}] -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/TopComment.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 44 | -------------------------------------------------------------------------------- /v2ex_scrapy/insert_ignore.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.ext.compiler import compiles 2 | from sqlalchemy.sql import Insert 3 | 4 | # copy and modified from https://github.com/sqlalchemy/sqlalchemy/issues/5374#issuecomment-752693165 5 | 6 | """ 7 | When imported, automatically make all insert not fail on duplicate keys 8 | """ 9 | 10 | 11 | # modified 12 | @compiles(Insert, "sqlite") 13 | def sqlite_insert_ignore(insert, compiler, **kw): 14 | statement = compiler.visit_insert(insert, **kw) 15 | # len("INSERT") == 6 16 | return f'{statement[:6]}{ " OR IGNORE "}{statement[6:]}' 17 | 18 | 19 | @compiles(Insert, "mysql") 20 | def mysql_insert_ignore(insert, compiler, **kw): 21 | return compiler.visit_insert(insert.prefix_with("IGNORE"), **kw) 22 | 23 | 24 | @compiles(Insert, "postgresql") 25 | def postgresql_on_conflict_do_nothing(insert, compiler, **kw): 26 | statement = compiler.visit_insert(insert, **kw) 27 | # IF we have a "RETURNING" clause, we must insert before it 28 | returning_position = statement.find("RETURNING") 29 | if returning_position >= 0: 30 | return ( 31 | statement[:returning_position] 32 | + "ON CONFLICT DO NOTHING " 33 | + statement[returning_position:] 34 | ) 35 | else: 36 | return statement + " ON CONFLICT DO NOTHING" 37 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-topic-by-favorite_count.json: -------------------------------------------------------------------------------- 1 | [{"id":473163,"title":"我花了 14 个小时找了一下长春长生们究竟卖到了哪里去","favorite_count":588},{"id":329592,"title":"防盗版软件黑科技又一案例","favorite_count":337},{"id":875975,"title":"ShadowTLS：更好用的 TLS 伪装代理","favorite_count":273},{"id":731552,"title":"有在工作后移民的吗？","favorite_count":255},{"id":728186,"title":"8 亿 QQ 手机绑定数据泄露？请问作为用户发现自己手机号被泄露的情况下应该如何做？","favorite_count":221},{"id":608625,"title":"Everything 使用不当会导致各种重要数据泄露公网，可能就有你的个人信息","favorite_count":220},{"id":588732,"title":"阿里 P6 肉翻到新加坡 Shopee 经验分享（可内推）","favorite_count":206},{"id":892528,"title":"全世界最大的电子图书馆被 ban 了~","favorite_count":205},{"id":557538,"title":"Openbilibili？ B 站在 Github 上公开了自己的后端源代码。","favorite_count":194},{"id":865906,"title":"感觉要出大事情了？","favorite_count":170},{"id":887715,"title":"国庆刚办完婚礼，现在感觉已经过不下去了","favorite_count":170},{"id":881086,"title":"如果想对经济、局势有一些自己的认识，应该阅读哪些书籍。","favorite_count":159},{"id":759570,"title":"分享一下近一年的欧盟 WeChat 账号与微信账号的使用与功能区别","favorite_count":146},{"id":596113,"title":"女友被打，派出所被迫和解，已投诉市长热线，求助，还应该做什么","favorite_count":142},{"id":271547,"title":"Vim 神级配置,IDE 弱爆了","favorite_count":134},{"id":518047,"title":"都说『明年的经济形势会更差』,你是如何『见微知著，以小见大』的？","favorite_count":125},{"id":611612,"title":"各位有看到比较好的，阐释西方价值观（或者民主、自由）本质的书籍吗？","favorite_count":120},{"id":859497,"title":"正经脱单教学帖丨关于唐山、上外事件有感。","favorite_count":120},{"id":946530,"title":"B 站 70 万粉丝 up 主秋葉 aaaki，月入大几万，互联网他是真玩明白了","favorite_count":119},{"id":521094,"title":"还是房产相关，多地已经触底","favorite_count":117}] -------------------------------------------------------------------------------- /v2ex_scrapy/spiders/V2exSpider.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | import scrapy.http.response.html 3 | 4 | from v2ex_scrapy.DB import DB 5 | from v2ex_scrapy.items import TopicItem 6 | from v2ex_scrapy.spiders.CommonSpider import CommonSpider 7 | 8 | 9 | class V2exSpider(scrapy.Spider): 10 | name = "v2ex" 11 | FORCE_UPDATE_TOPIC = False 12 | UPDATE_COMMENT = True 13 | 14 | def __init__(self, *args, **kwargs): 15 | super().__init__(*args, **kwargs) 16 | self.db = DB() 17 | self.start_id = 1 18 | self.end_id = 1000000 19 | self.common_spider = CommonSpider( 20 | self.logger, update_comment=self.UPDATE_COMMENT 21 | ) 22 | self.logger.info(f"start from topic id {self.start_id}, end at {self.end_id}") 23 | 24 | def start_requests(self): 25 | for i in range(self.start_id + 1, self.end_id + 1): 26 | if ( 27 | self.FORCE_UPDATE_TOPIC 28 | or (not self.db.exist(TopicItem, i)) 29 | or ( 30 | self.db.get_topic_comment_count(i) 31 | > self.db.get_comment_count_by_topic(i) 32 | ) 33 | ): 34 | yield scrapy.Request( 35 | url=f"https://www.v2ex.com/t/{i}", 36 | callback=self.common_spider.parse_topic, 37 | errback=self.common_spider.parse_topic_err, 38 | cb_kwargs={"topic_id": i}, 39 | ) 40 | else: 41 | self.logger.info(f"skip topic {i}") 42 | -------------------------------------------------------------------------------- /v2ex_scrapy/spiders/V2exMemberSpider.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | import scrapy.http.response.html 3 | from scrapy.spidermiddlewares.httperror import HttpError 4 | 5 | from v2ex_scrapy import v2ex_parser 6 | from v2ex_scrapy.DB import DB 7 | from v2ex_scrapy.items import MemberItem 8 | 9 | 10 | class V2exTopicSpider(scrapy.Spider): 11 | name = "v2ex-member" 12 | 13 | def __init__(self, start_id=1, end_id=635000, *args, **kwargs): 14 | super().__init__(*args, **kwargs) 15 | self.db = DB() 16 | self.start_id = start_id 17 | self.end_id = end_id 18 | self.logger.info(f"start from topic id {self.start_id}, end at {self.end_id}") 19 | 20 | def start_requests(self): 21 | for i in range(self.start_id, self.end_id + 1): 22 | if not self.db.exist(MemberItem, i): 23 | yield scrapy.Request( 24 | url=f"https://www.v2ex.com/uid/{i}", 25 | callback=self.parse, 26 | errback=self.member_err, 27 | cb_kwargs={"uid": i}, 28 | ) 29 | else: 30 | self.logger.info(f"skip member id:{i}, because it exists") 31 | 32 | def parse(self, response: scrapy.http.response.html.HtmlResponse, uid: int): 33 | for i in v2ex_parser.parse_member(response): 34 | i.uid = uid 35 | yield i 36 | 37 | def member_err(self, failure): 38 | if failure.check(HttpError): 39 | yield MemberItem( 40 | username="", 41 | avatar_url="", 42 | create_at=0, 43 | social_link=[], 44 | uid=failure.request.cb_kwargs["uid"], 45 | ) 46 | -------------------------------------------------------------------------------- /.github/workflows/gh-pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy GitHub Pages 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | # push: 7 | # branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Build job 26 | build: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v3 31 | - name: Setup Pages 32 | uses: actions/configure-pages@v3 33 | - uses: pnpm/action-setup@v2 34 | with: 35 | version: 8 36 | - name: Build 37 | run: | 38 | cd analysis/v2ex-analysis 39 | pnpm install 40 | pnpm vite build --base=/v2ex_scrapy 41 | - name: Upload artifact 42 | uses: actions/upload-pages-artifact@v1 43 | with: 44 | path: "analysis/v2ex-analysis/dist" 45 | # Deployment job 46 | deploy: 47 | environment: 48 | name: github-pages 49 | url: ${{ steps.deployment.outputs.page_url }} 50 | runs-on: ubuntu-latest 51 | needs: build 52 | steps: 53 | - name: Deploy to GitHub Pages 54 | id: deployment 55 | uses: actions/deploy-pages@v2 56 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/README.md: -------------------------------------------------------------------------------- 1 | # v2ex-analysis 2 | 3 | This template should help get you started developing with Vue 3 in Vite. 4 | 5 | ## Recommended IDE Setup 6 | 7 | [VSCode](https://code.visualstudio.com/) + [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) (and disable Vetur) + [TypeScript Vue Plugin (Volar)](https://marketplace.visualstudio.com/items?itemName=Vue.vscode-typescript-vue-plugin). 8 | 9 | ## Type Support for `.vue` Imports in TS 10 | 11 | TypeScript cannot handle type information for `.vue` imports by default, so we replace the `tsc` CLI with `vue-tsc` for type checking. In editors, we need [TypeScript Vue Plugin (Volar)](https://marketplace.visualstudio.com/items?itemName=Vue.vscode-typescript-vue-plugin) to make the TypeScript language service aware of `.vue` types. 12 | 13 | If the standalone TypeScript plugin doesn't feel fast enough to you, Volar has also implemented a [Take Over Mode](https://github.com/johnsoncodehk/volar/discussions/471#discussioncomment-1361669) that is more performant. You can enable it by the following steps: 14 | 15 | 1. Disable the built-in TypeScript Extension 16 | 1) Run `Extensions: Show Built-in Extensions` from VSCode's command palette 17 | 2) Find `TypeScript and JavaScript Language Features`, right click and select `Disable (Workspace)` 18 | 2. Reload the VSCode window by running `Developer: Reload Window` from the command palette. 19 | 20 | ## Customize configuration 21 | 22 | See [Vite Configuration Reference](https://vitejs.dev/config/). 23 | 24 | ## Project Setup 25 | 26 | ```sh 27 | npm install 28 | ``` 29 | 30 | ### Compile and Hot-Reload for Development 31 | 32 | ```sh 33 | npm run dev 34 | ``` 35 | 36 | ### Type-Check, Compile and Minify for Production 37 | 38 | ```sh 39 | npm run build 40 | ``` 41 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/TopMember.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 68 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/components/TopTopic.vue: -------------------------------------------------------------------------------- 1 | 34 | 35 | 76 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/src/App.vue: -------------------------------------------------------------------------------- 1 | 65 | 66 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /v2ex_scrapy/pipelines.py: -------------------------------------------------------------------------------- 1 | # Define your item pipelines here 2 | # 3 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 4 | # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html 5 | 6 | 7 | # useful for handling different item types with a single interface 8 | 9 | from typing import Any 10 | 11 | # don't remove 12 | from v2ex_scrapy.DB import DB 13 | from v2ex_scrapy.items import CommentItem, MemberItem, TopicItem, TopicSupplementItem 14 | 15 | ItemsType = TopicItem | CommentItem | MemberItem | TopicSupplementItem 16 | 17 | 18 | class TutorialScrapyPipeline: 19 | BATCH = 10 20 | 21 | def __init__(self): 22 | # Connect to SQLite database 23 | self.db = DB() 24 | self.data: dict[Any, list[ItemsType]] = { 25 | TopicItem: [], 26 | CommentItem: [], 27 | MemberItem: [], 28 | TopicSupplementItem: [], 29 | } 30 | 31 | def process_item( 32 | self, 33 | item: ItemsType | Any, 34 | spider, 35 | ): 36 | if isinstance(item, (TopicItem, CommentItem, MemberItem, TopicSupplementItem)): 37 | item_type = type(item) 38 | self.data[item_type].append(item) 39 | if len(self.data[item_type]) >= self.BATCH: 40 | self.process_it(self.data[item_type]) 41 | self.data[item_type] = [] 42 | return item 43 | 44 | def process_it(self, items: list[ItemsType]): 45 | if len(items) > 0 and isinstance(items[0], MemberItem): 46 | self.process_members(items) # type: ignore 47 | else: 48 | self.db.session.add_all(items) 49 | self.db.session.commit() 50 | 51 | def process_members(self, items: list[MemberItem]): 52 | for item in items: 53 | e = ( 54 | self.db.session.query(MemberItem) 55 | .where(MemberItem.username == item.username) 56 | .first() 57 | ) 58 | if e is None: 59 | self.db.session.add_all([item]) 60 | elif e.uid is None: 61 | e.uid = item.uid 62 | self.db.session.commit() 63 | 64 | def save_all(self): 65 | for _, v in self.data.items(): 66 | self.process_it(v) 67 | 68 | def close_spider(self, spider): 69 | self.save_all() 70 | self.db.close() 71 | -------------------------------------------------------------------------------- /analysis/main.py: -------------------------------------------------------------------------------- 1 | import pandas 2 | import sqlite3 3 | 4 | 5 | conn = sqlite3.connect("v2ex.sqlite") 6 | 7 | e = { 8 | "top-comment": """ 9 | select topic_id, c.id, c.content, c.thank_count, c.no, t.title 10 | from comment c 11 | left join topic t on t.id = c.topic_id 12 | order by c.thank_count desc 13 | """, 14 | "top-topic-by-thank_count": """ 15 | select id, title, thank_count 16 | from topic 17 | order by thank_count desc 18 | """, 19 | "top-topic-by-favorite_count": """ 20 | select id, title, favorite_count 21 | from topic 22 | order by favorite_count desc 23 | """, 24 | "top-topic-by-votes": """ 25 | select id, title, votes 26 | from topic 27 | order by votes desc 28 | """, 29 | "top-topic-by-clicks": """ 30 | select id, title, clicks 31 | from topic 32 | order by clicks desc 33 | """, 34 | "tag-usage-count": """ 35 | select t.value as tag, count(*) as count 36 | from topic, 37 | json_each(tag) as t 38 | group by t.value 39 | order by count desc 40 | """, 41 | "top-user-by-comment_count": """ 42 | select commenter as username, count(commenter) as comment_count 43 | from comment 44 | group by commenter 45 | order by comment_count desc 46 | """, 47 | "top-user-by-topic_count": """ 48 | select author as username, count(author) as topic_count 49 | from topic 50 | group by author 51 | order by topic_count desc 52 | """, 53 | } 54 | e2 = { 55 | "new-topic-every-month": """ 56 | SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS topic_count 57 | FROM topic 58 | GROUP BY date 59 | """, 60 | "new-comment-every-month": """ 61 | SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS comment_count 62 | FROM comment 63 | GROUP BY date 64 | """, 65 | "new-member-every-month": """ 66 | SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS member_count 67 | FROM member 68 | GROUP BY date 69 | """, 70 | } 71 | LIMIT = 20 72 | 73 | 74 | def f( 75 | sql: str, 76 | export_name: str, 77 | conn, 78 | limit: int | None = LIMIT, 79 | orient: str | None = "records", 80 | ): 81 | ae = "" if limit is None else f" limit {limit}" 82 | pandas.read_sql(f"{sql} {ae}", conn).to_json( 83 | f"./analysis/v2ex-analysis/public/{export_name}.json", 84 | force_ascii=False, 85 | orient=orient, 86 | ) 87 | 88 | 89 | for i, sql in e.items(): 90 | f(sql=sql, export_name=i, conn=conn) 91 | for i, sql in e2.items(): 92 | f(sql=sql, export_name=i, conn=conn, limit=None, orient=None) 93 | -------------------------------------------------------------------------------- /v2ex_scrapy/DB.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dataclasses import dataclass 3 | from typing import Type, Union 4 | 5 | from sqlalchemy import create_engine, text 6 | from sqlalchemy.orm import Mapped, Session, mapped_column 7 | 8 | from v2ex_scrapy.items import Base, CommentItem, MemberItem, TopicItem 9 | 10 | 11 | @dataclass(kw_only=True) 12 | class LogItem(Base): 13 | __tablename__ = "log" 14 | 15 | id_: Mapped[int] = mapped_column(name="id", primary_key=True, autoincrement="auto") 16 | url: Mapped[str] = mapped_column() 17 | status_code: Mapped[int] = mapped_column(nullable=False) 18 | create_at: Mapped[int] = mapped_column(nullable=False) 19 | 20 | 21 | class DB: 22 | _instance = None 23 | 24 | def __new__(cls): 25 | if cls._instance is None: 26 | cls._instance = super().__new__(cls) 27 | return cls._instance 28 | 29 | def __init__(self, database_name="v2ex.sqlite"): 30 | self.engine = create_engine( 31 | f"sqlite:///{database_name}", 32 | echo=False, 33 | json_serializer=lambda x: json.dumps(x, ensure_ascii=False), 34 | ) 35 | Base.metadata.create_all(self.engine) 36 | self.session = Session(self.engine) 37 | 38 | def close(self): 39 | self.session.commit() 40 | self.session.close() 41 | 42 | def exist( 43 | self, 44 | type_: Union[Type[TopicItem], Type[CommentItem], Type[MemberItem]], 45 | q: Union[str, int], 46 | ) -> bool: 47 | if type_ == MemberItem: 48 | query = text( 49 | f"SELECT * FROM {type_.__tablename__} WHERE {'username' if type(q) == str else 'uid'} = :q" 50 | ) 51 | else: 52 | query = text(f"SELECT * FROM {type_.__tablename__} WHERE id = :q") 53 | result = self.session.execute(query, {"q": q}).fetchone() 54 | return result is not None 55 | 56 | def get_max_topic_id(self) -> int: 57 | result = self.session.execute(text("SELECT max(id) FROM topic")).fetchone() 58 | if result is None or result[0] is None: 59 | return 1 60 | return int(result[0]) 61 | 62 | def get_topic_comment_count(self, topic_id) -> int: 63 | result = self.session.execute( 64 | text("select reply_count from topic where id = :q"), {"q": topic_id} 65 | ).fetchone() 66 | if result is None or result[0] is None: 67 | return 0 68 | return int(result[0]) 69 | 70 | def get_comment_count_by_topic(self, topic_id) -> int: 71 | result = self.session.execute( 72 | text("select count(*) from comment where topic_id = :q"), {"q": topic_id} 73 | ).fetchone() 74 | if result is None or result[0] is None: 75 | return 0 76 | return int(result[0]) -------------------------------------------------------------------------------- /v2ex_scrapy/spiders/V2exNodeTopicSpider.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | import scrapy 3 | import scrapy.http.response.html 4 | from parsel import Selector 5 | from scrapy.utils.project import get_project_settings 6 | 7 | from v2ex_scrapy.DB import DB 8 | from v2ex_scrapy.items import TopicItem 9 | from v2ex_scrapy.spiders.CommonSpider import CommonSpider 10 | from v2ex_scrapy import utils 11 | 12 | 13 | class V2exNodeTopicSpider(scrapy.Spider): 14 | name = "v2ex-node" 15 | 16 | UPDATE_TOPIC_WHEN_REPLY_CHANGE = True 17 | UPDATE_COMMENT = True # only work when UPDATE_TOPIC_WHEN_REPLY_CHANGE = True 18 | URL = "https://www.v2ex.com/go/" 19 | 20 | """ 21 | 现存在的几个问题，因为节点的排序是动态的，如果爬完一页后未爬的主题跑到爬完的页数里那就爬不到了。 22 | 解决方法1，开始爬取时先获取全部帖子ID再开始爬，获取ID的速度比较快所以排序改变的幅度不会很大。 23 | """ 24 | 25 | def __init__(self, node="flamewar", *args, **kwargs): 26 | super().__init__(*args, **kwargs) 27 | self.db = DB() 28 | self.node = node 29 | self.common_spider = CommonSpider( 30 | self.logger, update_comment=self.UPDATE_COMMENT 31 | ) 32 | settings = get_project_settings() 33 | resp = httpx.get( 34 | f"{self.URL}{self.node}", 35 | timeout=10, 36 | follow_redirects=True, 37 | cookies=utils.cookie_str2cookie_dict(settings.get("COOKIES", "")), # type: ignore 38 | headers={"User-Agent": settings.get("USER_AGENT", "")}, # type: ignore 39 | ).text 40 | max_page = ( 41 | Selector(text=resp) 42 | .xpath('//tr/td[@align="left" and @width="92%"]/a[last()]/text()') 43 | .get("1") 44 | ) 45 | self.max_page = int(max_page) 46 | 47 | def start_requests(self): 48 | for i in range(self.max_page, 0, -1): 49 | yield scrapy.Request( 50 | url=f"{self.URL}{self.node}?p={i}", 51 | callback=self.parse, 52 | cb_kwargs={"page": i}, 53 | ) 54 | 55 | def parse(self, response: scrapy.http.response.html.HtmlResponse, page: int): 56 | topics = [ 57 | (int(x), int(y)) 58 | for x, y in zip( 59 | response.xpath('//span[@class="item_title"]/a/@id').re(r"\d+"), 60 | # not correct when some comments are deleted, fuck 61 | response.xpath('//span[@class="item_title"]/a/@href').re(r"reply(\d+)"), 62 | ) 63 | ] 64 | for i, reply_count in topics: 65 | if not self.db.exist(TopicItem, i) or ( 66 | self.UPDATE_TOPIC_WHEN_REPLY_CHANGE 67 | and self.db.get_topic_comment_count(i) < reply_count 68 | ): 69 | yield scrapy.Request( 70 | url=f"https://www.v2ex.com/t/{i}", 71 | callback=self.common_spider.parse_topic, 72 | errback=self.common_spider.parse_topic_err, 73 | cb_kwargs={"topic_id": i}, 74 | ) 75 | -------------------------------------------------------------------------------- /query.sql: -------------------------------------------------------------------------------- 1 | SELECT (SELECT COUNT(*) FROM comment) AS 评论数, 2 | (SELECT COUNT(*) FROM member) AS 用户数, 3 | (SELECT COUNT(*) FROM topic) AS 主题数; 4 | 5 | -- where create_at between strftime('%s', '2013-01-01') and strftime('%s', '2014-12-31'); 6 | 7 | -- top comment by thank_count 8 | select topic_id, c.id, c.content, c.thank_count, c.no, t.title 9 | from comment c 10 | left join topic t on t.id = c.topic_id 11 | order by c.thank_count desc; 12 | 13 | -- top topic by thank_count 14 | select id, title, votes 15 | from topic 16 | order by thank_count desc; 17 | 18 | -- top topic by favorite_count 19 | select id, title, favorite_count 20 | from topic 21 | order by favorite_count desc; 22 | 23 | -- top topic by votes 24 | select id, title, votes 25 | from topic 26 | order by votes desc; 27 | 28 | -- top topic by clicks 29 | select id, title, clicks 30 | from topic 31 | order by clicks desc; 32 | 33 | 34 | -- top node 35 | select node, count(node) as count 36 | from topic 37 | group by node 38 | order by count desc; 39 | 40 | -- comment number group by user 41 | select commenter as username, count(commenter) as comment_count 42 | from comment 43 | group by commenter 44 | order by comment_count desc; 45 | 46 | -- topic number group by user 47 | select author as username, count(author) as topic_count 48 | from topic 49 | group by author 50 | order by topic_count desc; 51 | 52 | -- topic number group by year-month 53 | SELECT date, 54 | SUM(topic_count) OVER (ORDER BY date ) AS cumulative_topic_count 55 | FROM (SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS topic_count 56 | FROM topic 57 | GROUP BY date) 58 | ORDER BY date; 59 | 60 | -- user number group by year-month 61 | SELECT date, 62 | SUM(user_count) OVER (ORDER BY date ) AS cumulative_user_count 63 | FROM (SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS user_count 64 | FROM member 65 | GROUP BY date) 66 | ORDER BY date; 67 | 68 | -- comment number group by year-month 69 | SELECT date, 70 | SUM(comment_count) OVER (ORDER BY date ) AS cumulative_comment_count 71 | FROM (SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS comment_count 72 | FROM comment 73 | GROUP BY date) 74 | ORDER BY date; 75 | 76 | -- new topic number group by year-month 77 | SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS topic_count 78 | FROM topic 79 | GROUP BY date; 80 | 81 | -- new user number group by year-month 82 | SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS user_count 83 | FROM member 84 | GROUP BY date; 85 | 86 | -- new comment number group by year-month 87 | SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS comment_count 88 | FROM comment 89 | GROUP BY date; 90 | 91 | -- tag usage count 92 | select t.value as tag, count(*) as count 93 | from topic, 94 | json_each(tag) as t 95 | group by t.value 96 | order by count desc; 97 | 98 | -- node usage count 99 | select node, count(*) as count 100 | from topic 101 | group by node 102 | order by count desc; 103 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/new-topic-every-month.json: -------------------------------------------------------------------------------- 1 | {"date":{"0":"2010-08","1":"2010-11","2":"2010-12","3":"2011-01","4":"2011-02","5":"2011-03","6":"2011-04","7":"2011-05","8":"2011-07","9":"2011-09","10":"2011-11","11":"2011-12","12":"2012-01","13":"2012-04","14":"2012-05","15":"2012-06","16":"2012-07","17":"2012-08","18":"2012-09","19":"2012-10","20":"2012-11","21":"2012-12","22":"2013-01","23":"2013-02","24":"2013-03","25":"2013-04","26":"2013-05","27":"2013-06","28":"2013-07","29":"2013-08","30":"2013-09","31":"2013-11","32":"2013-12","33":"2014-01","34":"2014-02","35":"2014-05","36":"2014-06","37":"2014-07","38":"2014-08","39":"2014-09","40":"2014-10","41":"2014-11","42":"2014-12","43":"2015-01","44":"2015-02","45":"2015-03","46":"2015-04","47":"2015-05","48":"2015-06","49":"2015-07","50":"2015-08","51":"2015-09","52":"2015-10","53":"2015-11","54":"2015-12","55":"2016-01","56":"2016-02","57":"2016-03","58":"2016-04","59":"2016-05","60":"2016-06","61":"2016-07","62":"2016-08","63":"2016-09","64":"2016-10","65":"2016-11","66":"2016-12","67":"2017-01","68":"2017-02","69":"2017-03","70":"2017-04","71":"2017-05","72":"2017-06","73":"2017-07","74":"2017-08","75":"2017-09","76":"2017-10","77":"2017-11","78":"2017-12","79":"2018-01","80":"2018-02","81":"2018-03","82":"2018-04","83":"2018-05","84":"2018-06","85":"2018-07","86":"2018-08","87":"2018-09","88":"2018-10","89":"2018-11","90":"2018-12","91":"2019-01","92":"2019-02","93":"2019-03","94":"2019-04","95":"2019-05","96":"2019-06","97":"2019-07","98":"2019-08","99":"2019-09","100":"2019-10","101":"2019-11","102":"2019-12","103":"2020-01","104":"2020-02","105":"2020-03","106":"2020-04","107":"2020-05","108":"2020-06","109":"2020-07","110":"2020-08","111":"2020-09","112":"2020-10","113":"2020-11","114":"2020-12","115":"2021-01","116":"2021-02","117":"2021-03","118":"2021-04","119":"2021-05","120":"2021-06","121":"2021-07","122":"2021-08","123":"2021-09","124":"2021-10","125":"2021-11","126":"2021-12","127":"2022-01","128":"2022-02","129":"2022-03","130":"2022-04","131":"2022-05","132":"2022-06","133":"2022-07","134":"2022-08","135":"2022-09","136":"2022-10","137":"2022-11","138":"2022-12","139":"2023-01","140":"2023-02","141":"2023-03","142":"2023-04","143":"2023-05","144":"2023-06","145":"2023-07"},"topic_count":{"0":1,"1":2,"2":1,"3":4,"4":2,"5":2,"6":1,"7":1,"8":3,"9":3,"10":2,"11":2,"12":1,"13":6,"14":6,"15":1,"16":4,"17":1,"18":4,"19":3,"20":4,"21":4,"22":2,"23":5,"24":1,"25":4,"26":3,"27":1,"28":3,"29":9,"30":1,"31":2,"32":2,"33":1,"34":1,"35":3,"36":8,"37":6,"38":13,"39":7,"40":9,"41":5,"42":22,"43":34,"44":29,"45":31,"46":32,"47":17,"48":22,"49":21,"50":51,"51":29,"52":18,"53":25,"54":38,"55":39,"56":25,"57":33,"58":40,"59":46,"60":55,"61":50,"62":42,"63":62,"64":72,"65":43,"66":63,"67":47,"68":35,"69":55,"70":74,"71":49,"72":50,"73":88,"74":75,"75":60,"76":81,"77":106,"78":61,"79":84,"80":55,"81":84,"82":70,"83":66,"84":84,"85":80,"86":185,"87":118,"88":85,"89":95,"90":128,"91":136,"92":90,"93":152,"94":376,"95":261,"96":226,"97":83,"98":167,"99":148,"100":157,"101":118,"102":166,"103":211,"104":213,"105":125,"106":209,"107":174,"108":146,"109":149,"110":175,"111":179,"112":131,"113":143,"114":182,"115":241,"116":105,"117":259,"118":169,"119":103,"120":130,"121":103,"122":113,"123":66,"124":67,"125":89,"126":146,"127":112,"128":79,"129":165,"130":160,"131":142,"132":113,"133":136,"134":112,"135":156,"136":153,"137":238,"138":211,"139":37,"140":50,"141":98,"142":81,"143":102,"144":118,"145":23}} -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/new-comment-every-month.json: -------------------------------------------------------------------------------- 1 | {"date":{"0":"2010-08","1":"2010-11","2":"2010-12","3":"2011-01","4":"2011-02","5":"2011-03","6":"2011-04","7":"2011-05","8":"2011-06","9":"2011-07","10":"2011-08","11":"2011-09","12":"2011-10","13":"2011-11","14":"2011-12","15":"2012-01","16":"2012-02","17":"2012-04","18":"2012-05","19":"2012-06","20":"2012-07","21":"2012-08","22":"2012-09","23":"2012-10","24":"2012-11","25":"2012-12","26":"2013-01","27":"2013-02","28":"2013-03","29":"2013-04","30":"2013-05","31":"2013-06","32":"2013-07","33":"2013-08","34":"2013-09","35":"2013-10","36":"2013-11","37":"2013-12","38":"2014-01","39":"2014-02","40":"2014-03","41":"2014-04","42":"2014-05","43":"2014-06","44":"2014-07","45":"2014-08","46":"2014-09","47":"2014-10","48":"2014-11","49":"2014-12","50":"2015-01","51":"2015-02","52":"2015-03","53":"2015-04","54":"2015-05","55":"2015-06","56":"2015-07","57":"2015-08","58":"2015-09","59":"2015-10","60":"2015-11","61":"2015-12","62":"2016-01","63":"2016-02","64":"2016-03","65":"2016-04","66":"2016-05","67":"2016-06","68":"2016-07","69":"2016-08","70":"2016-09","71":"2016-10","72":"2016-11","73":"2016-12","74":"2017-01","75":"2017-02","76":"2017-03","77":"2017-04","78":"2017-05","79":"2017-06","80":"2017-07","81":"2017-08","82":"2017-09","83":"2017-10","84":"2017-11","85":"2017-12","86":"2018-01","87":"2018-02","88":"2018-03","89":"2018-04","90":"2018-05","91":"2018-06","92":"2018-07","93":"2018-08","94":"2018-09","95":"2018-10","96":"2018-11","97":"2018-12","98":"2019-01","99":"2019-02","100":"2019-03","101":"2019-04","102":"2019-05","103":"2019-06","104":"2019-07","105":"2019-08","106":"2019-09","107":"2019-10","108":"2019-11","109":"2019-12","110":"2020-01","111":"2020-02","112":"2020-03","113":"2020-04","114":"2020-05","115":"2020-06","116":"2020-07","117":"2020-08","118":"2020-09","119":"2020-10","120":"2020-11","121":"2020-12","122":"2021-01","123":"2021-02","124":"2021-03","125":"2021-04","126":"2021-05","127":"2021-06","128":"2021-07","129":"2021-08","130":"2021-09","131":"2021-10","132":"2021-11","133":"2021-12","134":"2022-01","135":"2022-02","136":"2022-03","137":"2022-04","138":"2022-05","139":"2022-06","140":"2022-07","141":"2022-08","142":"2022-09","143":"2022-10","144":"2022-11","145":"2022-12","146":"2023-01","147":"2023-02","148":"2023-03","149":"2023-04","150":"2023-05","151":"2023-06","152":"2023-07"},"comment_count":{"0":49,"1":42,"2":52,"3":182,"4":44,"5":34,"6":17,"7":48,"8":1,"9":60,"10":1,"11":65,"12":1,"13":48,"14":73,"15":57,"16":5,"17":180,"18":161,"19":18,"20":128,"21":104,"22":126,"23":180,"24":192,"25":105,"26":123,"27":164,"28":53,"29":99,"30":348,"31":4,"32":147,"33":757,"34":174,"35":4,"36":30,"37":26,"38":144,"39":56,"40":3,"41":3,"42":269,"43":644,"44":366,"45":648,"46":479,"47":449,"48":286,"49":839,"50":1179,"51":1016,"52":1527,"53":1905,"54":597,"55":1039,"56":1043,"57":2393,"58":1301,"59":707,"60":1745,"61":1825,"62":1840,"63":791,"64":1826,"65":2579,"66":2947,"67":2816,"68":2292,"69":2897,"70":3532,"71":3112,"72":2366,"73":2863,"74":3027,"75":1557,"76":2487,"77":3543,"78":1338,"79":2468,"80":4010,"81":3312,"82":2909,"83":2395,"84":3057,"85":2369,"86":2960,"87":2376,"88":2824,"89":2605,"90":2577,"91":3320,"92":3798,"93":8143,"94":5222,"95":2946,"96":4074,"97":4590,"98":6005,"99":3726,"100":6416,"101":10941,"102":8871,"103":6404,"104":4549,"105":8419,"106":6374,"107":7087,"108":6853,"109":6355,"110":6474,"111":6449,"112":6167,"113":6764,"114":7132,"115":5096,"116":6641,"117":7947,"118":8642,"119":5694,"120":6656,"121":8615,"122":8899,"123":4894,"124":8747,"125":4202,"126":3137,"127":6306,"128":3550,"129":4182,"130":2482,"131":2883,"132":3852,"133":5221,"134":4334,"135":2896,"136":5800,"137":6875,"138":7055,"139":6092,"140":6495,"141":6403,"142":7875,"143":7115,"144":13524,"145":11616,"146":2572,"147":3940,"148":6361,"149":5279,"150":7118,"151":8278,"152":1278}} -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/new-member-every-month.json: -------------------------------------------------------------------------------- 1 | {"date":{"0":"1970-01","1":"2010-04","2":"2010-05","3":"2010-06","4":"2010-07","5":"2010-08","6":"2010-09","7":"2010-10","8":"2010-11","9":"2010-12","10":"2011-01","11":"2011-02","12":"2011-03","13":"2011-04","14":"2011-05","15":"2011-06","16":"2011-07","17":"2011-08","18":"2011-09","19":"2011-10","20":"2011-11","21":"2011-12","22":"2012-01","23":"2012-02","24":"2012-03","25":"2012-04","26":"2012-05","27":"2012-06","28":"2012-07","29":"2012-08","30":"2012-09","31":"2012-10","32":"2012-11","33":"2012-12","34":"2013-01","35":"2013-02","36":"2013-03","37":"2013-04","38":"2013-05","39":"2013-06","40":"2013-07","41":"2013-08","42":"2013-09","43":"2013-10","44":"2013-11","45":"2013-12","46":"2014-01","47":"2014-02","48":"2014-03","49":"2014-04","50":"2014-05","51":"2014-06","52":"2014-07","53":"2014-08","54":"2014-09","55":"2014-10","56":"2014-11","57":"2014-12","58":"2015-01","59":"2015-02","60":"2015-03","61":"2015-04","62":"2015-05","63":"2015-06","64":"2015-07","65":"2015-08","66":"2015-09","67":"2015-10","68":"2015-11","69":"2015-12","70":"2016-01","71":"2016-02","72":"2016-03","73":"2016-04","74":"2016-05","75":"2016-06","76":"2016-07","77":"2016-08","78":"2016-09","79":"2016-10","80":"2016-11","81":"2016-12","82":"2017-01","83":"2017-02","84":"2017-03","85":"2017-04","86":"2017-05","87":"2017-06","88":"2017-07","89":"2017-08","90":"2017-09","91":"2017-10","92":"2017-11","93":"2017-12","94":"2018-01","95":"2018-02","96":"2018-03","97":"2018-04","98":"2018-05","99":"2018-06","100":"2018-07","101":"2018-08","102":"2018-09","103":"2018-10","104":"2018-11","105":"2018-12","106":"2019-01","107":"2019-02","108":"2019-03","109":"2019-04","110":"2019-05","111":"2019-06","112":"2019-07","113":"2019-08","114":"2019-09","115":"2019-10","116":"2019-11","117":"2019-12","118":"2020-01","119":"2020-02","120":"2020-03","121":"2020-04","122":"2020-05","123":"2020-06","124":"2020-07","125":"2020-08","126":"2020-09","127":"2020-10","128":"2020-11","129":"2020-12","130":"2021-01","131":"2021-02","132":"2021-03","133":"2021-04","134":"2021-05","135":"2021-06","136":"2021-07","137":"2021-08","138":"2021-09","139":"2021-10","140":"2021-11","141":"2021-12","142":"2022-01","143":"2022-02","144":"2022-03","145":"2022-04","146":"2022-05","147":"2022-06","148":"2022-07","149":"2022-08","150":"2022-09","151":"2022-10","152":"2022-11","153":"2022-12","154":"2023-01","155":"2023-02","156":"2023-03","157":"2023-04","158":"2023-05","159":"2023-06","160":"2023-07"},"member_count":{"0":385,"1":120,"2":57,"3":8,"4":66,"5":63,"6":57,"7":93,"8":80,"9":128,"10":118,"11":56,"12":139,"13":102,"14":79,"15":77,"16":85,"17":72,"18":84,"19":85,"20":116,"21":143,"22":72,"23":139,"24":204,"25":173,"26":157,"27":102,"28":156,"29":156,"30":142,"31":142,"32":127,"33":175,"34":174,"35":148,"36":208,"37":142,"38":177,"39":149,"40":170,"41":217,"42":191,"43":183,"44":234,"45":178,"46":190,"47":160,"48":219,"49":233,"50":254,"51":361,"52":351,"53":394,"54":344,"55":455,"56":511,"57":517,"58":677,"59":560,"60":523,"61":569,"62":423,"63":432,"64":425,"65":493,"66":378,"67":388,"68":344,"69":342,"70":325,"71":301,"72":389,"73":499,"74":402,"75":364,"76":391,"77":395,"78":377,"79":417,"80":342,"81":409,"82":330,"83":430,"84":477,"85":350,"86":412,"87":414,"88":600,"89":498,"90":460,"91":473,"92":597,"93":486,"94":749,"95":453,"96":567,"97":486,"98":463,"99":442,"100":715,"101":737,"102":608,"103":466,"104":521,"105":556,"106":556,"107":431,"108":615,"109":662,"110":562,"111":518,"112":463,"113":395,"114":373,"115":380,"116":387,"117":369,"118":291,"119":262,"120":366,"121":324,"122":277,"123":226,"124":248,"125":225,"126":236,"127":253,"128":254,"129":222,"130":246,"131":157,"132":210,"133":171,"134":126,"135":110,"136":121,"137":106,"138":114,"139":114,"140":149,"141":143,"142":126,"143":100,"144":129,"145":118,"146":138,"147":134,"148":113,"149":137,"150":99,"151":131,"152":122,"153":91,"154":72,"155":101,"156":107,"157":77,"158":83,"159":45,"160":1}} -------------------------------------------------------------------------------- /v2ex_scrapy/items.py: -------------------------------------------------------------------------------- 1 | # Define here the models for your scraped items 2 | # 3 | # See documentation in: 4 | # https://docs.scrapy.org/en/latest/topics/items.html 5 | 6 | import json 7 | from dataclasses import dataclass 8 | 9 | from sqlalchemy import Integer, Text, types 10 | from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column 11 | 12 | 13 | class JSONText(types.TypeDecorator): 14 | impl = types.Text 15 | 16 | def process_bind_param(self, value, dialect): 17 | if value is not None: 18 | value = json.dumps(value, ensure_ascii=False) 19 | return value 20 | 21 | def process_result_value(self, value, dialect): 22 | if value is not None: 23 | value = json.loads(value) 24 | return value 25 | 26 | 27 | @dataclass 28 | class Base(DeclarativeBase): 29 | type_annotation_map = { 30 | list[str]: JSONText, 31 | list[dict[str, str]]: JSONText, 32 | int: Integer, 33 | str: Text, 34 | } 35 | 36 | 37 | @dataclass(kw_only=True) 38 | class TopicItem(Base): 39 | __tablename__ = "topic" 40 | 41 | id_: Mapped[int] = mapped_column(name="id", primary_key=True) 42 | author: Mapped[str] = mapped_column(nullable=False) 43 | title: Mapped[str] = mapped_column(nullable=False) 44 | content: Mapped[str] = mapped_column() 45 | node: Mapped[str] = mapped_column(nullable=False) 46 | tag: Mapped[list[str]] = mapped_column(nullable=False) 47 | clicks: Mapped[int] = mapped_column(nullable=False) 48 | votes: Mapped[int] = mapped_column(nullable=False) 49 | create_at: Mapped[int] = mapped_column(nullable=False) 50 | thank_count: Mapped[int] = mapped_column(nullable=False) 51 | favorite_count: Mapped[int] = mapped_column(nullable=False) 52 | reply_count: Mapped[int] = mapped_column(nullable=False) 53 | 54 | @staticmethod 55 | def err_topic(topic_id: int): 56 | return TopicItem( 57 | id_=topic_id, 58 | author="", 59 | title="", 60 | content="", 61 | create_at=0, 62 | node="", 63 | tag=[], 64 | clicks=-1, 65 | votes=-1, 66 | thank_count=-1, 67 | favorite_count=-1, 68 | reply_count=-1, 69 | ) 70 | 71 | 72 | @dataclass(kw_only=True) 73 | class TopicSupplementItem(Base): 74 | __tablename__ = "topic_supplement" 75 | 76 | topic_id: Mapped[int] = mapped_column(primary_key=True) 77 | content: Mapped[str] = mapped_column(primary_key=True) 78 | create_at: Mapped[int] = mapped_column(primary_key=True) 79 | 80 | 81 | @dataclass(kw_only=True) 82 | class CommentItem(Base): 83 | __tablename__ = "comment" 84 | 85 | id_: Mapped[int] = mapped_column(name="id", primary_key=True) 86 | # index used for select count(*) from comment where topic_id = ?, see DB.get_topic_comment_count 87 | topic_id: Mapped[int] = mapped_column(nullable=False, index=True) 88 | commenter: Mapped[str] = mapped_column(nullable=False) 89 | content: Mapped[str] = mapped_column(nullable=False) 90 | thank_count: Mapped[int] = mapped_column(nullable=False) 91 | create_at: Mapped[int] = mapped_column(nullable=False) 92 | no: Mapped[int] = mapped_column(nullable=False) 93 | 94 | 95 | @dataclass(kw_only=True) 96 | class MemberItem(Base): 97 | __tablename__ = "member" 98 | """ 99 | crawl user from topic and comment, then crawl from uid 1 to 1000000, 100 | if get 404, username/uid will be ''/-1, so primary_key(uid ,username) 101 | """ 102 | uid: Mapped[int] = mapped_column(primary_key=True) 103 | username: Mapped[str] = mapped_column(primary_key=True, index=True) 104 | avatar_url: Mapped[str] 105 | create_at: Mapped[int] = mapped_column(nullable=False) 106 | social_link: Mapped[list[dict[str, str]]] 107 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | .vscode/settings.json 163 | *.7z 164 | *.sqlite 165 | -------------------------------------------------------------------------------- /v2ex_scrapy/settings.py: -------------------------------------------------------------------------------- 1 | # My Config 2 | 3 | PROXIES = [] 4 | 5 | COOKIES = """ 6 | """ 7 | 8 | # Scrapy settings for v2ex_scrapy project 9 | # 10 | # For simplicity, this file contains only settings considered important or 11 | # commonly used. You can find more settings consulting the documentation: 12 | # 13 | # https://docs.scrapy.org/en/latest/topics/settings.html 14 | # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html 15 | # https://docs.scrapy.org/en/latest/topics/spider-middleware.html 16 | 17 | BOT_NAME = "v2ex_scrapy" 18 | 19 | SPIDER_MODULES = ["v2ex_scrapy.spiders"] 20 | NEWSPIDER_MODULE = "v2ex_scrapy.spiders" 21 | 22 | # LOG_FILE = "v2ex_scrapy.log" 23 | LOG_FILE_APPEND = False 24 | 25 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 26 | USER_AGENT = "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148" 27 | 28 | # Obey robots.txt rules 29 | ROBOTSTXT_OBEY = False 30 | 31 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 32 | CONCURRENT_REQUESTS = 1 33 | 34 | # Configure a delay for requests for the same website (default: 0) 35 | # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay 36 | # See also autothrottle settings and docs 37 | # DOWNLOAD_DELAY = 3 38 | # The download delay setting will honor only one of: 39 | # CONCURRENT_REQUESTS_PER_DOMAIN = 16 40 | # CONCURRENT_REQUESTS_PER_IP = 16 41 | 42 | # Disable cookies (enabled by default) 43 | # COOKIES_ENABLED = False 44 | 45 | # Disable Telnet Console (enabled by default) 46 | # TELNETCONSOLE_ENABLED = False 47 | 48 | # Override the default request headers: 49 | # DEFAULT_REQUEST_HEADERS = { 50 | # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 51 | # "Accept-Language": "en", 52 | # } 53 | 54 | # Enable or disable spider middlewares 55 | # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html 56 | # SPIDER_MIDDLEWARES = { 57 | # "tutorial_scrapy.middlewares.TutorialScrapySpiderMiddleware": 543, 58 | # } 59 | 60 | # Enable or disable downloader middlewares 61 | # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html 62 | DOWNLOADER_MIDDLEWARES = { 63 | "v2ex_scrapy.middlewares.ProxyAndCookieDownloaderMiddleware": 543, 64 | # "v2ex_scrapy.middlewares.RandomUserAgentMiddleware": 544, 65 | "v2ex_scrapy.middlewares.SaveHttpStatusToDBMiddleware": 545, 66 | } 67 | 68 | # Enable or disable extensions 69 | # See https://docs.scrapy.org/en/latest/topics/extensions.html 70 | # EXTENSIONS = { 71 | # "scrapy.extensions.telnet.TelnetConsole": None, 72 | # } 73 | 74 | # Configure item pipelines 75 | # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html 76 | ITEM_PIPELINES = { 77 | "v2ex_scrapy.pipelines.TutorialScrapyPipeline": 300, 78 | } 79 | 80 | # Enable and configure the AutoThrottle extension (disabled by default) 81 | # See https://docs.scrapy.org/en/latest/topics/autothrottle.html 82 | # AUTOTHROTTLE_ENABLED = True 83 | # The initial download delay 84 | # AUTOTHROTTLE_START_DELAY = 5 85 | # The maximum download delay to be set in case of high latencies 86 | # AUTOTHROTTLE_MAX_DELAY = 60 87 | # The average number of requests Scrapy should be sending in parallel to 88 | # each remote server 89 | # AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 90 | # Enable showing throttling stats for every response received: 91 | # AUTOTHROTTLE_DEBUG = False 92 | 93 | # Enable and configure HTTP caching (disabled by default) 94 | # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 95 | # HTTPCACHE_ENABLED = True 96 | # HTTPCACHE_EXPIRATION_SECS = 0 97 | # HTTPCACHE_DIR = "httpcache" 98 | # HTTPCACHE_IGNORE_HTTP_CODES = [] 99 | # HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" 100 | 101 | # Set settings whose default value is deprecated to a future-proof value 102 | REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" 103 | TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" 104 | FEED_EXPORT_ENCODING = "utf-8" 105 | -------------------------------------------------------------------------------- /v2ex_scrapy/spiders/CommonSpider.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import scrapy 4 | import scrapy.http.response.html 5 | from scrapy.spidermiddlewares.httperror import HttpError 6 | 7 | from v2ex_scrapy import v2ex_parser 8 | from v2ex_scrapy.DB import DB 9 | from v2ex_scrapy.items import MemberItem, TopicItem 10 | 11 | 12 | class CommonSpider: 13 | def __init__(self, logger, update_member=False, update_comment=False): 14 | self.db = DB() 15 | self.logger = logger 16 | self.UPDATE_MEMBER = update_member 17 | self.UPDATE_COMMENT = update_comment 18 | 19 | def parse_topic_err(self, failure): 20 | if failure.check(HttpError): 21 | topic_id = failure.request.cb_kwargs["topic_id"] 22 | self.logger.warn(f"Crawl Topic Err {topic_id}") 23 | yield TopicItem.err_topic(topic_id=topic_id) 24 | 25 | def parse_topic( 26 | self, response: scrapy.http.response.html.HtmlResponse, topic_id: int 27 | ): 28 | self.logger.info(f"Crawl Topic {topic_id}") 29 | 30 | if response.status == 302: 31 | # need login or account too young 32 | yield TopicItem.err_topic(topic_id=topic_id) 33 | else: 34 | for i in v2ex_parser.parse_topic_supplement(response, topic_id): 35 | yield i 36 | for topic in v2ex_parser.parse_topic(response, topic_id): 37 | yield topic 38 | for i in self.crawl_member(topic.author, response): 39 | yield i 40 | for i in self.parse_comment(response, topic_id): 41 | yield i 42 | # crawl sub page comment 43 | topic_reply_count = int( 44 | response.css( 45 | "#Main > div:nth-child(4) > div:nth-child(1) > span::text" 46 | ).re_first(r"\d+", "-1") 47 | ) 48 | c = self.db.get_topic_comment_count(topic_id) 49 | if ( 50 | # 爬了一部分并且设置更新评论 51 | (0 < c < topic_reply_count) 52 | and self.UPDATE_COMMENT 53 | ) or ( 54 | # 没有爬并且有评论 55 | topic_reply_count > 0 56 | and c == 0 57 | ): 58 | total_page = math.ceil(topic_reply_count / 100) 59 | for i in range(max(2, math.ceil(c / 100)), total_page + 1): 60 | for j in self.crawl_comment(topic_id, i, response): 61 | yield j 62 | 63 | def crawl_comment(self, topic_id, page, response): 64 | yield response.follow( 65 | f"/t/{topic_id}?p={page}", 66 | callback=self.parse_comment, 67 | cb_kwargs={"topic_id": topic_id}, 68 | ) 69 | 70 | def parse_comment(self, response: scrapy.http.response.html.HtmlResponse, topic_id): 71 | for comment_item in v2ex_parser.parse_comment(response, topic_id): 72 | yield comment_item 73 | for i in self.crawl_member(comment_item.commenter, response): 74 | yield i 75 | 76 | def crawl_member(self, username, response: scrapy.http.response.html.HtmlResponse): 77 | if username != "" and ( 78 | self.UPDATE_MEMBER or not self.db.exist(MemberItem, username) 79 | ): 80 | yield response.follow( 81 | f"/member/{username}", 82 | callback=self.parse_member, 83 | errback=self.member_err, 84 | cb_kwargs={"username": username}, 85 | ) 86 | 87 | def member_err(self, failure): 88 | if failure.check(HttpError): 89 | username = failure.request.cb_kwargs["username"] 90 | self.logger.warn(f"Crawl Member Err {username}") 91 | yield MemberItem( 92 | username=username, 93 | avatar_url="", 94 | create_at=0, 95 | social_link=[], 96 | uid=-1, 97 | ) 98 | 99 | def parse_member( 100 | self, response: scrapy.http.response.html.HtmlResponse, username: str 101 | ): 102 | self.logger.info(f"Crawl Member {username}") 103 | for i in v2ex_parser.parse_member(response=response): 104 | yield i 105 | -------------------------------------------------------------------------------- /v2ex_scrapy/v2ex_parser.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | import scrapy.http.response.html 3 | 4 | import v2ex_scrapy.utils as utils 5 | from v2ex_scrapy.items import ( 6 | CommentItem, 7 | MemberItem, 8 | TopicItem, 9 | TopicSupplementItem, 10 | ) 11 | 12 | 13 | def parse_member(response: scrapy.http.response.html.HtmlResponse): 14 | """parse page like https://www.v2ex.com/member/oldshensheep""" 15 | 16 | username = response.xpath("//h1/text()").get("") 17 | avatar_url = response.css(".avatar::attr(src)").get("-1") 18 | 19 | t = response.xpath('//div[@class="cell"]//tr/td/span[@class="gray"]/text()') 20 | no = t.re_first(r"第 (\d+) 号", "-1") 21 | create_at = t.re_first(r"加入于 (.*)", "") 22 | 23 | social_list = [] 24 | for i in response.xpath('//div[@class="widgets"]//a'): 25 | social_list.append({i.xpath(".//img/@alt").get(): i.xpath("./@href").get()}) 26 | yield MemberItem( 27 | username=username, 28 | avatar_url=avatar_url, 29 | create_at=utils.time_to_timestamp(create_at), 30 | social_link=social_list, 31 | uid=int(no), 32 | ) 33 | 34 | 35 | def parse_comment(response: scrapy.http.response.html.HtmlResponse, topic_id): 36 | reply_box = response.css("#Main > .box > .cell[id] > table") 37 | for reply_row in reply_box: 38 | comment_id = reply_row.xpath("..").css(".cell::attr(id)").re_first(r"\d+", "-1") 39 | # if not self.db.exist(CommentItem, comment_id): 40 | cbox = reply_row.css("tr") 41 | author_name = cbox.css(".dark::text").get("-1") 42 | reply_content = cbox.xpath('.//div[@class="reply_content"]').get("") 43 | reply_time = cbox.css(".ago::attr(title)").get("") 44 | thank_count = cbox.css(".fade::text").get("0").strip() 45 | no = cbox.css(".no::text").get("-1").strip() 46 | yield CommentItem( 47 | id_=int(comment_id), 48 | no=int(no), 49 | commenter=author_name, 50 | topic_id=topic_id, 51 | content=reply_content, 52 | create_at=utils.time_to_timestamp(reply_time), 53 | thank_count=int(thank_count), 54 | ) 55 | 56 | 57 | def parse_topic(response: scrapy.http.response.html.HtmlResponse, topic_id): 58 | topic_title = response.xpath("string(//div[@class='header']/h1)").get("") 59 | topic_time = response.css(".header > small > span::attr(title)").get("0") 60 | topic_author = response.css(".header > small > a::text").get("") 61 | topic_node = response.css(".header > a:nth-child(4)::attr(href)").re_first( 62 | r"\/(\w+)$", "" 63 | ) 64 | topic_click_count = response.css(".header > small::text").re_first(r"\d+", "-1") 65 | topic_tags = response.css(".tag::attr(href)").re(r"/tag/(.*)") 66 | topic_vote = response.xpath('(//a[@class="vote"])[1]/text()').re_first(r"\d+", "0") 67 | # need login, some topics may not have 68 | topic_favorite_count = -1 69 | topic_thank_count = -1 70 | if response.css(".topic_stats::text").get() is not None: 71 | topic_favorite_count = response.css(".topic_stats::text").re_first( 72 | r"(\d+) 人收藏", "0" 73 | ) 74 | topic_thank_count = response.css(".topic_stats::text").re_first( 75 | r"(\d+) 人感谢", "0" 76 | ) 77 | 78 | topic_content = response.css(".cell .topic_content").get("") 79 | topic_reply_count = response.css(".box > .cell > .gray::text").re_first( 80 | r"(\d+) 条回复", "0" 81 | ) 82 | yield TopicItem( 83 | id_=topic_id, 84 | author=topic_author, 85 | title=topic_title, 86 | content=topic_content, 87 | create_at=utils.time_to_timestamp(topic_time), 88 | node=topic_node, 89 | tag=topic_tags, 90 | clicks=int(topic_click_count), 91 | votes=int(topic_vote), 92 | thank_count=int(topic_thank_count), 93 | favorite_count=int(topic_favorite_count), 94 | reply_count=int(topic_reply_count), 95 | ) 96 | 97 | 98 | def parse_topic_supplement(response: scrapy.http.response.html.HtmlResponse, topic_id): 99 | for i in response.css(".subtle"): 100 | subtle_content = i.xpath('string(div[@class="topic_content"])').get("") 101 | subtle_create_at = i.xpath("string(//span[@title])").get("") 102 | yield TopicSupplementItem( 103 | topic_id=topic_id, 104 | content=subtle_content, 105 | create_at=utils.time_to_timestamp(subtle_create_at), 106 | ) 107 | -------------------------------------------------------------------------------- /analysis/v2ex-analysis/public/top-comment.json: -------------------------------------------------------------------------------- 1 | [{"topic_id":898195,"id":12390171,"content":"

@FKekule<\/a> 麻木的傻狗<\/div>","thank_count":424,"no":7,"title":"一百年来，学生们从未改变"},{"topic_id":920707,"id":12767615,"content":"

属于狗娘养的不要碧莲的一种了<\/div>","thank_count":298,"no":8,"title":"你最好不给你的邻居连你的 wifi，对吧？"},{"topic_id":506105,"id":6428438,"content":"

一定要去公司闹一闹，万一人家公司一直是 965，岂不是自己朦在鼓里。<\/div>","thank_count":274,"no":5,"title":"我女朋友公司已经 9247 连续两个月了。我想去砸了他们公司。"},{"topic_id":525604,"id":6729688,"content":"

当年张小龙的微信界面长期不变化，是因为他的用户群体几亿，所以保持了对改变和加功能的克制；而今天对微信界面进行了巨大的改动，则是不畏惧巨大用户量勇敢地创新。张小龙不愧是产品经理之神。张小龙距离乔布斯只差去世了。<\/div>","thank_count":237,"no":1,"title":"张小龙已经成神了么， 4 个小时的演讲也看啊。。。"},{"topic_id":949869,"id":13237521,"content":"

“基本朝九晚五一年能到手 90 万左右，我知道论坛上很多人估计是看不上这个收入”，我以为我在看知乎<\/div>","thank_count":217,"no":3,"title":"这种情况下的我该不该润出北京"},{"topic_id":742124,"id":10026058,"content":"

@php8<\/a> 回老家过年是陋习？？？凡是你不喜欢的都是陋习？省省吧您？<\/div>","thank_count":214,"no":8,"title":"大部分地区都不鼓励回家了吧，你们过年还回去么？"},{"topic_id":779479,"id":10560980,"content":"

美好的一天从刷到这个帖结束<\/div>","thank_count":214,"no":2,"title":"35 了，咱们程序员未来的路咋走呀"},{"topic_id":861708,"id":11804119,"content":"