├── .github ├── dependabot.yml └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── apps ├── adb │ └── main.go └── proxy │ └── main.go ├── config.example.toml ├── download ├── avatar │ └── .gitignore ├── cover │ └── .gitignore └── video │ └── .gitignore ├── example └── example.gif ├── go.mod ├── go.sum ├── internal ├── adb │ ├── adb.go │ └── start.go ├── config │ └── config.go ├── core │ └── core.go ├── database │ ├── database.go │ └── model │ │ ├── json.go │ │ └── videos.go ├── proxy │ ├── cache.go │ ├── handler.go │ └── proxy.go └── utils │ └── utils.go ├── main.go ├── mitm-proxy.crt └── static └── adb ├── AdbWinApi.dll ├── AdbWinUsbApi.dll ├── adb.exe └── fastboot.exe /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "daily" 8 | 9 | - package-ecosystem: "gomod" 10 | directory: "/" 11 | schedule: 12 | interval: "daily" 13 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | 5 | vet: 6 | name: Vet 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 1.22 11 | uses: actions/setup-go@v5 12 | with: 13 | go-version: 1.22 14 | id: go 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v4 18 | 19 | - name: Vet 20 | run: make vet 21 | 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | config.toml 3 | database.db 4 | *.exe 5 | dist 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 cnbattle 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DIST := dist 2 | EXECUTABLE := douyin 3 | GOFMT ?= gofmt "-s" 4 | GO ?= go 5 | VERSION ?= v0.2.0 6 | 7 | TARGETS ?= linux windows js 8 | ARCHS ?= amd64 386 arm arm64 mips64 mips64le mips mipsle ppc64 ppc64le riscv64 s390x wasm 9 | PACKAGES ?= $(shell $(GO) list ./...) 10 | SOURCES ?= $(shell find . -name "*.go" -type f) 11 | TAGS ?= 12 | 13 | LDFLAGS ?= -X 'main.Version=$(VERSION)' -X 'main.DroneBuildNumber=$(DRONE_BUILD_NUMBER)' -X 'main.DroneTag=$(DRONE_TAG)' 14 | 15 | #ifneq ($(shell uname), Darwin) 16 | # EXTLDFLAGS = -extldflags "-static" $(null) 17 | ## EXTLDFLAGS = -extldflags $(null) 18 | #else 19 | # EXTLDFLAGS = 20 | #endif 21 | EXTLDFLAGS = -extldflags "-static" $(null) 22 | 23 | all: build 24 | 25 | c: vet misspell-check 26 | #c: vet lint misspell-check sec 27 | 28 | fmt: 29 | $(GOFMT) -w $(SOURCES) 30 | 31 | vet: 32 | $(GO) vet $(PACKAGES) 33 | 34 | lint: 35 | @hash revive > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 36 | $(GO) get -u github.com/mgechev/revive; \ 37 | fi 38 | revive -config .revive.toml ./... || exit 1 39 | 40 | .PHONY: misspell-check 41 | misspell-check: 42 | @hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 43 | $(GO) get -u github.com/client9/misspell/cmd/misspell; \ 44 | fi 45 | misspell -error $(SOURCES) 46 | 47 | .PHONY: misspell 48 | misspell: 49 | @hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 50 | $(GO) get -u github.com/client9/misspell/cmd/misspell; \ 51 | fi 52 | misspell -w $(SOURCES) 53 | 54 | sec: 55 | @hash gosec > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 56 | $(GO) get -u github.com/securego/gosec/v2/cmd/gosec; \ 57 | fi 58 | gosec -exclude=G401,G404,G302,G304,G307,G501 ./... 59 | 60 | .PHONY: fmt-check 61 | fmt-check: 62 | @diff=$$($(GOFMT) -d $(SOURCES)); \ 63 | if [ -n "$$diff" ]; then \ 64 | echo "Please run 'make fmt' and commit the result:"; \ 65 | echo "$${diff}"; \ 66 | exit 1; \ 67 | fi; 68 | 69 | verify: vet misspell-check fmt-check 70 | 71 | test: fmt-check 72 | @$(GO) test -v -cover -coverprofile coverage.txt ./... && echo "\n==>\033[32m Ok\033[m\n" || exit 1 73 | 74 | build: 75 | $(GO) build -v -tags "$(TAGS)" -ldflags "$(EXTLDFLAGS) -s -w $(LDFLAGS)" -o api ./apps/api/cmd/api.go 76 | 77 | build-cron: 78 | $(GO) build -v -tags "$(TAGS)" -ldflags "$(EXTLDFLAGS) -s -w $(LDFLAGS)" -o cron ./apps/cron/cmd/cron.go 79 | 80 | release: release-dirs release-build release-copy release-check 81 | 82 | release-dirs: 83 | rm -rf $(DIST); mkdir -p $(DIST)/binaries $(DIST)/release 84 | 85 | release-build: 86 | @which gox > /dev/null; if [ $$? -ne 0 ]; then \ 87 | $(GO) get -u github.com/mitchellh/gox; \ 88 | fi 89 | gox -os="$(TARGETS)" -arch="$(ARCHS)" -tags="$(TAGS)" -ldflags="-s -w $(LDFLAGS)" -output="$(DIST)/binaries/$(EXECUTABLE)-$(VERSION)-{{.OS}}-{{.Arch}}" 90 | 91 | release-copy: 92 | $(foreach file,$(wildcard $(DIST)/binaries/$(EXECUTABLE)-*),cp $(file) $(DIST)/release/$(notdir $(file));) 93 | 94 | release-check: 95 | cd $(DIST)/release; $(foreach file,$(wildcard $(DIST)/release/$(EXECUTABLE)-*),sha256sum $(notdir $(file)) > $(notdir $(file)).sha256;) 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 抖音推荐/搜索页视频列表视频爬虫方案 2 | 3 | > 最近测试可用时间：`2024.06.20` 4 | 5 | > adb暂未实现进入搜索页的操作，请根据自身技术栈实现相关点击操作及键入关键词等 6 | 7 | > 老版本请切换到`old`分支查看，`old`分支使用anyproxy抓取，更适合大多数人使用 8 | 9 | 基于APP爬取 10 | 11 | 技术栈：`golang` `adb` 12 | 13 | ![](example/example.gif) 14 | 15 | ## 特点 16 | - 可设置仅抓取大于xx赞的视频 17 | - 可自定义设置是否下载远程文件到本地 18 | 19 | ## 使用 20 | 21 | 1. 安装`mitm-proxy.crt`证书到相关设备，`虚拟机或真机` 22 | 23 | 2. 复制 `config.example.toml` 为 `config.toml`,并根据自己需求修改参数 24 | 25 | 3. 运行本项目程序 `go run main.go` 或 编辑运行 26 | 27 | 4. 若开启下载会生成一个 `database.db`的sqlite3数据库文件,字符详见`model/videos.go`文件,静态文件(用户头像,视频封面图,视频文件)将放在`download/[avatar,cover,video]`目录下 28 | 29 | 5. 最后根据个人需要上传处理即可 30 | 31 | ## 待优化的地方 32 | 33 | - 一直滑动请求推荐列表,APP会异常退出,所以暂时的方法是每爬取一段时间(配置文件里可自定义)后,强制关闭APP,重新打开继续爬取 34 | 35 | - 目前个人在用的`雷电模拟器`,长时间运行会卡死,暂不清楚是系统原因还是模拟器原因,有测试能稳定运行环境的朋友,方便的话请告诉我,谢谢 36 | 37 | ## 最后说明 38 | 39 | - 个人能力一般,有很多编码不规范的地方请包涵 40 | - 有能力的朋友可以根据个人需求修改,如果可以请提交pr 41 | - 如果使用有问题,请提交`issues` 或加我同名微信号,请备注github过来的,谢谢 42 | 43 | ## Stargazers over time 44 | 45 | [![Stargazers over time](https://starchart.cc/cnbattle/douyin.svg)](https://starchart.cc/cnbattle/douyin) 46 | 47 | 48 | ## Thanks 49 | 50 | ``` 51 | gorm.io/gorm 52 | github.com/ouqiang/goproxy 53 | github.com/spf13/viper 54 | ``` 55 | ## Development Tool (IDE) For Jetbrains 56 | 57 | Thanks for [Jetbrains Open Source Licenses](https://www.jetbrains.com/community/opensource/#support). 58 | 59 | ## 声明 60 | 61 | 本项目只做个人学习研究之用，不得用于商业用途！ 62 | -------------------------------------------------------------------------------- /apps/adb/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cnbattle/douyin/internal/adb" 4 | 5 | func main() { 6 | adb.Start() 7 | } 8 | -------------------------------------------------------------------------------- /apps/proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/cnbattle/douyin/internal/proxy" 5 | ) 6 | 7 | func main() { 8 | proxy.Start() 9 | } 10 | -------------------------------------------------------------------------------- /config.example.toml: -------------------------------------------------------------------------------- 1 | smallLike = 1000 # 抓取最小视频赞数,仅抓取大于此值的视频 2 | isDownload = 1 # 是否下载远程 用户头像 封面图 视频 到本地 0否1是 3 | 4 | [proxy] 5 | addr = ":8080" #设置 proxy listen 端口 6 | 7 | [app] 8 | packageName = "com.ss.android.ugc.aweme" # 抖音app包名 9 | startPath = ".main.MainActivity" # 抖音app包首页path 10 | restart = 60 # 重新打开时间 单位秒 11 | sleep = 60 # 重新打开睡眠时间 单位秒 12 | 13 | [click] 14 | # 请根据自己的设备调整 15 | # 菜单首页的像素坐标值,用于双击重新请求推荐列表数据 16 | X = 20 17 | Y = 1200 18 | sleep = 5 # 双击后等待时间下次双击的时间 单位毫秒 19 | -------------------------------------------------------------------------------- /download/avatar/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /download/cover/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /download/video/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /example/example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cnbattle/douyin/efa48a8839b842df17657a358634d4af7f75bcc6/example/example.gif -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cnbattle/douyin 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | github.com/glebarez/sqlite v1.11.0 9 | github.com/ouqiang/goproxy v1.3.2 10 | github.com/spf13/viper v1.20.1 11 | gorm.io/gorm v1.26.1 12 | ) 13 | 14 | require ( 15 | github.com/dustin/go-humanize v1.0.1 // indirect 16 | github.com/fsnotify/fsnotify v1.8.0 // indirect 17 | github.com/glebarez/go-sqlite v1.22.0 // indirect 18 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 19 | github.com/google/uuid v1.6.0 // indirect 20 | github.com/jinzhu/inflection v1.0.0 // indirect 21 | github.com/jinzhu/now v1.1.5 // indirect 22 | github.com/mattn/go-isatty v0.0.20 // indirect 23 | github.com/ncruces/go-strftime v0.1.9 // indirect 24 | github.com/ouqiang/websocket v1.6.2 // indirect 25 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 26 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 27 | github.com/sagikazarmark/locafero v0.7.0 // indirect 28 | github.com/sourcegraph/conc v0.3.0 // indirect 29 | github.com/spf13/afero v1.12.0 // indirect 30 | github.com/spf13/cast v1.7.1 // indirect 31 | github.com/spf13/pflag v1.0.6 // indirect 32 | github.com/subosito/gotenv v1.6.0 // indirect 33 | github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 // indirect 34 | go.uber.org/multierr v1.11.0 // indirect 35 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect 36 | golang.org/x/sys v0.29.0 // indirect 37 | golang.org/x/text v0.21.0 // indirect 38 | gopkg.in/yaml.v3 v3.0.1 // indirect 39 | modernc.org/libc v1.61.0 // indirect 40 | modernc.org/mathutil v1.6.0 // indirect 41 | modernc.org/memory v1.8.0 // indirect 42 | modernc.org/sqlite v1.33.1 // indirect 43 | ) 44 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 4 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 5 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 6 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 7 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 8 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 9 | github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= 10 | github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= 11 | github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= 12 | github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= 13 | github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= 14 | github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= 15 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 16 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 17 | github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= 18 | github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= 19 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 20 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 21 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 22 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 23 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 24 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 25 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 26 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 27 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 28 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 29 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 30 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 31 | github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= 32 | github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= 33 | github.com/ouqiang/goproxy v1.3.2 h1:+3uBRrM0RU4LFcsH0lbWsdUCoHIzoRxk+ISPbIS3lTk= 34 | github.com/ouqiang/goproxy v1.3.2/go.mod h1:yF0a+DlUi0Zff28iUeuqLov90bivevUX9uOn3Yk9rww= 35 | github.com/ouqiang/websocket v1.6.2 h1:LGQIySbQO3ahZCl34v9xBVb0yncDk8yIcuEIbWBab/U= 36 | github.com/ouqiang/websocket v1.6.2/go.mod h1:fIROJIHRlQwgCyUFTMzaaIcs4HIwUj2xlOW43u9Sf+M= 37 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 38 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 39 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 40 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 41 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= 42 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 43 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 44 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 45 | github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= 46 | github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= 47 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 48 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 49 | github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= 50 | github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= 51 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 52 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 53 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 54 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 55 | github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= 56 | github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= 57 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 58 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 59 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 60 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 61 | github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ= 62 | github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= 63 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 64 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 65 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= 66 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= 67 | golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= 68 | golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 69 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 70 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 71 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 72 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 73 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 74 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 75 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 76 | golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= 77 | golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= 78 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 79 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 80 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 81 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 82 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 83 | gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw= 84 | gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= 85 | modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= 86 | modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= 87 | modernc.org/ccgo/v4 v4.21.0 h1:kKPI3dF7RIag8YcToh5ZwDcVMIv6VGa0ED5cvh0LMW4= 88 | modernc.org/ccgo/v4 v4.21.0/go.mod h1:h6kt6H/A2+ew/3MW/p6KEoQmrq/i3pr0J/SiwiaF/g0= 89 | modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= 90 | modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= 91 | modernc.org/gc/v2 v2.5.0 h1:bJ9ChznK1L1mUtAQtxi0wi5AtAs5jQuw4PrPHO5pb6M= 92 | modernc.org/gc/v2 v2.5.0/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= 93 | modernc.org/libc v1.61.0 h1:eGFcvWpqlnoGwzZeZe3PWJkkKbM/3SUGyk1DVZQ0TpE= 94 | modernc.org/libc v1.61.0/go.mod h1:DvxVX89wtGTu+r72MLGhygpfi3aUGgZRdAYGCAVVud0= 95 | modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= 96 | modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= 97 | modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= 98 | modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= 99 | modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= 100 | modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= 101 | modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= 102 | modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= 103 | modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= 104 | modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= 105 | modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= 106 | modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= 107 | modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= 108 | modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= 109 | -------------------------------------------------------------------------------- /internal/adb/adb.go: -------------------------------------------------------------------------------- 1 | package adb 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "runtime" 7 | ) 8 | 9 | // Command 自定义参数 10 | func Command(arg ...string) { 11 | cmd := exec.Command(getAdbCli(), arg...) 12 | cmd.Stdout = os.Stdout 13 | _ = cmd.Run() 14 | } 15 | 16 | // RunApp 运行App 17 | func RunApp(startAppPath string) { 18 | cmd := exec.Command(getAdbCli(), "shell", "am", "start", "-n", startAppPath) 19 | cmd.Stdout = os.Stdout 20 | _ = cmd.Run() 21 | } 22 | 23 | // CloseApp 关闭 24 | func CloseApp(packageName string) { 25 | cmd := exec.Command(getAdbCli(), "shell", "am", "force-stop", packageName) 26 | cmd.Stdout = os.Stdout 27 | _ = cmd.Run() 28 | } 29 | 30 | // Swipe 滑动 31 | func Swipe(StartX, StartY, EndX, EndY string) { 32 | cmd := exec.Command(getAdbCli(), "shell", "input", "swipe", 33 | StartX, StartY, EndX, EndY, 34 | ) 35 | cmd.Stdout = os.Stdout 36 | _ = cmd.Run() 37 | } 38 | 39 | // InputText 输入文本 40 | func InputText(text string) { 41 | cmd := exec.Command(getAdbCli(), "shell", "input", "text", text) 42 | cmd.Stdout = os.Stdout 43 | _ = cmd.Run() 44 | } 45 | 46 | // InputKeyEvent 输入KeyEvent 47 | func InputKeyEvent(text string) { 48 | cmd := exec.Command(getAdbCli(), "shell", "input", "keyevent", text) 49 | cmd.Stdout = os.Stdout 50 | _ = cmd.Run() 51 | } 52 | 53 | // InputTextByADBKeyBoard 输入文本 54 | func InputTextByADBKeyBoard(text string) { 55 | cmd := exec.Command(getAdbCli(), "shell", "am", "broadcast", "-a", "ADB_INPUT_TEXT", "--es", "msg", text) 56 | cmd.Stdout = os.Stdout 57 | _ = cmd.Run() 58 | } 59 | 60 | // Click 点击某一像素点 61 | func Click(X, Y string) { 62 | cmd := exec.Command(getAdbCli(), "shell", "input", "tap", X, Y) 63 | cmd.Stdout = os.Stdout 64 | _ = cmd.Run() 65 | } 66 | 67 | // ClickKeyCode 点击android对应的keycode 68 | func ClickKeyCode(code string) { 69 | cmd := exec.Command(getAdbCli(), "shell", "input", "keyevent", code) 70 | cmd.Stdout = os.Stdout 71 | _ = cmd.Run() 72 | } 73 | 74 | // ClickHome 点击hone键 75 | func ClickHome() { 76 | ClickKeyCode("3") 77 | } 78 | 79 | // ClickHome 点击返回键 80 | func ClickBack() { 81 | ClickKeyCode("4") 82 | } 83 | 84 | // getAdbCli 获取adb cli 85 | func getAdbCli() string { 86 | if runtime.GOOS == "windows" { 87 | return "./static/adb/adb.exe" 88 | } 89 | return "adb" 90 | } 91 | -------------------------------------------------------------------------------- /internal/adb/start.go: -------------------------------------------------------------------------------- 1 | package adb 2 | 3 | import ( 4 | "github.com/cnbattle/douyin/internal/config" 5 | "log" 6 | "time" 7 | ) 8 | 9 | func init() { 10 | log.Println("[ADB] start") 11 | } 12 | 13 | func Start() { 14 | START: 15 | start := time.Now().Unix() 16 | CloseApp(config.V.GetString("app.packageName")) 17 | RunApp(config.V.GetString("app.packageName") + "/" + config.V.GetString("app.startPath")) 18 | time.Sleep(10 * time.Second) 19 | for { 20 | now := time.Now().Unix() 21 | if now > start+config.V.GetInt64("app.restart") { 22 | time.Sleep(config.V.GetDuration("app.sleep") * time.Second) 23 | goto START 24 | } 25 | Click(config.V.GetString("click.x"), config.V.GetString("click.y")) 26 | time.Sleep(300 * time.Millisecond) 27 | Click(config.V.GetString("click.x"), config.V.GetString("click.y")) 28 | 29 | time.Sleep(config.V.GetDuration("click.sleep") * time.Second) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/spf13/viper" 5 | ) 6 | 7 | var V *viper.Viper 8 | 9 | func init() { 10 | V = viper.New() 11 | V.SetConfigName("config") 12 | V.AddConfigPath("./") 13 | V.SetConfigType("toml") 14 | if err := V.ReadInConfig(); err != nil { 15 | panic(err) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /internal/core/core.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strconv" 9 | 10 | "github.com/cnbattle/douyin/internal/config" 11 | "github.com/cnbattle/douyin/internal/database" 12 | "github.com/cnbattle/douyin/internal/database/model" 13 | "github.com/cnbattle/douyin/internal/utils" 14 | ) 15 | 16 | func HandleJson(data model.Data) { 17 | for _, item := range data.AwemeList { 18 | // 判断是否是广告 点赞数是否大于设定值 19 | if item.IsAds == true || item.Statistics.DiggCount < config.V.GetInt("smallLike") { 20 | log.Println("数据:", item.Desc, "continue") 21 | continue 22 | } 23 | log.Println("开始处理数据:", item.Desc) 24 | 25 | isDownload := config.V.GetInt("isDownload") 26 | var localAvatar, localCover, localVideo string 27 | var err error 28 | coverUrl, videoUrl := getCoverVideo(&item) 29 | if isDownload == 1 { 30 | // 下载封面图 视频 头像图 31 | localAvatar, localCover, localVideo, err = downloadHttpFile(item.Author.AvatarThumb.UrlList[0], videoUrl, coverUrl) 32 | if err != nil { 33 | log.Println("下载封面图 视频 头像图失败:", err) 34 | continue 35 | } 36 | } else { 37 | localAvatar, localCover, localVideo = item.Author.AvatarThumb.UrlList[0], coverUrl, videoUrl 38 | } 39 | // 写入数据库 40 | var video model.Video 41 | video.AwemeId = item.AwemeId 42 | video.AuthorId = item.Author.Uid 43 | video.Nickname = item.Author.Nickname 44 | video.Avatar = localAvatar 45 | video.Desc = item.Desc 46 | video.DiggCount = strconv.Itoa(item.Statistics.DiggCount) 47 | video.CommentCount = strconv.Itoa(item.Statistics.CommentCount) 48 | video.ShareUrl = item.ShareInfo.ShareUrl 49 | video.CoverPath = localCover 50 | video.VideoPath = localVideo 51 | video.IsDownload = isDownload 52 | database.Local.Create(&video) 53 | } 54 | } 55 | 56 | // downloadHttpFile 下载远程图片 57 | func downloadHttpFile(avatarUrl, videoUrl string, coverUrl string) (string, string, string, error) { 58 | var localAvatar, localCover, localVideo string 59 | localAvatar = "download/avatar/" + utils.Md5(avatarUrl) + ".jpeg" 60 | localVideo = "download/video/" + utils.Md5(videoUrl) + ".mp4" 61 | localCover = "download/cover/" + utils.Md5(coverUrl) + ".jpeg" 62 | err := download(avatarUrl, localAvatar) 63 | if err != nil { 64 | return "", "", "", err 65 | } 66 | err = download(videoUrl, localVideo) 67 | if err != nil { 68 | return "", "", "", err 69 | } 70 | err = download(coverUrl, localCover) 71 | if err != nil { 72 | return "", "", "", err 73 | } 74 | return localAvatar, localCover, localVideo, nil 75 | } 76 | 77 | // getCoverVideo 获取封面图视频地址 78 | func getCoverVideo(item *model.Item) (coverUrl, videoUrl string) { 79 | isDownload := config.V.GetInt("isDownload") 80 | coverUrl = item.Video.Cover.UrlList[0] 81 | if isDownload == 1 { 82 | videoUrl = item.Video.PlayAddr.UrlList[0] 83 | return 84 | } 85 | videoUrl = item.Video.PlayAddr.UrlList[len(item.Video.PlayAddr.UrlList)-1] 86 | return 87 | } 88 | 89 | // download 下载文件 90 | func download(url, saveFile string) error { 91 | client := &http.Client{} 92 | req, err := http.NewRequest("GET", url, nil) 93 | req.Header.Add("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1") 94 | req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3") 95 | req.Header.Add("Accept-Encoding", "gzip, deflate, br") 96 | req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") 97 | res, err := client.Do(req) 98 | if err != nil { 99 | return err 100 | } 101 | defer res.Body.Close() 102 | f, err := os.Create(saveFile) 103 | defer f.Close() 104 | if err != nil { 105 | _ = os.Remove(saveFile) 106 | return err 107 | } 108 | _, err = io.Copy(f, res.Body) 109 | if err != nil { 110 | _ = os.Remove(saveFile) 111 | return err 112 | } 113 | return nil 114 | } 115 | -------------------------------------------------------------------------------- /internal/database/database.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "github.com/cnbattle/douyin/internal/database/model" 5 | // "gorm.io/driver/sqlite" 6 | "log" 7 | 8 | "github.com/glebarez/sqlite" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | var ( 13 | Local *gorm.DB 14 | localDialect = "sqlite3" 15 | localArgs = "./database.db" 16 | ) 17 | 18 | func init() { 19 | var err error 20 | Local, err = gorm.Open(sqlite.Open(localArgs), &gorm.Config{}) 21 | if err != nil { 22 | log.Panic(err) 23 | } 24 | Local.AutoMigrate(&model.Video{}) 25 | } 26 | -------------------------------------------------------------------------------- /internal/database/model/json.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Data struct { 4 | StatusCode int `json:"status_code"` 5 | MinCursor int `json:"min_cursor"` 6 | MaxCursor int `json:"max_cursor"` 7 | HasMore int `json:"has_more"` 8 | AwemeList []Item `json:"aweme_list"` 9 | } 10 | 11 | type Item struct { 12 | AwemeId string `json:"aweme_id"` 13 | Desc string `json:"desc"` 14 | CreateTime int `json:"create_time"` 15 | Author author `json:"author"` 16 | Video video `json:"video"` 17 | IsAds bool `json:"is_ads"` 18 | Duration int `json:"duration"` 19 | GroupId string `json:"group_id"` 20 | AuthorUserId int64 `json:"author_user_id"` 21 | LongVideo []longVideo `json:"long_video"` 22 | Statistics statistics `json:"statistics"` 23 | ShareInfo shareInfo `json:"share_info"` 24 | } 25 | 26 | type author struct { 27 | Uid string `json:"uid"` 28 | Nickname string `json:"nickname"` 29 | Gender int `json:"gender"` 30 | UniqueId string `json:"unique_id"` 31 | AvatarThumb uriStr `json:"avatar_thumb"` 32 | } 33 | 34 | type video struct { 35 | PlayAddr uriStr `json:"play_addr"` 36 | Cover uriStr `json:"cover"` 37 | } 38 | 39 | type longVideo struct { 40 | Video video `json:"video"` 41 | TrailerStartTime int `json:"trailer_start_time"` 42 | } 43 | 44 | type uriStr struct { 45 | Uri string `json:"uri"` 46 | UrlList []string `json:"url_list"` 47 | Width int `json:"width"` 48 | Height int `json:"height"` 49 | } 50 | 51 | type statistics struct { 52 | AwemeId string `json:"aweme_id"` 53 | CommentCount int `json:"comment_count"` 54 | DiggCount int `json:"digg_count"` 55 | DownloadCount int `json:"download_count"` 56 | PlayCount int `json:"play_count"` 57 | ShareCount int `json:"share_count"` 58 | ForwardCount int `json:"forward_count"` 59 | LoseCount int `json:"lose_count"` 60 | LoseComment_count int `json:"lose_comment_count"` 61 | } 62 | 63 | type shareInfo struct { 64 | ShareUrl string `json:"share_url"` 65 | ShareTitle string `json:"share_title"` 66 | } 67 | -------------------------------------------------------------------------------- /internal/database/model/videos.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "gorm.io/gorm" 4 | 5 | type Video struct { 6 | gorm.Model 7 | AwemeId string `gorm:"unique;not null"` 8 | AuthorId string `gorm:"varchar(64)"` 9 | Nickname string `gorm:"varchar(64)"` 10 | Avatar string `gorm:"varchar(64)"` 11 | Desc string `gorm:"varchar(255)"` 12 | DiggCount string `gorm:"varchar(32)"` 13 | CommentCount string `gorm:"varchar(32)"` 14 | CoverPath string `gorm:"varchar(255)"` 15 | VideoPath string `gorm:"varchar(255)"` 16 | ShareUrl string `gorm:"varchar(255)"` 17 | IsDownload int `gorm:"default(0);type:tinyint(1)"` 18 | Status int `gorm:"default(0);type:tinyint(1)"` 19 | } 20 | -------------------------------------------------------------------------------- /internal/proxy/cache.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "crypto/tls" 5 | "sync" 6 | ) 7 | 8 | // 实现证书缓存接口 9 | type Cache struct { 10 | m sync.Map 11 | } 12 | 13 | func (c *Cache) Set(host string, cert *tls.Certificate) { 14 | c.m.Store(host, cert) 15 | } 16 | func (c *Cache) Get(host string) *tls.Certificate { 17 | v, ok := c.m.Load(host) 18 | if !ok { 19 | return nil 20 | } 21 | return v.(*tls.Certificate) 22 | } 23 | -------------------------------------------------------------------------------- /internal/proxy/handler.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "github.com/ouqiang/goproxy" 7 | "io/ioutil" 8 | "log" 9 | "net" 10 | "net/http" 11 | "net/url" 12 | "strings" 13 | 14 | "github.com/cnbattle/douyin/internal/core" 15 | "github.com/cnbattle/douyin/internal/database/model" 16 | "github.com/cnbattle/douyin/internal/utils" 17 | ) 18 | 19 | type EventHandler struct{} 20 | 21 | func (e *EventHandler) WebSocketSendMessage(*goproxy.Context, *int, *[]byte) {} 22 | 23 | func (e *EventHandler) WebSocketReceiveMessage(*goproxy.Context, *int, *[]byte) {} 24 | 25 | func (e *EventHandler) Connect(ctx *goproxy.Context, rw http.ResponseWriter) { 26 | // 保存的数据可以在后面的回调方法中获取 27 | ctx.Data["req_id"] = "uuid" 28 | // 禁止访问某个域名 29 | if strings.Contains(ctx.Req.URL.Host, "example.com") { 30 | rw.WriteHeader(http.StatusForbidden) 31 | ctx.Abort() 32 | return 33 | } 34 | } 35 | 36 | func (e *EventHandler) Auth(*goproxy.Context, http.ResponseWriter) {} 37 | 38 | func (e *EventHandler) BeforeRequest(ctx *goproxy.Context) { 39 | // 修改header 40 | ctx.Req.Header.Add("X-Request-Id", ctx.Data["req_id"].(string)) 41 | // 设置X-Forwarded-For 42 | if clientIP, _, err := net.SplitHostPort(ctx.Req.RemoteAddr); err == nil { 43 | if prior, ok := ctx.Req.Header["X-Forwarded-For"]; ok { 44 | clientIP = strings.Join(prior, ", ") + ", " + clientIP 45 | } 46 | ctx.Req.Header.Set("X-Forwarded-For", clientIP) 47 | } 48 | // 读取Body 49 | body, err := ioutil.ReadAll(ctx.Req.Body) 50 | if err != nil { 51 | // 错误处理 52 | return 53 | } 54 | // Request.Body只能读取一次, 读取后必须再放回去 55 | // Response.Body同理 56 | ctx.Req.Body = ioutil.NopCloser(bytes.NewReader(body)) 57 | } 58 | 59 | func (e *EventHandler) BeforeResponse(ctx *goproxy.Context, resp *http.Response, err error) { 60 | if err != nil { 61 | return 62 | } 63 | // /aweme/v1/general/search/single/ 综合搜索 64 | // /aweme/v1/search/item/ 视频 65 | //if strings.EqualFold(ctx.Req.URL.Path, "/aweme/v1/general/search/single/") { 66 | // response, err := ioutil.ReadAll(resp.Body) 67 | // if err != nil { 68 | // log.Println(err) 69 | // return 70 | // } 71 | // // gzip 72 | // body, err := utils.ParseGzip(response) 73 | // if err != nil { 74 | // log.Println(err) 75 | // return 76 | // } 77 | // var filename = "./single.json" 78 | // var f *os.File 79 | // /***************************** 第一种方式: 使用 io.WriteString 写入文件 ***********************************************/ 80 | // if utils.CheckFileIsExist(filename) { //如果文件存在 81 | // f, _ = os.OpenFile(filename, os.O_APPEND, 0666) //打开文件 82 | // fmt.Println("文件存在") 83 | // } else { 84 | // f, _ = os.Create(filename) //创建文件 85 | // fmt.Println("文件不存在") 86 | // } 87 | // _, _ = io.WriteString(f, string(body)) //写入文件(字符串) 88 | // //var data model.Data 89 | // //err = json.Unmarshal(body, &data) 90 | // //if err != nil { 91 | // // log.Println(err) 92 | // // return 93 | // //} 94 | // //go core.HandleJson(data) 95 | // // resp.Body 只能读取一次, 读取后必须再放回去 96 | // resp.Body = ioutil.NopCloser(bytes.NewReader(response)) 97 | //} 98 | 99 | // 处理 推荐列表接口 搜索页视频列表接口 100 | if strings.EqualFold(ctx.Req.URL.Path, "/aweme/v1/feed/") || strings.EqualFold(ctx.Req.URL.Path, "/aweme/v1/search/item/") { 101 | response, err := ioutil.ReadAll(resp.Body) 102 | if err != nil { 103 | log.Println(err) 104 | return 105 | } 106 | // gzip 107 | body, err := utils.ParseGzip(response) 108 | if err != nil { 109 | log.Println(err) 110 | return 111 | } 112 | var data model.Data 113 | err = json.Unmarshal(body, &data) 114 | if err != nil { 115 | log.Println(err) 116 | return 117 | } 118 | go core.HandleJson(data) 119 | // resp.Body 只能读取一次, 读取后必须再放回去 120 | resp.Body = ioutil.NopCloser(bytes.NewReader(response)) 121 | } 122 | } 123 | 124 | // ParentProxy 设置上级代理 125 | func (e *EventHandler) ParentProxy(*http.Request) (*url.URL, error) { 126 | return nil, nil 127 | } 128 | 129 | // Finish 请求结束 130 | func (e *EventHandler) Finish(*goproxy.Context) {} 131 | 132 | // ErrorLog 记录错误日志 133 | func (e *EventHandler) ErrorLog(error) {} 134 | -------------------------------------------------------------------------------- /internal/proxy/proxy.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/cnbattle/douyin/internal/config" 9 | 10 | "github.com/ouqiang/goproxy" 11 | ) 12 | 13 | var addr = ":8080" 14 | 15 | func init() { 16 | addr = config.V.GetString("proxy.addr") 17 | log.Println("[Proxy] Listen:", addr) 18 | } 19 | 20 | func Start() { 21 | proxy := goproxy.New(goproxy.WithDelegate(&EventHandler{}), goproxy.WithDecryptHTTPS(&Cache{})) 22 | server := &http.Server{ 23 | Addr: addr, 24 | Handler: proxy, 25 | ReadTimeout: 1 * time.Minute, 26 | WriteTimeout: 1 * time.Minute, 27 | } 28 | err := server.ListenAndServe() 29 | if err != nil { 30 | panic(err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/md5" 7 | "encoding/binary" 8 | "encoding/hex" 9 | "io/ioutil" 10 | "os" 11 | ) 12 | 13 | // Md5 字符串加密 14 | func Md5(str string) string { 15 | h := md5.New() 16 | h.Write([]byte(str)) 17 | return hex.EncodeToString(h.Sum(nil)) 18 | } 19 | 20 | func CheckFileIsExist(filename string) bool { 21 | var exist = true 22 | if _, err := os.Stat(filename); os.IsNotExist(err) { 23 | exist = false 24 | } 25 | return exist 26 | } 27 | 28 | // ParseGzip gzip 解压 29 | func ParseGzip(data []byte) ([]byte, error) { 30 | b := new(bytes.Buffer) 31 | binary.Write(b, binary.LittleEndian, data) 32 | r, err := gzip.NewReader(b) 33 | if err != nil { 34 | return nil, err 35 | } else { 36 | defer r.Close() 37 | undatas, err := ioutil.ReadAll(r) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return undatas, nil 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/cnbattle/douyin/internal/adb" 5 | "github.com/cnbattle/douyin/internal/proxy" 6 | ) 7 | 8 | func main() { 9 | go proxy.Start() 10 | go adb.Start() 11 | select {} 12 | } 13 | -------------------------------------------------------------------------------- /mitm-proxy.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICJzCCAcygAwIBAgIITWWCIQf8/VIwCgYIKoZIzj0EAwIwUzEOMAwGA1UEBhMF 3 | Q2hpbmExDzANBgNVBAgTBkZ1SmlhbjEPMA0GA1UEBxMGWGlhbWVuMRAwDgYDVQQK 4 | EwdHb3Byb3h5MQ0wCwYDVQQDEwRNYXJzMB4XDTIyMDMyNTA1NDgwMFoXDTQyMDQy 5 | NTA1NDgwMFowUzEOMAwGA1UEBhMFQ2hpbmExDzANBgNVBAgTBkZ1SmlhbjEPMA0G 6 | A1UEBxMGWGlhbWVuMRAwDgYDVQQKEwdHb3Byb3h5MQ0wCwYDVQQDEwRNYXJzMFkw 7 | EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf0mhVJmuTmxnLimKshdEE4+PYdxvBfQX 8 | mRgsFV5KHHmxOrVJBFC/nDetmGowkARShWtBsX1Irm4w6i6Qk2QliKOBiTCBhjAO 9 | BgNVHQ8BAf8EBAMCAQYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIG 10 | A1UdEwEB/wQIMAYBAf8CAQIwHQYDVR0OBBYEFBI5TkWYcvUIWsBAdffs833FnBrI 11 | MCIGA1UdEQQbMBmBF3FpbmdxaWFubHVkYW9AZ21haWwuY29tMAoGCCqGSM49BAMC 12 | A0kAMEYCIQCk1DhW7AmIW/n/QLftQq8BHZKLevWYJ813zdrNr5kXlwIhAIVvqglY 13 | 9BkYWg4NEe/mVO4C5Vtu4FnzNU9I+rFpXVSO 14 | -----END CERTIFICATE----- 15 | -------------------------------------------------------------------------------- /static/adb/AdbWinApi.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cnbattle/douyin/efa48a8839b842df17657a358634d4af7f75bcc6/static/adb/AdbWinApi.dll -------------------------------------------------------------------------------- /static/adb/AdbWinUsbApi.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cnbattle/douyin/efa48a8839b842df17657a358634d4af7f75bcc6/static/adb/AdbWinUsbApi.dll -------------------------------------------------------------------------------- /static/adb/adb.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cnbattle/douyin/efa48a8839b842df17657a358634d4af7f75bcc6/static/adb/adb.exe -------------------------------------------------------------------------------- /static/adb/fastboot.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cnbattle/douyin/efa48a8839b842df17657a358634d4af7f75bcc6/static/adb/fastboot.exe --------------------------------------------------------------------------------