├── codecov.yml ├── static ├── logo.png └── jetbrains-variant-3.svg ├── config ├── version.go └── config.go ├── compress.bat ├── compress.sh ├── extractors ├── types │ ├── defs.go │ └── types.go ├── pornhub │ ├── pornhub_test.go │ └── pornhub.go ├── douyu │ ├── douyu_test.go │ └── douyu.go ├── yinyuetai │ ├── yinyuetai_test.go │ ├── types.go │ └── yinyuetai.go ├── udn │ ├── udn_test.go │ └── udn.go ├── pixivision │ ├── pixivision_test.go │ └── pixivision.go ├── youku │ ├── youku_test.go │ └── youku.go ├── bcy │ ├── bcy_test.go │ └── bcy.go ├── geekbang │ ├── geekbang_test.go │ └── geekbang.go ├── miaopai │ ├── miaopai_test.go │ └── miaopai.go ├── douyin │ ├── douyin_test.go │ └── douyin.go ├── universal │ ├── universal_test.go │ └── universal.go ├── facebook │ ├── facebook_test.go │ └── facebook.go ├── xvideos │ ├── xvideos_test.go │ └── xvideos.go ├── vimeo │ ├── vimeo_test.go │ └── vimeo.go ├── tiktok │ ├── tiktok_test.go │ └── tiktok.go ├── netease │ ├── netease_test.go │ └── netease.go ├── mgtv │ ├── mgtv_test.go │ └── mgtv.go ├── tumblr │ ├── tumblr_test.go │ └── tumblr.go ├── instagram │ ├── instagram_test.go │ └── instagram.go ├── qq │ ├── qq_test.go │ └── qq.go ├── twitter │ ├── twitter_test.go │ └── twitter.go ├── iqiyi │ ├── iqiyi_test.go │ └── iqiyi.go ├── weibo │ ├── weibo_test.go │ └── weibo.go ├── tangdou │ ├── tangdou_test.go │ └── tangdou.go ├── bilibili │ ├── types.go │ ├── bilibili_test.go │ └── bilibili.go ├── youtube │ ├── youtube_test.go │ └── youtube.go └── extractors.go ├── .golangci.yml ├── .travis.yml ├── .gitignore ├── go.test.sh ├── utils ├── pool_test.go ├── download.go ├── pool.go ├── ffmpeg.go ├── download_test.go ├── ffmpeg_bk.go ├── utils.go └── utils_test.go ├── go.mod ├── downloader ├── types.go ├── utils.go └── downloader_test.go ├── CONTRIBUTING.md ├── LICENSE ├── .goreleaser.yml ├── parser ├── parser.go └── parser_test.go ├── test └── utils.go ├── request ├── request_test.go └── request.go ├── go.sum └── main.go /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | token: e0f2d44f-c6a7-469a-a688-37c72c0f18f9 3 | -------------------------------------------------------------------------------- /static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hubs/annie/master/static/logo.png -------------------------------------------------------------------------------- /config/version.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // VERSION version of annie 4 | const VERSION = "0.9.8" 5 | -------------------------------------------------------------------------------- /compress.bat: -------------------------------------------------------------------------------- 1 | :: Please install upx first, https://github.com/upx/upx/releases 2 | for /f "delims=" %%i in ('dir /b /a-d /s "annie*"') do upx --best "%%i" 3 | -------------------------------------------------------------------------------- /compress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Please install upx first, https://github.com/upx/upx/releases 3 | find ./ -xdev -maxdepth 1 -type f -iname 'annie*' -executable -exec upx --best --brute --ultra-brute {} \; 4 | -------------------------------------------------------------------------------- /extractors/types/defs.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | // ErrURLParseFailed defines url parse failed error. 9 | ErrURLParseFailed = errors.New("url parse failed") 10 | // ErrLoginRequired defines login required error. 11 | ErrLoginRequired = errors.New("login required") 12 | ) 13 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | concurrency: 2 3 | timeout: 5m 4 | 5 | linter-settings: 6 | goconst: 7 | min-len: 2 8 | min-occurrences: 2 9 | 10 | linters: 11 | enable: 12 | - golint 13 | - goconst 14 | - gofmt 15 | - goimports 16 | - misspell 17 | - unparam 18 | 19 | issues: 20 | exclude-use-default: false 21 | exclude-rules: 22 | - path: _test.go 23 | linters: 24 | - errcheck 25 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // FakeHeaders fake http headers 4 | var FakeHeaders = map[string]string{ 5 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 6 | "Accept-Charset": "UTF-8,*;q=0.5", 7 | "Accept-Encoding": "gzip,deflate,sdch", 8 | "Accept-Language": "en-US,en;q=0.8", 9 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 10 | } 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | matrix: 3 | fast_finish: true 4 | 5 | language: go 6 | go: 7 | - "1.13.x" 8 | 9 | before_install: 10 | - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.24.0 11 | 12 | script: 13 | - golangci-lint run 14 | - ./go.test.sh 15 | 16 | after_success: 17 | - bash <(curl -s https://codecov.io/bash) 18 | 19 | branches: 20 | only: 21 | - master 22 | 23 | notifications: 24 | email: false 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | coverage.txt 14 | 15 | # Python 16 | *.pyc 17 | 18 | .vscode 19 | .idea 20 | dist/ 21 | *_token 22 | downloader/*.jpg 23 | 24 | # Ignore compiled binaries 25 | # - native 26 | annie 27 | # - gox builds 28 | annie_* 29 | 30 | # macOS 31 | .DS_Store 32 | -------------------------------------------------------------------------------- /go.test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | #set -x # debug/verbose execution 5 | 6 | # Ensures removal of file 7 | trap 'rm -f profile.out downloader/*.{mp4,download,jpg}' SIGINT EXIT 8 | 9 | # go -race needs CGO_ENABLED to proceed 10 | export CGO_ENABLED=1; 11 | 12 | for d in $(go list ./... | grep -v vendor); do 13 | go test -v -race -coverprofile=profile.out -covermode=atomic "${d}" 14 | if [ -f profile.out ]; then 15 | cat profile.out > coverage.txt 16 | rm profile.out 17 | fi 18 | done 19 | -------------------------------------------------------------------------------- /utils/pool_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "sync/atomic" 5 | "testing" 6 | ) 7 | 8 | func TestWaitGroupPool(t *testing.T) { 9 | wgp := NewWaitGroupPool(10) 10 | 11 | var total uint32 12 | 13 | for i := 0; i < 100; i++ { 14 | wgp.Add() 15 | go func(total *uint32) { 16 | defer wgp.Done() 17 | atomic.AddUint32(total, 1) 18 | }(&total) 19 | } 20 | wgp.Wait() 21 | 22 | if total != 100 { 23 | t.Fatalf("The size '%d' of the pool did not meet expectations.", total) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /extractors/pornhub/pornhub_test.go: -------------------------------------------------------------------------------- 1 | package pornhub 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestPornhub(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://www.pornhub.com/view_video.php?viewkey=ph5cb5fc41c6ebd", 19 | Title: "Must watch Milf drilled by the fireplace", 20 | }, 21 | }, 22 | } 23 | for _, tt := range tests { 24 | t.Run(tt.name, func(t *testing.T) { 25 | New().Extract(tt.args.URL, types.Options{}) 26 | }) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /extractors/douyu/douyu_test.go: -------------------------------------------------------------------------------- 1 | package douyu 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://v.douyu.com/show/l0Q8mMY3wZqv49Ad", 19 | Title: "每日撸报_每日撸报：有些人死了其实它还可以把你带走_斗鱼视频 - 最6的弹幕视频网站", 20 | Size: 10558080, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | New().Extract(tt.args.URL, types.Options{}) 27 | }) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /extractors/yinyuetai/yinyuetai_test.go: -------------------------------------------------------------------------------- 1 | package yinyuetai 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "http://v.yinyuetai.com/video/3386385", 19 | Title: "什么是爱/ What is Love", 20 | Size: 20028736, 21 | Quality: "流畅", 22 | }, 23 | }, 24 | } 25 | for _, tt := range tests { 26 | t.Run(tt.name, func(t *testing.T) { 27 | New().Extract(tt.args.URL, types.Options{}) 28 | }) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /extractors/udn/udn_test.go: -------------------------------------------------------------------------------- 1 | package udn 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestExtract(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://video.udn.com/embed/news/300040", 19 | Title: `生物老師男變女 全校挺"做自己"`, 20 | Size: 12740874, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | data, err := New().Extract(tt.args.URL, types.Options{}) 27 | test.CheckError(t, err) 28 | test.Check(t, tt.args, data[0]) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /extractors/pixivision/pixivision_test.go: -------------------------------------------------------------------------------- 1 | package pixivision 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://www.pixivision.net/zh/a/3271", 19 | Title: "Don't ask me to choose! Tiny Breasts VS Huge Breasts", 20 | }, 21 | }, 22 | } 23 | for _, tt := range tests { 24 | t.Run(tt.name, func(t *testing.T) { 25 | data, err := New().Extract(tt.args.URL, types.Options{}) 26 | test.CheckError(t, err) 27 | test.Check(t, tt.args, data[0]) 28 | }) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /extractors/youku/youku_test.go: -------------------------------------------------------------------------------- 1 | package youku 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "http://v.youku.com/v_show/id_XMzUzMjE3NDczNg==.html", 19 | Title: "车事儿: 智能汽车已经不在遥远 东风风光iX5发布", 20 | Size: 22692900, 21 | Quality: "mp4hd2v2 1280x720", 22 | }, 23 | }, 24 | } 25 | for _, tt := range tests { 26 | t.Run(tt.name, func(t *testing.T) { 27 | New().Extract(tt.args.URL, types.Options{ 28 | YoukuCcode: "0590", 29 | }) 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /extractors/bcy/bcy_test.go: -------------------------------------------------------------------------------- 1 | package bcy 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://bcy.net/item/detail/6558738153367142664", 19 | Title: "cos正片 命运石之门 牧濑红莉栖 克里斯蒂娜… - 半次元 - ACG爱好者社区", 20 | Size: 13035763, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | data, err := New().Extract(tt.args.URL, types.Options{}) 27 | test.CheckError(t, err) 28 | test.Check(t, tt.args, data[0]) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /extractors/geekbang/geekbang_test.go: -------------------------------------------------------------------------------- 1 | package geekbang 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://time.geekbang.org/course/detail/190-97203", 19 | Title: "02 | 内容综述 - 玩转webpack", 20 | Size: 10752472, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | data, err := New().Extract(tt.args.URL, types.Options{}) 27 | test.CheckError(t, err) 28 | test.Check(t, tt.args, data[0]) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /extractors/miaopai/miaopai_test.go: -------------------------------------------------------------------------------- 1 | package miaopai 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "http://n.miaopai.com/media/Dqg5Pmb~I6lChdvOb-~r1BpKzzDu~MPr", 19 | Title: "小学霸6点半起床学习:想赢在起跑线", 20 | Size: 6743958, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | data, err := New().Extract(tt.args.URL, types.Options{}) 27 | test.CheckError(t, err) 28 | test.Check(t, tt.args, data[0]) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/iawia002/annie 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 7 | github.com/PuerkitoBio/goquery v1.4.1 8 | github.com/andybalholm/cascadia v1.0.0 // indirect 9 | github.com/cheggaaa/pb v1.0.25 10 | github.com/fatih/color v1.7.0 11 | github.com/kr/pretty v0.1.0 12 | github.com/mattn/go-colorable v0.0.9 // indirect 13 | github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff 14 | github.com/rylio/ytdl v0.6.2 15 | github.com/tidwall/gjson v1.3.2 16 | gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect 17 | gopkg.in/sourcemap.v1 v1.0.5 // indirect 18 | ) 19 | 20 | replace github.com/rylio/ytdl => github.com/mihaiav/ytdl v0.6.3-0.20200510100116-5f2bf8b4fec0 21 | -------------------------------------------------------------------------------- /extractors/douyin/douyin_test.go: -------------------------------------------------------------------------------- 1 | package douyin 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://www.douyin.com/share/video/6557825773007277319/?mid=6557826301539912456", 19 | Title: "跟特效师一起学跳舞，看变形金刚擎天柱怎么跳，你也来试试！@抖音小助手", 20 | }, 21 | }, 22 | } 23 | for _, tt := range tests { 24 | t.Run(tt.name, func(t *testing.T) { 25 | data, err := New().Extract(tt.args.URL, types.Options{}) 26 | test.CheckError(t, err) 27 | test.Check(t, tt.args, data[0]) 28 | }) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /extractors/universal/universal_test.go: -------------------------------------------------------------------------------- 1 | package universal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg", 19 | Title: "1f5a87801a0711e898b12b640777720f", 20 | Size: 1051042, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | data, err := New().Extract(tt.args.URL, types.Options{}) 27 | test.CheckError(t, err) 28 | test.Check(t, tt.args, data[0]) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /extractors/facebook/facebook_test.go: -------------------------------------------------------------------------------- 1 | package facebook 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://www.facebook.com/groups/314070194112/permalink/10155168902769113/", 19 | Title: "Ukrainian Scientists Worldwide Public Group | Facebook", 20 | Size: 336975453, 21 | Quality: "hd", 22 | }, 23 | }, 24 | } 25 | for _, tt := range tests { 26 | t.Run(tt.name, func(t *testing.T) { 27 | data, err := New().Extract(tt.args.URL, types.Options{}) 28 | test.CheckError(t, err) 29 | test.Check(t, tt.args, data[0]) 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /extractors/xvideos/xvideos_test.go: -------------------------------------------------------------------------------- 1 | package xvideos 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestExtract(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://www.xvideos.com/video29018757/asian_chick_enjoying_sex_debut._hd_full_at_nanairo.co", 19 | Title: "Asian chick enjoying sex debut. HD FULL at: nanairo.co - XVIDEOS.COM", 20 | Size: 16574766, 21 | }, 22 | }, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | data, err := New().Extract(tt.args.URL, types.Options{}) 27 | test.CheckError(t, err) 28 | test.Check(t, tt.args, data[0]) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /extractors/vimeo/vimeo_test.go: -------------------------------------------------------------------------------- 1 | package vimeo 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://player.vimeo.com/video/259325107", 19 | Title: "prfm 20180309", 20 | Size: 131051118, 21 | Quality: "1080p", 22 | }, 23 | }, 24 | { 25 | name: "normal test", 26 | args: test.Args{ 27 | URL: "https://vimeo.com/254865724", 28 | Title: "MAGIC DINER PT. II", 29 | Size: 138966306, 30 | Quality: "1080p", 31 | }, 32 | }, 33 | } 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | New().Extract(tt.args.URL, types.Options{}) 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /extractors/tiktok/tiktok_test.go: -------------------------------------------------------------------------------- 1 | package tiktok 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://www.tiktok.com/@therock/video/6768158408110624005", 19 | Title: "#bestfriend check.", 20 | }, 21 | }, 22 | { 23 | name: "short url test", 24 | args: test.Args{ 25 | URL: "https://vm.tiktok.com/C998PY/", 26 | Title: "Who saw that coming? 🍁 #leaves #fall", 27 | }, 28 | }, 29 | } 30 | for _, tt := range tests { 31 | t.Run(tt.name, func(t *testing.T) { 32 | data, err := New().Extract(tt.args.URL, types.Options{}) 33 | test.CheckError(t, err) 34 | test.Check(t, tt.args, data[0]) 35 | }) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /extractors/netease/netease_test.go: -------------------------------------------------------------------------------- 1 | package netease 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "mv test 1", 17 | args: test.Args{ 18 | URL: "https://music.163.com/#/mv?id=5547010", 19 | Title: "There For You", 20 | Size: 24249078, 21 | }, 22 | }, 23 | { 24 | name: "video test 1", 25 | args: test.Args{ 26 | URL: "https://music.163.com/#/video?id=C8C9D11629798595BD28451DE3AC9FF4", 27 | Title: "＃金曜日の新垣结衣 总集編〈全9編〉", 28 | Size: 37408123, 29 | }, 30 | }, 31 | } 32 | for _, tt := range tests { 33 | t.Run(tt.name, func(t *testing.T) { 34 | data, err := New().Extract(tt.args.URL, types.Options{}) 35 | test.CheckError(t, err) 36 | test.Check(t, tt.args, data[0]) 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /downloader/types.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | // Aria2RPCData defines the data structure of json RPC 2.0 info for Aria2 4 | type Aria2RPCData struct { 5 | // More info about RPC interface please refer to 6 | // https://aria2.github.io/manual/en/html/aria2c.html#rpc-interface 7 | JSONRPC string `json:"jsonrpc"` 8 | ID string `json:"id"` 9 | // For a simple download, only inplemented `addUri` 10 | Method string `json:"method"` 11 | // secret, uris, options 12 | Params [3]interface{} `json:"params"` 13 | } 14 | 15 | // Aria2Input is options for `aria2.addUri` 16 | // https://aria2.github.io/manual/en/html/aria2c.html#id3 17 | type Aria2Input struct { 18 | // The file name of the downloaded file 19 | Out string `json:"out"` 20 | // For a simple download, only add headers 21 | Header []string `json:"header"` 22 | } 23 | 24 | // FilePartMeta defines the data structure of file meta info. 25 | type FilePartMeta struct { 26 | Index float32 27 | Start int64 28 | End int64 29 | Cur int64 30 | } 31 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | * [Style Guide](#style-guide) 4 | * [Build](#build) 5 | * [Features Requested](#features-requested) 6 | 7 | 8 | ## Style Guide 9 | ### Code format 10 | Annie uses [gofmt](https://golang.org/cmd/gofmt) to format the code, you must use [gofmt](https://golang.org/cmd/gofmt) to format your code before submitting. 11 | 12 | ### linter 13 | We recommend using [golint](https://github.com/golang/lint) or [gometalinter](https://github.com/alecthomas/gometalinter) to check your code format. 14 | 15 | 16 | ## Build 17 | 18 | Make sure that this folder is in `GOPATH`, then: 19 | 20 | ```bash 21 | $ go build 22 | ``` 23 | 24 | 25 | ## Features Requested 26 | There are several [features](https://github.com/iawia002/annie/issues?q=is%3Aissue+is%3Aopen+label%3Afeature-request) requested by the community. If you have any idea, feel free to fork the repo, follow the style guide above, push and merge it after passing the test. Besides, you are welcomed to propose new features through the issue. 27 | -------------------------------------------------------------------------------- /utils/download.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | ) 7 | 8 | // NeedDownloadList return the indices of playlist that need download 9 | func NeedDownloadList(items string, itemStart, itemEnd, length int) []int { 10 | if items != "" { 11 | var itemList []int 12 | var selStart, selEnd int 13 | temp := strings.Split(items, ",") 14 | 15 | for _, i := range temp { 16 | selection := strings.Split(i, "-") 17 | selStart, _ = strconv.Atoi(strings.TrimSpace(selection[0])) 18 | 19 | if len(selection) >= 2 { 20 | selEnd, _ = strconv.Atoi(strings.TrimSpace(selection[1])) 21 | } else { 22 | selEnd = selStart 23 | } 24 | 25 | for item := selStart; item <= selEnd; item++ { 26 | itemList = append(itemList, item) 27 | } 28 | } 29 | return itemList 30 | } 31 | 32 | if itemStart < 1 { 33 | itemStart = 1 34 | } 35 | if itemEnd == 0 { 36 | itemEnd = length 37 | } 38 | if itemEnd < itemStart { 39 | itemEnd = itemStart 40 | } 41 | return Range(itemStart, itemEnd) 42 | } 43 | -------------------------------------------------------------------------------- /utils/pool.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "math" 5 | "sync" 6 | ) 7 | 8 | // WaitGroupPool pool of WaitGroup 9 | type WaitGroupPool struct { 10 | pool chan struct{} 11 | wg *sync.WaitGroup 12 | } 13 | 14 | // NewWaitGroupPool creates a sized pool for WaitGroup 15 | func NewWaitGroupPool(size int) *WaitGroupPool { 16 | if size <= 0 { 17 | size = math.MaxInt32 18 | } 19 | return &WaitGroupPool{ 20 | pool: make(chan struct{}, size), 21 | wg: &sync.WaitGroup{}, 22 | } 23 | } 24 | 25 | // Add increments the WaitGroup counter by one. 26 | // See sync.WaitGroup documentation for more information. 27 | func (p *WaitGroupPool) Add() { 28 | p.pool <- struct{}{} 29 | p.wg.Add(1) 30 | } 31 | 32 | // Done decrements the WaitGroup counter by one. 33 | // See sync.WaitGroup documentation for more information. 34 | func (p *WaitGroupPool) Done() { 35 | <-p.pool 36 | p.wg.Done() 37 | } 38 | 39 | // Wait blocks until the WaitGroup counter is zero. 40 | // See sync.WaitGroup documentation for more information. 41 | func (p *WaitGroupPool) Wait() { 42 | p.wg.Wait() 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2018-present, iawia002 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /extractors/mgtv/mgtv_test.go: -------------------------------------------------------------------------------- 1 | package mgtv 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test 1", 17 | args: test.Args{ 18 | URL: "https://www.mgtv.com/b/322712/4317248.html", 19 | Title: "我是大侦探 先导片：何炅吴磊邓伦穿越破案", 20 | Size: 86169236, 21 | Quality: "超清", 22 | }, 23 | }, 24 | { 25 | name: "normal test 2", 26 | args: test.Args{ 27 | URL: "https://www.mgtv.com/b/308703/4197072.html", 28 | Title: "芒果捞星闻 2017 诺一为爷爷和姥爷做翻译超萌", 29 | Size: 6486376, 30 | Quality: "超清", 31 | }, 32 | }, 33 | { 34 | name: "vip test", 35 | args: test.Args{ 36 | URL: "https://www.mgtv.com/b/322865/4352046.html", 37 | Title: "向往的生活 第二季 先导片：何炅黄磊回归质朴生活", 38 | Size: 453246944, 39 | Quality: "超清", 40 | }, 41 | }, 42 | } 43 | for _, tt := range tests { 44 | t.Run(tt.name, func(t *testing.T) { 45 | New().Extract(tt.args.URL, types.Options{}) 46 | }) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /extractors/yinyuetai/types.go: -------------------------------------------------------------------------------- 1 | package yinyuetai 2 | 3 | type yinyuetaiMvData struct { 4 | Error bool `json:"error"` 5 | Message string `json:"message"` 6 | VideoInfo videoInfo `json:"videoInfo"` 7 | } 8 | 9 | type videoInfo struct { 10 | CoreVideoInfo coreVideoInfo `json:"coreVideoInfo"` 11 | } 12 | 13 | type coreVideoInfo struct { 14 | ArtistNames string `json:"artistNames"` 15 | Duration int `json:"duration"` 16 | Error bool `json:"error"` 17 | ErrorMsg string `json:"errorMsg"` 18 | VideoID int `json:"videoID"` 19 | VideoName string `json:"videoName"` 20 | VideoURLModels []videoURLModel `json:"videoURLModels"` 21 | } 22 | 23 | type videoURLModel struct { 24 | Bitrate int `json:"bitrate"` 25 | BitrateType int `json:"bitrateType"` 26 | FileSize int64 `json:"fileSize"` 27 | MD5 string `json:"md5"` 28 | SHA1 string `json:"sha1"` 29 | QualityLevel string `json:"qualityLevel"` 30 | QualityLevelName string `json:"qualityLevelName"` 31 | VideoURL string `json:"videoURL"` 32 | } 33 | -------------------------------------------------------------------------------- /extractors/tumblr/tumblr_test.go: -------------------------------------------------------------------------------- 1 | package tumblr 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "image test 1", 17 | args: test.Args{ 18 | URL: "http://fuckyeah-fx.tumblr.com/post/170392654141/180202-%E5%AE%8B%E8%8C%9C", 19 | Title: "f(x)", 20 | }, 21 | }, 22 | { 23 | name: "image test 2", 24 | args: test.Args{ 25 | URL: "http://therealautoblog.tumblr.com/post/171623222197/paganis-new-projects-huayra-successor-with", 26 | Title: "Autoblog • Pagani’s new projects: Huayra successor with...", 27 | }, 28 | }, 29 | { 30 | name: "video test", 31 | args: test.Args{ 32 | URL: "https://boomgoestheprower.tumblr.com/post/174127507696", 33 | Title: "Out of Context Sonic Boom", 34 | }, 35 | }, 36 | } 37 | for _, tt := range tests { 38 | t.Run(tt.name, func(t *testing.T) { 39 | data, err := New().Extract(tt.args.URL, types.Options{}) 40 | test.CheckError(t, err) 41 | test.Check(t, tt.args, data[0]) 42 | }) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /extractors/universal/universal.go: -------------------------------------------------------------------------------- 1 | package universal 2 | 3 | import ( 4 | "github.com/iawia002/annie/extractors/types" 5 | "github.com/iawia002/annie/request" 6 | "github.com/iawia002/annie/utils" 7 | ) 8 | 9 | type extractor struct{} 10 | 11 | // New returns a youtube extractor. 12 | func New() types.Extractor { 13 | return &extractor{} 14 | } 15 | 16 | // Extract is the main function to extract the data. 17 | func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { 18 | filename, ext, err := utils.GetNameAndExt(url) 19 | if err != nil { 20 | return nil, err 21 | } 22 | size, err := request.Size(url, url) 23 | if err != nil { 24 | return nil, err 25 | } 26 | streams := map[string]*types.Stream{ 27 | "default": { 28 | Parts: []*types.Part{ 29 | { 30 | URL: url, 31 | Size: size, 32 | Ext: ext, 33 | }, 34 | }, 35 | Size: size, 36 | }, 37 | } 38 | contentType, err := request.ContentType(url, url) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return []*types.Data{ 44 | { 45 | Site: "Universal", 46 | Title: filename, 47 | Type: types.DataType(contentType), 48 | Streams: streams, 49 | URL: url, 50 | }, 51 | }, nil 52 | } 53 | -------------------------------------------------------------------------------- /extractors/instagram/instagram_test.go: -------------------------------------------------------------------------------- 1 | package instagram 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "video test", 17 | args: test.Args{ 18 | URL: "https://www.instagram.com/p/BlIka1ZFCNr", 19 | Title: "P!NK on Instagram: “AFL got us hyped! #adelaideadventures #iwanttoplay”", 20 | Size: 2741413, 21 | }, 22 | }, 23 | { 24 | name: "image test", 25 | args: test.Args{ 26 | URL: "https://www.instagram.com/p/Bl5oVUyl9Yx", 27 | Title: "P!NK on Instagram: “Australia:heaven”", 28 | Size: 250596, 29 | }, 30 | }, 31 | { 32 | name: "image album test", 33 | args: test.Args{ 34 | URL: "https://www.instagram.com/p/Bjyr-gxF4Rb", 35 | Title: "P!NK on Instagram: “Nature. Nurture.\nKiddos. Gratitude”", 36 | Size: 4599909, 37 | }, 38 | }, 39 | } 40 | for _, tt := range tests { 41 | t.Run(tt.name, func(t *testing.T) { 42 | data, err := New().Extract(tt.args.URL, types.Options{}) 43 | test.CheckError(t, err) 44 | test.Check(t, tt.args, data[0]) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /extractors/qq/qq_test.go: -------------------------------------------------------------------------------- 1 | package qq 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://v.qq.com/x/page/n0687peq62x.html", 19 | Title: "世界杯第一期：100秒速成！“伪球迷”世界杯生存指南", 20 | Size: 23759683, 21 | Quality: "蓝光;(1080P)", 22 | }, 23 | }, 24 | // { 25 | // name: "movie and vid test", 26 | // args: test.Args{ 27 | // URL: "https://v.qq.com/x/cover/e5qmd3z5jr0uigk.html", 28 | // Title: "赌侠（粤语版）", 29 | // Size: 1046910811, 30 | // Quality: "超清;(720P)", 31 | // }, 32 | // }, 33 | { 34 | name: "fmt ID test", 35 | args: test.Args{ 36 | URL: "https://v.qq.com/x/cover/2aya3ibdmft6vdw/e0765r4mwcr.html", 37 | Title: "《卡路里》出圈！妖娆男子教学广场舞版，大妈表情亮了！", 38 | Size: 14112979, 39 | Quality: "超清;(720P)", 40 | }, 41 | }, 42 | } 43 | for _, tt := range tests { 44 | t.Run(tt.name, func(t *testing.T) { 45 | data, err := New().Extract(tt.args.URL, types.Options{}) 46 | test.CheckError(t, err) 47 | test.Check(t, tt.args, data[0]) 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /extractors/twitter/twitter_test.go: -------------------------------------------------------------------------------- 1 | package twitter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "https://twitter.com/justinbieber/status/898217160060698624", 19 | Title: "Justin Bieber on Twitter 898217160060698624", 20 | Quality: "720x1280", 21 | }, 22 | }, 23 | { 24 | name: "abnormal uri test1", 25 | args: test.Args{ 26 | URL: "https://twitter.com/twitter/statuses/898567934192177153", 27 | Title: "Justin Bieber on Twitter 898567934192177153", 28 | Quality: "1280x720", 29 | }, 30 | }, 31 | { 32 | name: "abnormal uri test2", 33 | args: test.Args{ 34 | URL: "https://twitter.com/kyoudera/status/971819131711373312/video/1/", 35 | Title: "ネメシス 京寺 on Twitter 971819131711373312", 36 | Quality: "1280x720", 37 | }, 38 | }, 39 | } 40 | // The file size changes every time (caused by CDN?), so the size is not checked here 41 | for _, tt := range tests { 42 | t.Run(tt.name, func(t *testing.T) { 43 | New().Extract(tt.args.URL, types.Options{}) 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /extractors/iqiyi/iqiyi_test.go: -------------------------------------------------------------------------------- 1 | package iqiyi 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "normal test", 17 | args: test.Args{ 18 | URL: "http://www.iqiyi.com/v_19rrbdmaj0.html", 19 | Title: "新一轮降水将至 冷空气影响中东部地区-资讯-完整版视频在线观看-爱奇艺", 20 | Size: 2952228, 21 | Quality: "896x504", 22 | }, 23 | }, 24 | { 25 | name: "title test 1", 26 | args: test.Args{ 27 | URL: "http://www.iqiyi.com/v_19rqy2z83w.html", 28 | Title: "收了创意视频2018：58天环球飞行记", 29 | Size: 76186786, 30 | Quality: "1920x1080", 31 | }, 32 | }, 33 | { 34 | name: "curid test 1", 35 | args: test.Args{ 36 | URL: "https://www.iqiyi.com/v_19rro0jdls.html#curid=350289100_6e6601aae889d0b1004586a52027c321", 37 | Title: "Shawn Mendes - Never Be Alone", 38 | Size: 79921894, 39 | Quality: "1920x800", 40 | }, 41 | }, 42 | } 43 | for _, tt := range tests { 44 | t.Run(tt.name, func(t *testing.T) { 45 | data, err := New().Extract(tt.args.URL, types.Options{}) 46 | test.CheckError(t, err) 47 | test.Check(t, tt.args, data[0]) 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /extractors/weibo/weibo_test.go: -------------------------------------------------------------------------------- 1 | package weibo 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | }{ 15 | { 16 | name: "title test", 17 | args: test.Args{ 18 | URL: "https://m.weibo.cn/status/4237529215145705", 19 | Title: `近日，日本视错觉大师、明治大学特任教授\"杉原厚吉的“错觉箭头“作品又引起世界人民的关注。反射，透视和视角的巧妙结合产生了这种惊人的幻觉：箭头向右？转过来还是向右？\n\n引用杉原教授的经典描述：“我们看外面的世界的方式——也就是我们的知觉——都是由大脑机制间接产生的，所以所有知觉在某`, 20 | Size: 1125984, 21 | }, 22 | }, 23 | { 24 | name: "weibo.com test", 25 | args: test.Args{ 26 | URL: "https://weibo.com/1642500775/GjbO5ByzE", 27 | Title: "让人怦然心动的小姐姐们 via@大懒糖", 28 | Size: 2002420, 29 | }, 30 | }, 31 | { 32 | name: "weibo.com/tv test", 33 | args: test.Args{ 34 | URL: "https://weibo.com/tv/v/jGz6llNZ1?fid=1034:4298353237002268", 35 | Title: "做了这么一个屌炸天的视频我也不知道起什么标题好 @DRock-Art @毒液-致命守护者 @漫威影业 #绘画# #blender# #漫威#", 36 | Quality: "720", 37 | Size: 7520929, 38 | }, 39 | }, 40 | } 41 | for _, tt := range tests { 42 | t.Run(tt.name, func(t *testing.T) { 43 | data, err := New().Extract(tt.args.URL, types.Options{}) 44 | test.CheckError(t, err) 45 | test.Check(t, tt.args, data[0]) 46 | }) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: annie 2 | env: 3 | - GO111MODULE=on 4 | builds: 5 | - env: 6 | - CGO_ENABLED=0 7 | binary: annie 8 | goos: 9 | - windows 10 | - darwin 11 | - linux 12 | - freebsd 13 | - openbsd 14 | - netbsd 15 | goarch: 16 | - 386 17 | - amd64 18 | - arm 19 | - arm64 20 | ignore: 21 | - goos: freebsd 22 | goarch: arm 23 | goarm: 6 24 | - goos: openbsd 25 | goarch: arm 26 | goarm: 6 27 | # hooks: 28 | # Please install upx first, https://github.com/upx/upx/releases 29 | # post: compress.bat 30 | # post: ./compress.sh 31 | env_files: 32 | github_token: ./github_token 33 | changelog: 34 | sort: asc 35 | filters: 36 | exclude: 37 | - '^docs' 38 | - '^tests' 39 | - Merge pull request 40 | - Merge branch 41 | archive: 42 | name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}_v{{ .Arm }}{{ end }}' 43 | format: tar.gz 44 | format_overrides: 45 | - goos: windows 46 | format: zip 47 | files: 48 | - none* 49 | wrap_in_directory: false 50 | replacements: 51 | amd64: 64-bit 52 | 386: 32-bit 53 | arm: ARM 54 | arm64: ARM64 55 | darwin: macOS 56 | linux: Linux 57 | windows: Windows 58 | openbsd: OpenBSD 59 | netbsd: NetBSD 60 | freebsd: FreeBSD 61 | release: 62 | draft: true 63 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/PuerkitoBio/goquery" 8 | ) 9 | 10 | // GetDoc return Document object of the HTML string 11 | func GetDoc(html string) (*goquery.Document, error) { 12 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) 13 | if err != nil { 14 | return nil, err 15 | } 16 | return doc, nil 17 | } 18 | 19 | // GetImages find the img with a given class name 20 | func GetImages(html, imgClass string, urlHandler func(string) string) (string, []string, error) { 21 | doc, err := GetDoc(html) 22 | if err != nil { 23 | return "", nil, err 24 | } 25 | title := Title(doc) 26 | urls := make([]string, 0) 27 | doc.Find(fmt.Sprintf("img[class=\"%s\"]", imgClass)).Each( 28 | func(i int, s *goquery.Selection) { 29 | url, _ := s.Attr("src") 30 | if urlHandler != nil { 31 | // Handle URL as needed 32 | url = urlHandler(url) 33 | } 34 | urls = append(urls, url) 35 | }, 36 | ) 37 | return title, urls, nil 38 | } 39 | 40 | // Title get title 41 | func Title(doc *goquery.Document) string { 42 | var title string 43 | title = strings.Replace( 44 | strings.TrimSpace(doc.Find("h1").First().Text()), "\n", "", -1, 45 | ) 46 | if title == "" { 47 | // Bilibili: Some movie page got no h1 tag 48 | title, _ = doc.Find("meta[property=\"og:title\"]").Attr("content") 49 | } 50 | if title == "" { 51 | title = doc.Find("title").Text() 52 | } 53 | return title 54 | } 55 | -------------------------------------------------------------------------------- /extractors/pixivision/pixivision.go: -------------------------------------------------------------------------------- 1 | package pixivision 2 | 3 | import ( 4 | "github.com/iawia002/annie/extractors/types" 5 | "github.com/iawia002/annie/parser" 6 | "github.com/iawia002/annie/request" 7 | "github.com/iawia002/annie/utils" 8 | ) 9 | 10 | type extractor struct{} 11 | 12 | // New returns a youtube extractor. 13 | func New() types.Extractor { 14 | return &extractor{} 15 | } 16 | 17 | // Extract is the main function to extract the data. 18 | func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { 19 | html, err := request.Get(url, url, nil) 20 | if err != nil { 21 | return nil, err 22 | } 23 | title, urls, err := parser.GetImages(html, "am__work__illust ", nil) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | parts := make([]*types.Part, 0, len(urls)) 29 | for _, u := range urls { 30 | _, ext, err := utils.GetNameAndExt(u) 31 | if err != nil { 32 | return nil, err 33 | } 34 | size, err := request.Size(u, url) 35 | if err != nil { 36 | return nil, err 37 | } 38 | parts = append(parts, &types.Part{ 39 | URL: u, 40 | Size: size, 41 | Ext: ext, 42 | }) 43 | } 44 | 45 | streams := map[string]*types.Stream{ 46 | "default": { 47 | Parts: parts, 48 | }, 49 | } 50 | 51 | return []*types.Data{ 52 | { 53 | Site: "pixivision pixivision.net", 54 | Title: title, 55 | Type: types.DataTypeImage, 56 | Streams: streams, 57 | URL: url, 58 | }, 59 | }, nil 60 | } 61 | -------------------------------------------------------------------------------- /test/utils.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/iawia002/annie/extractors/types" 8 | ) 9 | 10 | // Args Arguments for extractor tests 11 | type Args struct { 12 | URL string 13 | Title string 14 | Quality string 15 | Size int64 16 | } 17 | 18 | // CheckData check the given data 19 | func CheckData(args, data Args) bool { 20 | if args.Title != data.Title { 21 | return false 22 | } 23 | // not every video got quality information 24 | if args.Quality != "" && args.Quality != data.Quality { 25 | return false 26 | } 27 | if args.Size != 0 && args.Size != data.Size { 28 | return false 29 | } 30 | return true 31 | } 32 | 33 | // Check check the result 34 | func Check(t *testing.T, args Args, data *types.Data) { 35 | // get the default stream 36 | sortedStreams := make([]*types.Stream, 0, len(data.Streams)) 37 | for _, s := range data.Streams { 38 | sortedStreams = append(sortedStreams, s) 39 | } 40 | sort.Slice(sortedStreams, func(i, j int) bool { return sortedStreams[i].Size > sortedStreams[j].Size }) 41 | defaultData := sortedStreams[0] 42 | 43 | temp := Args{ 44 | Title: data.Title, 45 | Quality: defaultData.Quality, 46 | Size: defaultData.Size, 47 | } 48 | if !CheckData(args, temp) { 49 | t.Errorf("Got: %v\nExpected: %v", temp, args) 50 | } 51 | } 52 | 53 | // CheckError check the error 54 | func CheckError(t *testing.T, err error) { 55 | if err != nil { 56 | t.Fatalf("Unexpected error: %v", err) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /extractors/tangdou/tangdou_test.go: -------------------------------------------------------------------------------- 1 | package tangdou 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestTangDou(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | playlist bool 15 | }{ 16 | { 17 | name: "contains video URL test directly and can get title from body's div tag", 18 | args: test.Args{ 19 | URL: "http://www.tangdou.com/v95/dAOQNgMjwT2D5w2.html", 20 | Title: "杨丽萍广场舞《好日子天天过》喜庆双扇扇子舞", 21 | }, 22 | }, 23 | { 24 | name: "need call share url first and get the signed video URL test and can get title from head's title tag", 25 | args: test.Args{ 26 | URL: "http://m.tangdou.com/v94/dAOMMYNjwT1T2Q2.html", 27 | Title: "吉美广场舞《再唱山歌给党听》民族形体舞 附教学视频在线观看", 28 | Size: 50710318, 29 | }, 30 | }, 31 | { 32 | name: "playlist test", 33 | args: test.Args{ 34 | URL: "http://www.tangdou.com/playlist/view/2816/page/4", 35 | Title: "茉莉广场舞 我向草原问个好 原创藏族风民族舞附教学", 36 | Size: 66284484, 37 | }, 38 | playlist: true, 39 | }, 40 | } 41 | for _, tt := range tests { 42 | t.Run(tt.name, func(t *testing.T) { 43 | var ( 44 | data []*types.Data 45 | err error 46 | ) 47 | if tt.playlist { 48 | // playlist mode 49 | _, err = New().Extract(tt.args.URL, types.Options{ 50 | Playlist: true, 51 | ThreadNumber: 9, 52 | }) 53 | test.CheckError(t, err) 54 | } else { 55 | data, err = New().Extract(tt.args.URL, types.Options{}) 56 | test.CheckError(t, err) 57 | test.Check(t, tt.args, data[0]) 58 | } 59 | }) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /extractors/facebook/facebook.go: -------------------------------------------------------------------------------- 1 | package facebook 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/request" 8 | "github.com/iawia002/annie/utils" 9 | ) 10 | 11 | type extractor struct{} 12 | 13 | // New returns a youtube extractor. 14 | func New() types.Extractor { 15 | return &extractor{} 16 | } 17 | 18 | // Extract is the main function to extract the data. 19 | func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { 20 | var err error 21 | html, err := request.Get(url, url, nil) 22 | if err != nil { 23 | return nil, err 24 | } 25 | titles := utils.MatchOneOf(html, `(.+)`) 26 | if titles == nil || len(titles) < 2 { 27 | return nil, types.ErrURLParseFailed 28 | } 29 | title := titles[1] 30 | 31 | streams := make(map[string]*types.Stream, 2) 32 | for _, quality := range []string{"sd", "hd"} { 33 | srcElement := utils.MatchOneOf( 34 | html, fmt.Sprintf(`%s_src_no_ratelimit:"(.+?)"`, quality), 35 | ) 36 | if srcElement == nil || len(srcElement) < 2 { 37 | continue 38 | } 39 | 40 | u := srcElement[1] 41 | size, err := request.Size(u, url) 42 | if err != nil { 43 | return nil, err 44 | } 45 | urlData := &types.Part{ 46 | URL: u, 47 | Size: size, 48 | Ext: "mp4", 49 | } 50 | streams[quality] = &types.Stream{ 51 | Parts: []*types.Part{urlData}, 52 | Size: size, 53 | Quality: quality, 54 | } 55 | } 56 | 57 | return []*types.Data{ 58 | { 59 | Site: "Facebook facebook.com", 60 | Title: title, 61 | Type: types.DataTypeVideo, 62 | Streams: streams, 63 | URL: url, 64 | }, 65 | }, nil 66 | } 67 | -------------------------------------------------------------------------------- /extractors/bilibili/types.go: -------------------------------------------------------------------------------- 1 | package bilibili 2 | 3 | type qualityInfo struct { 4 | Description []string `json:"accept_description"` 5 | Quality []int `json:"accept_quality"` 6 | } 7 | 8 | type dURLData struct { 9 | Size int64 `json:"size"` 10 | URL string `json:"url"` 11 | Order int `json:"order"` 12 | } 13 | 14 | type bilibiliData struct { 15 | DURL []dURLData `json:"durl"` 16 | Format string `json:"format"` 17 | Quality int `json:"quality"` 18 | } 19 | 20 | // {"code":0,"message":"0","ttl":1,"data":{"token":"aaa"}} 21 | // {"code":-101,"message":"账号未登录","ttl":1} 22 | type tokenData struct { 23 | Token string `json:"token"` 24 | } 25 | 26 | type token struct { 27 | Code int `json:"code"` 28 | Message string `json:"message"` 29 | Data tokenData `json:"data"` 30 | } 31 | 32 | type bangumiEpData struct { 33 | Aid int `json:"aid"` 34 | Cid int `json:"cid"` 35 | ID int `json:"id"` 36 | EpID int `json:"ep_id"` 37 | } 38 | 39 | type bangumiData struct { 40 | EpInfo bangumiEpData `json:"epInfo"` 41 | EpList []bangumiEpData `json:"epList"` 42 | } 43 | 44 | type videoPagesData struct { 45 | Cid int `json:"cid"` 46 | Part string `json:"part"` 47 | Page int `json:"page"` 48 | } 49 | 50 | type multiPageVideoData struct { 51 | Title string `json:"title"` 52 | Pages []videoPagesData `json:"pages"` 53 | } 54 | 55 | type multiPage struct { 56 | Aid int `json:"aid"` 57 | VideoData multiPageVideoData `json:"videoData"` 58 | } 59 | 60 | var qualityString = map[int]string{ 61 | 116: "高清 1080P60", 62 | 74: "高清 720P60", 63 | 112: "高清 1080P+", 64 | 80: "高清 1080P", 65 | 64: "高清 720P", 66 | 48: "高清 720P", 67 | 32: "清晰 480P", 68 | 16: "流畅 360P", 69 | 15: "流畅 360P", 70 | } 71 | -------------------------------------------------------------------------------- /utils/ffmpeg.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | ) 9 | 10 | func runMergeCmd(cmd *exec.Cmd, paths []string, mergeFilePath string) error { 11 | var stderr bytes.Buffer 12 | cmd.Stderr = &stderr 13 | err := cmd.Run() 14 | if err != nil { 15 | return fmt.Errorf("%s\n%s", err, stderr.String()) 16 | } 17 | 18 | if mergeFilePath != "" { 19 | os.Remove(mergeFilePath) // nolint 20 | } 21 | // remove parts 22 | for _, path := range paths { 23 | os.Remove(path) // nolint 24 | } 25 | return nil 26 | } 27 | 28 | // MergeFilesWithSameExtension merges files that have the same extension into one. 29 | // Can also handle merging audio and video. 30 | func MergeFilesWithSameExtension(paths []string, mergedFilePath string) error { 31 | cmds := []string{ 32 | "-y", 33 | } 34 | for _, path := range paths { 35 | cmds = append(cmds, "-i", path) 36 | } 37 | cmds = append(cmds, "-c:v", "copy", "-c:a", "copy", mergedFilePath) 38 | return runMergeCmd(exec.Command("ffmpeg", cmds...), paths, "") 39 | } 40 | 41 | // MergeToMP4 merges video parts to an MP4 file. 42 | func MergeToMP4(paths []string, mergedFilePath string, filename string) error { 43 | //mergeFilePath := filename + ".txt" // merge list file should be in the current directory 44 | mergeFilePath := mergedFilePath +"_"+filename + ".txt" 45 | 46 | // write ffmpeg input file list 47 | mergeFile, _ := os.Create(mergeFilePath) 48 | for _, path := range paths { 49 | mergeFile.Write([]byte(fmt.Sprintf("file '%s'\n", path))) // nolint 50 | } 51 | mergeFile.Close() // nolint 52 | 53 | cmd := exec.Command( 54 | "ffmpeg", "-y", "-f", "concat", "-safe", "-1", 55 | "-i", mergeFilePath, "-c", "copy", "-bsf:a", "aac_adtstoasc", mergedFilePath, 56 | ) 57 | return runMergeCmd(cmd, paths, mergeFilePath) 58 | } 59 | -------------------------------------------------------------------------------- /extractors/youtube/youtube_test.go: -------------------------------------------------------------------------------- 1 | package youtube 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/iawia002/annie/extractors/types" 7 | "github.com/iawia002/annie/test" 8 | ) 9 | 10 | func TestYoutube(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | args test.Args 14 | playlist bool 15 | }{ 16 | { 17 | name: "normal test", 18 | args: test.Args{ 19 | URL: "https://www.youtube.com/watch?v=Gnbch2osEeo", 20 | Title: "Multifandom Mashup 2017", 21 | }, 22 | }, 23 | { 24 | name: "signature test", 25 | args: test.Args{ 26 | URL: "https://www.youtube.com/watch?v=ZtgzKBrU1GY", 27 | Title: "Halo Infinite - E3 2019 - Discover Hope", 28 | }, 29 | }, 30 | { 31 | name: "playlist test", 32 | args: test.Args{ 33 | URL: "https://www.youtube.com/watch?v=Lt2pwLxJxgA&list=PLIYAO-qLriEtYm7UcXPH3SOJxgqjwRrIw", 34 | Title: "papi酱 - 你有酱婶儿的朋友吗？", 35 | }, 36 | playlist: true, 37 | }, 38 | { 39 | name: "url_encoded_fmt_stream_map test", 40 | args: test.Args{ 41 | URL: "https://youtu.be/DNaOZovrSVo", 42 | Title: "QNAP Case Study - Scorptec", 43 | }, 44 | }, 45 | { 46 | name: "stream 404 test 1", 47 | args: test.Args{ 48 | URL: "https://www.youtube.com/watch?v=MRJ8NnUXacY", 49 | Title: "FreeFileSync: Mirror Synchronization", 50 | }, 51 | }, 52 | } 53 | for _, tt := range tests { 54 | t.Run(tt.name, func(t *testing.T) { 55 | var ( 56 | data []*types.Data 57 | err error 58 | ) 59 | if tt.playlist { 60 | // playlist mode 61 | _, err = New().Extract(tt.args.URL, types.Options{ 62 | Playlist: true, 63 | ThreadNumber: 9, 64 | }) 65 | test.CheckError(t, err) 66 | } else { 67 | data, err = New().Extract(tt.args.URL, types.Options{}) 68 | test.CheckError(t, err) 69 | test.Check(t, tt.args, data[0]) 70 | } 71 | }) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /utils/download_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestNeedDownloadList(t *testing.T) { 9 | type args struct { 10 | len int 11 | } 12 | tests := []struct { 13 | name string 14 | args args 15 | want []int 16 | start int 17 | end int 18 | items string 19 | }{ 20 | { 21 | name: "start end test 1", 22 | args: args{ 23 | len: 3, 24 | }, 25 | start: 2, 26 | end: 2, 27 | want: []int{2}, 28 | }, 29 | { 30 | name: "start end test 2", 31 | args: args{ 32 | len: 3, 33 | }, 34 | end: 2, 35 | want: []int{1, 2}, 36 | }, 37 | { 38 | name: "start end test 3", 39 | args: args{ 40 | len: 3, 41 | }, 42 | start: 2, 43 | end: 0, 44 | want: []int{2, 3}, 45 | }, 46 | { 47 | name: "start end test 4", 48 | args: args{ 49 | len: 3, 50 | }, 51 | start: 2, 52 | end: 1, 53 | want: []int{2}, 54 | }, 55 | { 56 | name: "items test", 57 | args: args{ 58 | len: 3, 59 | }, 60 | items: "1, 3", 61 | want: []int{1, 3}, 62 | }, 63 | { 64 | name: "from to item selection 1", 65 | args: args{ 66 | len: 10, 67 | }, 68 | items: "1-3, 5, 7-8, 10", 69 | want: []int{1, 2, 3, 5, 7, 8, 10}, 70 | }, 71 | { 72 | name: "from to item selection 2", 73 | args: args{ 74 | len: 10, 75 | }, 76 | items: "1,2, 4 , 5, 7-8 , 10", 77 | want: []int{1, 2, 4, 5, 7, 8, 10}, 78 | }, 79 | { 80 | name: "from to item selection 3", 81 | args: args{ 82 | len: 10, 83 | }, 84 | items: "5-1, 2", 85 | want: []int{2}, 86 | }, 87 | } 88 | for _, tt := range tests { 89 | t.Run(tt.name, func(t *testing.T) { 90 | if got := NeedDownloadList(tt.items, tt.start, tt.end, tt.args.len); !reflect.DeepEqual(got, tt.want) { 91 | t.Errorf("NeedDownloadList() = %v, want %v", got, tt.want) 92 | } 93 | }) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /extractors/netease/netease.go: -------------------------------------------------------------------------------- 1 | package netease 2 | 3 | import ( 4 | "errors" 5 | netURL "net/url" 6 | "strings" 7 | 8 | "github.com/iawia002/annie/extractors/types" 9 | "github.com/iawia002/annie/request" 10 | "github.com/iawia002/annie/utils" 11 | ) 12 | 13 | type extractor struct{} 14 | 15 | // New returns a youtube extractor. 16 | func New() types.Extractor { 17 | return &extractor{} 18 | } 19 | 20 | // Extract is the main function to extract the data. 21 | func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { 22 | url = strings.Replace(url, "/#/", "/", 1) 23 | vid := utils.MatchOneOf(url, `/(mv|video)\?id=(\w+)`) 24 | if vid == nil { 25 | return nil, errors.New("invalid url for netease music") 26 | } 27 | 28 | html, err := request.Get(url, url, nil) 29 | if err != nil { 30 | return nil, err 31 | } 32 | if strings.Contains(html, "u-errlg-404") { 33 | return nil, errors.New("404 music not found") 34 | } 35 | 36 | titles := utils.MatchOneOf(html, ``) 37 | if titles == nil || len(titles) < 2 { 38 | return nil, types.ErrURLParseFailed 39 | } 40 | title := titles[1] 41 | 42 | realURLs := utils.MatchOneOf(html, ``) 43 | if realURLs == nil || len(realURLs) < 2 { 44 | return nil, types.ErrURLParseFailed 45 | } 46 | realURL, _ := netURL.QueryUnescape(realURLs[1]) 47 | 48 | size, err := request.Size(realURL, url) 49 | if err != nil { 50 | return nil, err 51 | } 52 | urlData := &types.Part{ 53 | URL: realURL, 54 | Size: size, 55 | Ext: "mp4", 56 | } 57 | streams := map[string]*types.Stream{ 58 | "default": { 59 | Parts: []*types.Part{urlData}, 60 | Size: size, 61 | }, 62 | } 63 | return []*types.Data{ 64 | { 65 | Site: "网易云音乐 music.163.com", 66 | Title: title, 67 | Type: types.DataTypeVideo, 68 | Streams: streams, 69 | URL: url, 70 | }, 71 | }, nil 72 | } 73 | -------------------------------------------------------------------------------- /downloader/utils.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | 7 | "github.com/fatih/color" 8 | 9 | "github.com/iawia002/annie/extractors/types" 10 | ) 11 | 12 | var ( 13 | blue = color.New(color.FgBlue) 14 | cyan = color.New(color.FgCyan) 15 | ) 16 | 17 | func genSortedStreams(streams map[string]*types.Stream) []*types.Stream { 18 | sortedStreams := make([]*types.Stream, 0, len(streams)) 19 | for _, data := range streams { 20 | sortedStreams = append(sortedStreams, data) 21 | } 22 | if len(sortedStreams) > 1 { 23 | sort.Slice( 24 | sortedStreams, func(i, j int) bool { return sortedStreams[i].Size > sortedStreams[j].Size }, 25 | ) 26 | } 27 | return sortedStreams 28 | } 29 | 30 | func printHeader(data *types.Data) { 31 | fmt.Println() 32 | cyan.Printf(" Site: ") // nolint 33 | fmt.Println(data.Site) 34 | cyan.Printf(" Title: ") // nolint 35 | fmt.Println(data.Title) 36 | cyan.Printf(" Type: ") // nolint 37 | fmt.Println(data.Type) 38 | } 39 | 40 | func printStream(stream *types.Stream) { 41 | blue.Println(fmt.Sprintf(" [%s] -------------------", stream.ID)) // nolint 42 | if stream.Quality != "" { 43 | cyan.Printf(" Quality: ") // nolint 44 | fmt.Println(stream.Quality) 45 | } 46 | cyan.Printf(" Size: ") // nolint 47 | fmt.Printf("%.2f MiB (%d Bytes)\n", float64(stream.Size)/(1024*1024), stream.Size) 48 | cyan.Printf(" # download with: ") // nolint 49 | fmt.Printf("annie -f %s ...\n\n", stream.ID) 50 | } 51 | 52 | func printInfo(data *types.Data, sortedStreams []*types.Stream) { 53 | printHeader(data) 54 | 55 | cyan.Printf(" Streams: ") // nolint 56 | fmt.Println("# All available quality") 57 | for _, stream := range sortedStreams { 58 | printStream(stream) 59 | } 60 | } 61 | 62 | func printStreamInfo(data *types.Data, stream *types.Stream) { 63 | printHeader(data) 64 | 65 | cyan.Printf(" Stream: ") // nolint 66 | fmt.Println() 67 | printStream(stream) 68 | } 69 | -------------------------------------------------------------------------------- /extractors/tiktok/tiktok.go: -------------------------------------------------------------------------------- 1 | package tiktok 2 | 3 | import ( 4 | "github.com/iawia002/annie/extractors/types" 5 | "github.com/iawia002/annie/request" 6 | "github.com/iawia002/annie/utils" 7 | ) 8 | 9 | type extractor struct{} 10 | 11 | // New returns a youtube extractor. 12 | func New() types.Extractor { 13 | return &extractor{} 14 | } 15 | 16 | // Extract is the main function to extract the data. 17 | func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { 18 | html, err := request.Get(url, url, nil) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | // There are a few json objects loaded into the html that are useful. We're able to parse the video url from the 24 | // videoObject json. 25 | 26 | videoScriptTag := utils.MatchOneOf(html, ``) 55 | var flashvarJS string 56 | for _, s := range scripts { 57 | for _, js := range s[1:] { 58 | if strings.Contains(js, "flashvars_") { 59 | flashvarJS = js 60 | break 61 | } 62 | } 63 | } 64 | vm := otto.New() 65 | vm.Run(flashvarJS) // nolint 66 | var urls []string 67 | for i := 0; i < len(pornhubs); i++ { 68 | if value, err := vm.Get(fmt.Sprintf("media_%d", i)); err == nil { 69 | urls = append(urls, value.String()) 70 | } 71 | } 72 | 73 | streams := make(map[string]*types.Stream, len(pornhubs)) 74 | for i, data := range pornhubs { 75 | if data.Format != "mp4" { 76 | continue 77 | } 78 | 79 | if bytes.ContainsRune(data.Quality, '[') { 80 | // skip the case where the quality value is an array 81 | // "quality": [ 82 | // 720, 83 | // 480, 84 | // 240 85 | // ] 86 | continue 87 | } 88 | quality := string(data.Quality) 89 | 90 | realURL := urls[i] 91 | if len(realURL) == 0 { 92 | continue 93 | } 94 | size, err := request.Size(realURL, url) 95 | if err != nil { 96 | return nil, err 97 | } 98 | urlData := &types.Part{ 99 | URL: realURL, 100 | Size: size, 101 | Ext: "mp4", 102 | } 103 | streams[quality] = &types.Stream{ 104 | Parts: []*types.Part{urlData}, 105 | Size: size, 106 | Quality: fmt.Sprintf("%sP", quality), 107 | } 108 | } 109 | 110 | return []*types.Data{ 111 | { 112 | Site: "Pornhub pornhub.com", 113 | Title: title, 114 | Type: types.DataTypeVideo, 115 | Streams: streams, 116 | URL: url, 117 | }, 118 | }, nil 119 | } 120 | -------------------------------------------------------------------------------- /extractors/tangdou/tangdou.go: -------------------------------------------------------------------------------- 1 | package tangdou 2 | 3 | import ( 4 | "github.com/iawia002/annie/extractors/types" 5 | "github.com/iawia002/annie/request" 6 | "github.com/iawia002/annie/utils" 7 | ) 8 | 9 | const referer = "http://www.tangdou.com/html/playlist/view/4173" 10 | 11 | type extractor struct{} 12 | 13 | // New returns a youtube extractor. 14 | func New() types.Extractor { 15 | return &extractor{} 16 | } 17 | 18 | // Extract is the main function to extract the data. 19 | func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { 20 | if !option.Playlist { 21 | return []*types.Data{tangdouDownload(url)}, nil 22 | } 23 | 24 | html, err := request.Get(url, referer, nil) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | videoIDs := utils.MatchAll(html, ``) 30 | needDownloadItems := utils.NeedDownloadList(option.Items, option.ItemStart, option.ItemEnd, len(videoIDs)) 31 | extractedData := make([]*types.Data, len(needDownloadItems)) 32 | wgp := utils.NewWaitGroupPool(option.ThreadNumber) 33 | dataIndex := 0 34 | for index, videoID := range videoIDs { 35 | if !utils.ItemInSlice(index+1, needDownloadItems) || len(videoID) < 2 { 36 | continue 37 | } 38 | wgp.Add() 39 | go func(index int, videURI string, extractedData []*types.Data) { 40 | defer wgp.Done() 41 | extractedData[index] = tangdouDownload(videURI) 42 | }(dataIndex, videoID[1], extractedData) 43 | dataIndex++ 44 | } 45 | wgp.Wait() 46 | return extractedData, nil 47 | } 48 | 49 | // tangdouDownload download function for single url 50 | func tangdouDownload(uri string) *types.Data { 51 | html, err := request.Get(uri, referer, nil) 52 | if err != nil { 53 | return types.EmptyData(uri, err) 54 | } 55 | 56 | titles := utils.MatchOneOf( 57 | html, `
(.+?)
`, `(.+?)`, 58 | ) 59 | if titles == nil || len(titles) < 2 { 60 | return types.EmptyData(uri, types.ErrURLParseFailed) 61 | } 62 | title := titles[1] 63 | 64 | var realURL string 65 | videoURLs := utils.MatchOneOf( 66 | html, `video:'(.+?)'`, `video:"(.+?)"`, `\s*`, 45 | ) 46 | if jsonStrings == nil || len(jsonStrings) < 2 { 47 | return nil, types.ErrURLParseFailed 48 | } 49 | jsonString := jsonStrings[1] 50 | 51 | var totalSize int64 52 | urls := make([]*types.Part, 0, 1) 53 | if strings.Contains(jsonString, `"image":{"@list"`) { 54 | // there are two data structures in the same field(image) 55 | var imageList tumblrImageList 56 | if err := json.Unmarshal([]byte(jsonString), &imageList); err != nil { 57 | return nil, err 58 | } 59 | for _, u := range imageList.Image.List { 60 | urlData, size, err := genURLData(u, url) 61 | if err != nil { 62 | return nil, err 63 | } 64 | totalSize += size 65 | urls = append(urls, urlData) 66 | } 67 | } else { 68 | var image tumblrImage 69 | if err := json.Unmarshal([]byte(jsonString), &image); err != nil { 70 | return nil, err 71 | } 72 | 73 | urlData, size, err := genURLData(image.Image, url) 74 | if err != nil { 75 | return nil, err 76 | } 77 | totalSize = size 78 | urls = append(urls, urlData) 79 | } 80 | streams := map[string]*types.Stream{ 81 | "default": { 82 | Parts: urls, 83 | Size: totalSize, 84 | }, 85 | } 86 | 87 | return []*types.Data{ 88 | { 89 | Site: "Tumblr tumblr.com", 90 | Title: title, 91 | Type: types.DataTypeImage, 92 | Streams: streams, 93 | URL: url, 94 | }, 95 | }, nil 96 | } 97 | 98 | func tumblrVideoDownload(url, html, title string) ([]*types.Data, error) { 99 | videoURLs := utils.MatchOneOf(html, `