├── .gitignore
├── .travis.yml
├── LICENSE
├── cmd
└── piko
│ ├── main.go
│ ├── main_test.go
│ └── utils.go
├── go.mod
├── go.sum
├── readme.md
├── service
├── base.go
├── facebook
│ ├── facebook.go
│ ├── facebook_test.go
│ └── testdata
│ │ └── TestIteratorNext-resp.golden
├── fourchan
│ ├── fourchan.go
│ ├── fourchan_test.go
│ └── testdata
│ │ └── TestIteratorNext-resp.golden
├── imgur
│ ├── imgur.go
│ ├── imgur_test.go
│ └── testdata
│ │ └── TestIteratorNext-resp.golden
├── instagram
│ ├── instagram.go
│ ├── instagram_test.go
│ └── testdata
│ │ └── TestIteratorNext-resp.golden
├── soundcloud
│ ├── soundcloud.go
│ ├── soundcloud_test.go
│ └── testdata
│ │ └── TestIteratorNext-resp.golden
├── testutil
│ └── testutil.go
├── twitter
│ ├── testdata
│ │ └── TestIteratorNext-resp.golden
│ ├── twitter.go
│ ├── twitter_test.go
│ └── utils.go
├── utils.go
└── youtube
│ ├── testdata
│ ├── TestExtractURLs_long_playlist_over_100_videos-resp.golden
│ ├── TestExtractURLs_short_playlist-resp.golden
│ └── TestIteratorNext-resp.golden
│ ├── youtube.go
│ ├── youtube_test.go
│ └── ytdl
│ ├── LICENSE
│ ├── README.md
│ ├── format.go
│ ├── signature.go
│ └── utils.go
└── utils.go
/.gitignore:
--------------------------------------------------------------------------------
1 | downloads
2 | deps.nix
3 | shell.nix
4 | default.nix
5 | .go
6 | .vscode
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: go
2 | go: 1.12.x
3 |
4 | matrix:
5 | fast_finish: true
6 | include:
7 | - os: linux
8 | - os: osx
9 | env: SUFFIX=-macos
10 | - os: windows
11 | env: SUFFIX=.exe
12 |
13 | before_deploy:
14 | - mkdir -p result
15 | - CGO_ENABLED=0 go build -a -ldflags="-s -w" -installsuffix cgo -o "result/piko$SUFFIX" cmd/piko/*
16 | deploy:
17 | provider: releases
18 | api_key:
19 | secure: ZQReulfHiO6PLlwh7RGRg/+hq5D7hpHTioHO13Pb1L2tyv93sRi5+QxpOkP+oKu738MOQucDbh4ZZOCSF+2fJjo4lPASUR9tbKzzjdseCxW6wS8YF+vv+NpTBsTAdZ9+ZpiT7zQfv7N0uLTxiLgHujhdBYJXhVTFQQ60YkEw5V3+rdKYikkQD2dQhHP/bOrinUY7gB1ad/oiufCObsy20pQHWBrAI421wEPqSp36VYxK3lHMxFFCQ6O8A4m/t2eUzLNyb9aCTXQj2HAkZj7AX4eyb5ZxnZq34gSTXU1YtfXiPaChz0VSqVjooMAgCMfXJxIJw0UbLcBSAttXlHZ6dvY+/nU6CgsEigFbLTuo2mRI7Jgrh31ndp9ApsT1Z5FJFdsPNX1IlWIgPOteHXSIvz6nGwGS5NL1c+ZyyAC6O+huShA9sBGaohMdnbt5HRX1D4uESUaudMv9wiABfLGpTlfcBbelexG/Z+qWATpuj6cIkFhvfVV6R65fNKt+U494csh9jrdjMjw7OcNJ/rAOb6mklPNCpc5zcX3Vc0p6rZObBEvBzJPLf9V3gy4I7iAUU5zg2rruapTronEhRcwdRP1uvapKAdL4H7C/2I/aCvsEuChNjkmh1GT6MrYqVv8mmL5rgqcFFoJIEQrq676ldxF9Gbbt1sDP7BhzXl4oilU=
20 | skip_cleanup: true
21 | file_glob: true
22 | file: result/*
23 | on:
24 | repo: mlvzk/piko
25 | tags: true
26 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | GNU LESSER GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 |
4 | Copyright (C) 2007 Free Software Foundation, Inc.
5 | Everyone is permitted to copy and distribute verbatim copies
6 | of this license document, but changing it is not allowed.
7 |
8 |
9 | This version of the GNU Lesser General Public License incorporates
10 | the terms and conditions of version 3 of the GNU General Public
11 | License, supplemented by the additional permissions listed below.
12 |
13 | 0. Additional Definitions.
14 |
15 | As used herein, "this License" refers to version 3 of the GNU Lesser
16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU
17 | General Public License.
18 |
19 | "The Library" refers to a covered work governed by this License,
20 | other than an Application or a Combined Work as defined below.
21 |
22 | An "Application" is any work that makes use of an interface provided
23 | by the Library, but which is not otherwise based on the Library.
24 | Defining a subclass of a class defined by the Library is deemed a mode
25 | of using an interface provided by the Library.
26 |
27 | A "Combined Work" is a work produced by combining or linking an
28 | Application with the Library. The particular version of the Library
29 | with which the Combined Work was made is also called the "Linked
30 | Version".
31 |
32 | The "Minimal Corresponding Source" for a Combined Work means the
33 | Corresponding Source for the Combined Work, excluding any source code
34 | for portions of the Combined Work that, considered in isolation, are
35 | based on the Application, and not on the Linked Version.
36 |
37 | The "Corresponding Application Code" for a Combined Work means the
38 | object code and/or source code for the Application, including any data
39 | and utility programs needed for reproducing the Combined Work from the
40 | Application, but excluding the System Libraries of the Combined Work.
41 |
42 | 1. Exception to Section 3 of the GNU GPL.
43 |
44 | You may convey a covered work under sections 3 and 4 of this License
45 | without being bound by section 3 of the GNU GPL.
46 |
47 | 2. Conveying Modified Versions.
48 |
49 | If you modify a copy of the Library, and, in your modifications, a
50 | facility refers to a function or data to be supplied by an Application
51 | that uses the facility (other than as an argument passed when the
52 | facility is invoked), then you may convey a copy of the modified
53 | version:
54 |
55 | a) under this License, provided that you make a good faith effort to
56 | ensure that, in the event an Application does not supply the
57 | function or data, the facility still operates, and performs
58 | whatever part of its purpose remains meaningful, or
59 |
60 | b) under the GNU GPL, with none of the additional permissions of
61 | this License applicable to that copy.
62 |
63 | 3. Object Code Incorporating Material from Library Header Files.
64 |
65 | The object code form of an Application may incorporate material from
66 | a header file that is part of the Library. You may convey such object
67 | code under terms of your choice, provided that, if the incorporated
68 | material is not limited to numerical parameters, data structure
69 | layouts and accessors, or small macros, inline functions and templates
70 | (ten or fewer lines in length), you do both of the following:
71 |
72 | a) Give prominent notice with each copy of the object code that the
73 | Library is used in it and that the Library and its use are
74 | covered by this License.
75 |
76 | b) Accompany the object code with a copy of the GNU GPL and this license
77 | document.
78 |
79 | 4. Combined Works.
80 |
81 | You may convey a Combined Work under terms of your choice that,
82 | taken together, effectively do not restrict modification of the
83 | portions of the Library contained in the Combined Work and reverse
84 | engineering for debugging such modifications, if you also do each of
85 | the following:
86 |
87 | a) Give prominent notice with each copy of the Combined Work that
88 | the Library is used in it and that the Library and its use are
89 | covered by this License.
90 |
91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license
92 | document.
93 |
94 | c) For a Combined Work that displays copyright notices during
95 | execution, include the copyright notice for the Library among
96 | these notices, as well as a reference directing the user to the
97 | copies of the GNU GPL and this license document.
98 |
99 | d) Do one of the following:
100 |
101 | 0) Convey the Minimal Corresponding Source under the terms of this
102 | License, and the Corresponding Application Code in a form
103 | suitable for, and under terms that permit, the user to
104 | recombine or relink the Application with a modified version of
105 | the Linked Version to produce a modified Combined Work, in the
106 | manner specified by section 6 of the GNU GPL for conveying
107 | Corresponding Source.
108 |
109 | 1) Use a suitable shared library mechanism for linking with the
110 | Library. A suitable mechanism is one that (a) uses at run time
111 | a copy of the Library already present on the user's computer
112 | system, and (b) will operate properly with a modified version
113 | of the Library that is interface-compatible with the Linked
114 | Version.
115 |
116 | e) Provide Installation Information, but only if you would otherwise
117 | be required to provide such information under section 6 of the
118 | GNU GPL, and only to the extent that such information is
119 | necessary to install and execute a modified version of the
120 | Combined Work produced by recombining or relinking the
121 | Application with a modified version of the Linked Version. (If
122 | you use option 4d0, the Installation Information must accompany
123 | the Minimal Corresponding Source and Corresponding Application
124 | Code. If you use option 4d1, you must provide the Installation
125 | Information in the manner specified by section 6 of the GNU GPL
126 | for conveying Corresponding Source.)
127 |
128 | 5. Combined Libraries.
129 |
130 | You may place library facilities that are a work based on the
131 | Library side by side in a single library together with other library
132 | facilities that are not Applications and are not covered by this
133 | License, and convey such a combined library under terms of your
134 | choice, if you do both of the following:
135 |
136 | a) Accompany the combined library with a copy of the same work based
137 | on the Library, uncombined with any other library facilities,
138 | conveyed under the terms of this License.
139 |
140 | b) Give prominent notice with the combined library that part of it
141 | is a work based on the Library, and explaining where to find the
142 | accompanying uncombined form of the same work.
143 |
144 | 6. Revised Versions of the GNU Lesser General Public License.
145 |
146 | The Free Software Foundation may publish revised and/or new versions
147 | of the GNU Lesser General Public License from time to time. Such new
148 | versions will be similar in spirit to the present version, but may
149 | differ in detail to address new problems or concerns.
150 |
151 | Each version is given a distinguishing version number. If the
152 | Library as you received it specifies that a certain numbered version
153 | of the GNU Lesser General Public License "or any later version"
154 | applies to it, you have the option of following the terms and
155 | conditions either of that published version or of any later version
156 | published by the Free Software Foundation. If the Library as you
157 | received it does not specify a version number of the GNU Lesser
158 | General Public License, you may choose any version of the GNU Lesser
159 | General Public License ever published by the Free Software Foundation.
160 |
161 | If the Library as you received it specifies that a proxy can decide
162 | whether future versions of the GNU Lesser General Public License shall
163 | apply, that proxy's public statement of acceptance of any version is
164 | permanent authorization for you to choose that version for the
165 | Library.
--------------------------------------------------------------------------------
/cmd/piko/main.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package main
18 |
19 | import (
20 | "fmt"
21 | "io"
22 | "log"
23 | "os"
24 | "path/filepath"
25 | "reflect"
26 | "strings"
27 |
28 | "github.com/mlvzk/piko"
29 | "github.com/mlvzk/piko/service"
30 | "github.com/mlvzk/qtils/commandparser"
31 | "github.com/mlvzk/qtils/commandparser/commandhelper"
32 | "gopkg.in/cheggaaa/pb.v1"
33 | )
34 |
35 | var (
36 | discoveryMode bool
37 | stdoutMode bool
38 | formatStr string
39 | targets []string
40 | userOptions = map[string]string{}
41 | )
42 |
43 | func handleArgv(argv []string) {
44 | parser := commandparser.New()
45 | helper := commandhelper.New()
46 |
47 | helper.SetName("piko")
48 | helper.SetVersion("alpha")
49 |
50 | helper.AddUsage(
51 | "piko [urls...]",
52 | "piko 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'",
53 | "piko 'https://www.youtube.com/watch?v=dQw4w9WgXcQ' --stdout | mpv -",
54 | )
55 |
56 | parser.AddOption(helper.EatOption(
57 | commandhelper.NewOption("help").Alias("h").Boolean().Description("Prints this page"),
58 | commandhelper.
59 | NewOption("format").
60 | Alias("f").
61 | Description(`File path format, ex: --format %[id].%[ext]. "id" and "ext" are meta tags(see --discover).
62 | Use %[default] to fill with default format, ex: downloads/%[default]`),
63 | commandhelper.
64 | NewOption("option").
65 | Alias("o").
66 | Arrayed().
67 | ValidateBind(commandhelper.ValidateKeyValue("=")).
68 | Description("Download options, ex: --option quality=best"),
69 | commandhelper.
70 | NewOption("discover").
71 | Alias("d").
72 | Boolean().
73 | Description("Discovery mode, doesn't download anything, only outputs information"),
74 | commandhelper.NewOption("stdout").Boolean().Description("Output download media to stdout"),
75 | )...)
76 |
77 | cmd, err := parser.Parse(argv)
78 | if err != nil {
79 | log.Println(err)
80 | os.Exit(1)
81 | }
82 |
83 | if cmd.Booleans["help"] || len(cmd.Positionals) == 0 {
84 | fmt.Print(helper.Help())
85 | os.Exit(1)
86 | }
87 |
88 | cmd.Args = helper.FillDefaults(cmd.Args)
89 | errs := helper.Verify(cmd.Args, cmd.Arrayed)
90 | for _, err := range errs {
91 | log.Println(err)
92 | }
93 | if len(errs) != 0 {
94 | os.Exit(1)
95 | }
96 |
97 | formatStr = cmd.Args["format"]
98 | discoveryMode = cmd.Booleans["discover"]
99 | stdoutMode = cmd.Booleans["stdout"]
100 |
101 | for _, option := range cmd.Arrayed["option"] {
102 | keyValue := strings.Split(option, "=")
103 | key, value := keyValue[0], keyValue[1]
104 |
105 | userOptions[key] = value
106 | }
107 |
108 | targets = cmd.Positionals
109 | }
110 |
111 | func main() {
112 | // handleArgv can not be in init()
113 | // because it would be called(and errored)
114 | // if tests were run in main_test
115 | handleArgv(os.Args)
116 |
117 | services := piko.GetAllServices()
118 |
119 | // targets = append(targets, "https://boards.4channel.org/adv/thread/20765545/i-want-to-be-the-very-best-like-no-one-ever-was")
120 | // targets = append(targets, "https://imgur.com/t/article13/EfY6CxU")
121 | // targets = append(targets, "https://www.youtube.com/watch?v=Gs069dndIYk")
122 | // targets = append(targets, "https://www.youtube.com/watch?v=7IwYakbxmxo")
123 | // targets = append(targets, "https://www.instagram.com/p/Bv9MJCsAvZV/")
124 | // targets = append(targets, "https://soundcloud.com/musicpromouser/mac-miller-ok-ft-tyler-the-creator")
125 | // targets = append(targets, "https://twitter.com/deadprogram/status/1090554988768698368")
126 | // targets = append(targets, "https://www.facebook.com/groups/veryblessedimages/permalink/478153699389793/")
127 |
128 | for _, target := range targets {
129 | if target == "" {
130 | log.Println("Target can't be empty")
131 | continue
132 | }
133 |
134 | var foundAnyService bool
135 | for _, s := range services {
136 | if !s.IsValidTarget(target) {
137 | continue
138 | }
139 |
140 | foundAnyService = true
141 | log.Printf("Found valid service: %s\n", reflect.TypeOf(s).Name())
142 | iterator, err := s.FetchItems(target)
143 | if err != nil {
144 | log.Printf("failed to fetch items: %v; target: %v\n", err, target)
145 | break
146 | }
147 |
148 | for !iterator.HasEnded() {
149 | items, err := iterator.Next()
150 | if err != nil {
151 | log.Printf("Iteration error: %v; target: %v\n", err, target)
152 | continue
153 | }
154 |
155 | for _, item := range items {
156 | handleItem(s, item)
157 | }
158 | }
159 |
160 | break
161 | }
162 | if !foundAnyService {
163 | log.Printf("Error: Couldn't find a valid service for url '%s'. Your link is probably unsupported.\n", target)
164 | }
165 | }
166 | }
167 |
168 | func handleItem(s service.Service, item service.Item) {
169 | if discoveryMode {
170 | log.Println("Item:\n" + prettyPrintItem(item))
171 | return
172 | }
173 |
174 | options := mergeStringMaps(item.DefaultOptions, userOptions)
175 |
176 | reader, err := s.Download(item.Meta, options)
177 | if err != nil {
178 | log.Printf("Download error: %v, item: %+v\n", err, item)
179 | return
180 | }
181 | defer tryClose(reader)
182 |
183 | if stdoutMode {
184 | io.Copy(os.Stdout, reader)
185 | return
186 | }
187 |
188 | nameFormat := item.DefaultName
189 | if formatStr != "" {
190 | nameFormat = strings.Replace(formatStr, "%[default]", item.DefaultName, -1)
191 | }
192 | name := format(nameFormat, item.Meta)
193 |
194 | if sizedIO, ok := reader.(service.Sized); ok {
195 | bar := pb.New64(int64(sizedIO.Size())).SetUnits(pb.U_BYTES)
196 | bar.Prefix(truncateString(name, 25))
197 | bar.Start()
198 | defer bar.Finish()
199 |
200 | reader = bar.NewProxyReader(reader)
201 | }
202 |
203 | dir := filepath.Dir(name)
204 | err = os.MkdirAll(dir, os.ModePerm)
205 | if err != nil {
206 | log.Printf("Error creating directory: %v; dir: '%v'\n", err, dir)
207 | return
208 | }
209 |
210 | file, err := os.Create(name)
211 | if err != nil {
212 | log.Printf("Error creating file: %v, name: %v\n", err, name)
213 | return
214 | }
215 | defer file.Close()
216 |
217 | _, err = io.Copy(file, reader)
218 | if err != nil {
219 | log.Printf("Error copying from source to file: %v, item: %+v", err, item)
220 | return
221 | }
222 | }
223 |
--------------------------------------------------------------------------------
/cmd/piko/main_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package main
18 |
19 | import (
20 | "testing"
21 | )
22 |
23 | func TestFormat(t *testing.T) {
24 | cases := []struct {
25 | name string
26 | formatStr string
27 | meta map[string]string
28 | expected string
29 | }{
30 | {"empty meta", "%[unknown].png", map[string]string{}, "unknown.png"},
31 | {"one in meta", "ab%[id].jpg", map[string]string{"id": "456"}, "ab456.jpg"},
32 | {"one in meta, double usage", "%[id].%[id].jpg", map[string]string{"id": "456"}, "456.456.jpg"},
33 | {"two in meta", "ab%[id]c%[name]d.jpg", map[string]string{
34 | "id": "123",
35 | "name": "test",
36 | }, "ab123ctestd.jpg"},
37 | }
38 |
39 | for _, c := range cases {
40 | t.Run(c.name, func(t *testing.T) {
41 | result := format(c.formatStr, c.meta)
42 | if result != c.expected {
43 | t.Errorf("Format error, got: %s, expected: %s\n", result, c.expected)
44 | }
45 | })
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/cmd/piko/utils.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package main
18 |
19 | import (
20 | "fmt"
21 | "io"
22 | "regexp"
23 | "strings"
24 |
25 | "github.com/mlvzk/piko/service"
26 | )
27 |
28 | var formatRegexp = regexp.MustCompile(`%\[[[:alnum:]]*\]`)
29 |
30 | func format(formatter string, meta map[string]string) string {
31 | return formatRegexp.ReplaceAllStringFunc(formatter, func(str string) string {
32 | // remove "%[" and "]"
33 | key := str[2 : len(str)-1]
34 |
35 | v, ok := meta[key]
36 | if !ok {
37 | return key
38 | }
39 |
40 | return sanitizeFileName(v)
41 | })
42 | }
43 |
44 | // order of args matters
45 | func mergeStringMaps(maps ...map[string]string) map[string]string {
46 | merged := map[string]string{}
47 |
48 | for _, m := range maps {
49 | for k, v := range m {
50 | merged[k] = v
51 | }
52 | }
53 |
54 | return merged
55 | }
56 |
57 | func tryClose(reader interface{}) error {
58 | if closer, ok := reader.(io.Closer); ok {
59 | return closer.Close()
60 | }
61 |
62 | return nil
63 | }
64 |
65 | var seps = regexp.MustCompile(`[\r\n &_=+:/']`)
66 |
67 | func sanitizeFileName(name string) string {
68 | name = strings.TrimSpace(name)
69 | name = seps.ReplaceAllString(name, "-")
70 |
71 | return name
72 | }
73 |
74 | func prettyPrintItem(item service.Item) string {
75 | builder := strings.Builder{}
76 |
77 | builder.WriteString("Default Name: " + item.DefaultName + "\n")
78 |
79 | builder.WriteString("Meta:\n")
80 | for k, v := range item.Meta {
81 | if k[0] == '_' {
82 | continue
83 | }
84 |
85 | builder.WriteString(fmt.Sprintf("\t%s=%s\n", k, v))
86 | }
87 |
88 | builder.WriteString("Available Options:\n")
89 | for key, values := range item.AvailableOptions {
90 | builder.WriteString("\t" + key + ":\n")
91 | for _, v := range values {
92 | builder.WriteString("\t\t- " + v + "\n")
93 | }
94 | }
95 |
96 | builder.WriteString("Default Options:\n")
97 | for k, v := range item.DefaultOptions {
98 | builder.WriteString(fmt.Sprintf("\t%s=%s\n", k, v))
99 | }
100 |
101 | return builder.String()
102 | }
103 |
104 | func truncateString(str string, limit int) string {
105 | if len(str)-3 <= limit {
106 | return str
107 | }
108 |
109 | return string([]rune(str)[:limit]) + "..."
110 | }
111 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/mlvzk/piko
2 |
3 | go 1.12
4 |
5 | require (
6 | github.com/PuerkitoBio/goquery v1.5.0
7 | github.com/fatih/color v1.7.0 // indirect
8 | github.com/kylelemons/godebug v1.1.0
9 | github.com/mattn/go-colorable v0.1.1 // indirect
10 | github.com/mattn/go-isatty v0.0.8 // indirect
11 | github.com/mattn/go-runewidth v0.0.4 // indirect
12 | github.com/mlvzk/qtils v0.4.1
13 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect
14 | golang.org/x/net v0.0.0-20190522135303-fa69b94a3b58 // indirect
15 | golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 // indirect
16 | golang.org/x/text v0.3.2 // indirect
17 | golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd // indirect
18 | gopkg.in/cheggaaa/pb.v1 v1.0.28
19 | )
20 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
2 | github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
3 | github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
4 | github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
5 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
6 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
7 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
8 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
9 | github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
10 | github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
11 | github.com/kylelemons/godebug v1.0.0 h1:CzXccTNDr74vi1ciak1YZkJ36Uk8LBAuYAJRcq10SNk=
12 | github.com/kylelemons/godebug v1.0.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
13 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
14 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
15 | github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
16 | github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
17 | github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
18 | github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
19 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
20 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
21 | github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
22 | github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
23 | github.com/mlvzk/qtils v0.3.1 h1:+hzZ9f2/ZBfcdFKOUfoChTJTbizQod69ehp7dJKul2Y=
24 | github.com/mlvzk/qtils v0.3.1/go.mod h1:BmDUojjYAjRb5LyaMyhTk3TZk+P3uTbED6CQ+HONhqQ=
25 | github.com/mlvzk/qtils v0.4.0 h1:mpuXJNVXvHgPu70hrfVAVbf2MzjWAnq0N3R48nl05Ck=
26 | github.com/mlvzk/qtils v0.4.0/go.mod h1:BmDUojjYAjRb5LyaMyhTk3TZk+P3uTbED6CQ+HONhqQ=
27 | github.com/mlvzk/qtils v0.4.1 h1:B1KafnFhTLjYz55VCYoSL3P9V8DoxRjoPEK6Und5tRU=
28 | github.com/mlvzk/qtils v0.4.1/go.mod h1:BmDUojjYAjRb5LyaMyhTk3TZk+P3uTbED6CQ+HONhqQ=
29 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
30 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
31 | golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
32 | golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
33 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
34 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
35 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
36 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
37 | golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI=
38 | golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
39 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
40 | golang.org/x/net v0.0.0-20190419010253-1f3472d942ba h1:h0zCzEL5UW1mERvwTN6AXcc75PpLkY6OcReia6Dq1BM=
41 | golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
42 | golang.org/x/net v0.0.0-20190522135303-fa69b94a3b58 h1:AZ8FNE2w7DVDFDK6u/iC9/Mqh73UupjaqSd/2qMoECQ=
43 | golang.org/x/net v0.0.0-20190522135303-fa69b94a3b58/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
44 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
45 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
46 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
47 | golang.org/x/sys v0.0.0-20190329044733-9eb1bfa1ce65 h1:hOY+O8MxdkPV10pNf7/XEHaySCiPKxixMKUshfHsGn0=
48 | golang.org/x/sys v0.0.0-20190329044733-9eb1bfa1ce65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
49 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
50 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
51 | golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be h1:mI+jhqkn68ybP0ORJqunXn+fq+Eeb4hHKqLQcFICjAc=
52 | golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
53 | golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 h1:f005F/Jl5JLP036x7QIvUVhNTqxvSYwFIiyOh2q12iU=
54 | golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
55 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
56 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
57 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
58 | golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
59 | gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
60 | gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
61 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # piko
2 |
3 | Light and simple media downloader with support for:
4 | - Youtube - single /watch?v= videos and playlists(only 100 first videos)
5 | - Soundcloud - single songs
6 | - Imgur - albums
7 | - Facebook - single and multiple images/videos in one post
8 | - Twitter - \*/status/\* links, single and multiple images/videos of single posts
9 | - Instagram - single and multiple images of single posts
10 | - 4chan - all images and videos of a thread and it's posts
11 |
12 | TODO:
13 | - Twitch - recording livestreams and watching with `-stdout | mpv -`
14 | - Youtube - support more than 100 videos in playlists(might need API key which has quota limit)
15 | - Soundcloud - support playlists
16 | - Facebook - support downloading all images/videos posted by a page
17 | - Twitter - support downloading all images/videos posted by an account
18 | - Instagram - support downloading videos and downloading all images/videos of an account
19 | - 4chan - support downloading all images of alive threads in a board
20 |
21 | # Installation
22 |
23 | Grab the executable for your OS from [the releases page of this repository](https://github.com/mlvzk/piko/releases) and put it in your $PATH (on Linux typically `/usr/bin/`)
24 |
25 | Or from source:
26 | ```sh
27 | go get -u github.com/mlvzk/piko/...
28 | ```
29 |
30 | ## Optional dependencies
31 |
32 | - ffmpeg (youtube, for better video and audio quality)
33 |
34 | # Usage
35 |
36 | ```sh
37 | piko --help
38 | piko [urls...]
39 | ```
40 |
41 | # Examples
42 |
43 | ```sh
44 | # downloads the video(with audio) to a file with default name format (see --discover example below)
45 | # if ffmpeg is not installed, the quality might be bad
46 | piko 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
47 | ```
48 |
49 | ```sh
50 | # tell youtube service to choose best quality
51 | # save to file with name format %[title].%[ext] (see --discover example below)
52 | piko --option quality=best --format "%[title].%[ext]" 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
53 | ```
54 |
55 | ```sh
56 | # you can discover options and meta information for formats with --discover flag
57 | piko --discover 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
58 |
59 | # output:
60 | Default Name: %[title].%[ext]
61 | Meta:
62 | ext=mkv
63 | title=Rick Astley - Never Gonna Give You Up (Video)
64 | author=RickAstleyVEVO
65 | Available Options:
66 | onlyAudio:
67 | - yes
68 | - no
69 | quality:
70 | - best
71 | - medium
72 | - worst
73 | useFfmpeg:
74 | - yes
75 | - no
76 | Default Options:
77 | useFfmpeg=yes
78 | onlyAudio=no
79 | quality=medium
80 | ```
81 |
82 | ```sh
83 | # output to stdout, pipe to mpv which reads from stdin
84 | piko 'https://www.youtube.com/watch?v=dQw4w9WgXcQ' --stdout | mpv -
85 | ```
86 |
87 | ```sh
88 | # download only audio, with best quality, output to stdout, pipe to mpv which reads from stdin
89 | piko --option onlyAudio=yes --option quality=best 'https://www.youtube.com/watch?v=dQw4w9WgXcQ' --stdout | mpv -
90 | ```
91 |
92 | # Contributors
93 |
94 | - [mlvzk](https://github.com/mlvzk) - creator and maintainer
95 | - [gwu](https://github.com/gwimm) - the idea came from gwu's [shovel](http://gitlab.com/gwu/shovel) project
96 |
--------------------------------------------------------------------------------
/service/base.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package service
18 |
19 | import "io"
20 |
21 | type Item struct {
22 | Meta map[string]string
23 | DefaultName string
24 | AvailableOptions map[string]([]string)
25 | DefaultOptions map[string]string
26 | }
27 |
28 | type ServiceIterator interface {
29 | Next() ([]Item, error)
30 | HasEnded() bool
31 | }
32 |
33 | type Service interface {
34 | IsValidTarget(target string) bool
35 | FetchItems(target string) (ServiceIterator, error)
36 | Download(meta, options map[string]string) (io.Reader, error)
37 | }
38 |
39 | type Sized interface {
40 | Size() uint64
41 | }
42 |
--------------------------------------------------------------------------------
/service/facebook/facebook.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package facebook
18 |
19 | import (
20 | "bytes"
21 | "errors"
22 | "fmt"
23 | "io"
24 | "io/ioutil"
25 | "net/http"
26 | "net/url"
27 | "path/filepath"
28 | "regexp"
29 | "strings"
30 |
31 | "github.com/PuerkitoBio/goquery"
32 | "github.com/mlvzk/piko/service"
33 | )
34 |
35 | type output struct {
36 | io.ReadCloser
37 | length uint64
38 | }
39 |
40 | func (o output) Size() uint64 {
41 | return o.length
42 | }
43 |
44 | type Facebook struct{}
45 | type FacebookIterator struct {
46 | url string
47 | end bool
48 | }
49 |
50 | func New() Facebook {
51 | return Facebook{}
52 | }
53 |
54 | func (s Facebook) IsValidTarget(target string) bool {
55 | return strings.Contains(target, "facebook.com/")
56 | }
57 |
58 | func (s Facebook) FetchItems(target string) (service.ServiceIterator, error) {
59 | return &FacebookIterator{
60 | url: target,
61 | }, nil
62 | }
63 |
64 | func (s Facebook) Download(meta, options map[string]string) (io.Reader, error) {
65 | downloadURL, hasDownloadURL := meta["downloadURL"]
66 | if !hasDownloadURL {
67 | return nil, errors.New("Missing meta downloadURL")
68 | }
69 |
70 | res, err := http.Get(downloadURL)
71 | if err != nil {
72 | return nil, err
73 | }
74 |
75 | if res.ContentLength == -1 {
76 | return res.Body, nil
77 | }
78 |
79 | return output{
80 | ReadCloser: res.Body,
81 | length: uint64(res.ContentLength),
82 | }, nil
83 | }
84 |
85 | var srcRegexp = regexp.MustCompile(`(sd|hd)_src:"(.+?)"`)
86 |
87 | func (i *FacebookIterator) Next() ([]service.Item, error) {
88 | i.end = true
89 |
90 | resp, err := http.Get(i.url)
91 | if err != nil {
92 | return nil, err
93 | }
94 | defer resp.Body.Close()
95 | if resp.StatusCode != 200 {
96 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", i.url, resp.StatusCode)
97 | }
98 |
99 | bodyBytes, err := ioutil.ReadAll(resp.Body)
100 | if err != nil {
101 | return nil, err
102 | }
103 |
104 | doc, err := goquery.NewDocumentFromReader(bytes.NewReader(bodyBytes))
105 | if err != nil {
106 | return nil, err
107 | }
108 |
109 | title := doc.Find(`title`).Text()
110 | description, _ := doc.Find(`meta[name="description"]`).Attr("content")
111 | image, hasImage := doc.Find(`meta[property="og:image"]`).Attr("content")
112 |
113 | var author string
114 | titleParts := strings.Split(title, " - ")
115 | if len(titleParts) > 0 {
116 | author = titleParts[len(titleParts)-1]
117 | }
118 |
119 | var bestVideo, worstVideo string
120 | matches := srcRegexp.FindAllSubmatch(bodyBytes, -1)
121 | if matches != nil {
122 | for _, match := range matches {
123 | if len(match) < 3 {
124 | continue
125 | }
126 |
127 | if string(match[1]) == "hd" {
128 | bestVideo = string(match[len(match)-1])
129 | } else {
130 | worstVideo = string(match[len(match)-1])
131 | }
132 | }
133 | }
134 |
135 | items := []service.Item{}
136 |
137 | if hasImage && image != "" {
138 | imageURL, err := url.Parse(image)
139 | if err != nil {
140 | return nil, err
141 | }
142 | pathParts := strings.Split(imageURL.Path, "_")
143 | id := ""
144 | if len(pathParts) > 2 {
145 | id = pathParts[2]
146 | }
147 |
148 | items = append(items, service.Item{
149 | Meta: map[string]string{
150 | "id": id,
151 | "author": author,
152 | "description": description,
153 | "type": "image",
154 | "ext": "jpg",
155 | "downloadURL": image,
156 | },
157 | DefaultName: "%[author]-%[id].%[ext]",
158 | })
159 | }
160 |
161 | if bestVideo != "" || worstVideo != "" {
162 | video := bestVideo
163 | if video == "" {
164 | video = worstVideo
165 | }
166 |
167 | videoURL, err := url.Parse(video)
168 | if err != nil {
169 | return nil, err
170 | }
171 |
172 | id := ""
173 | if pathParts := strings.Split(videoURL.Path, "_"); len(pathParts) > 2 {
174 | id = pathParts[2]
175 | }
176 |
177 | items = append(items, service.Item{
178 | Meta: map[string]string{
179 | "id": id,
180 | "author": author,
181 | "description": description,
182 | "type": "video",
183 | "ext": "mp4",
184 | "downloadURL": video,
185 | },
186 | DefaultName: "%[author]-%[id].%[ext]",
187 | })
188 | }
189 |
190 | // don't care if this fails
191 | // this code fetches all images from album
192 | func() {
193 | hiddenElemText, found := "", false
194 | doc.Find("div.hidden_elem code").Each(func(_ int, sel *goquery.Selection) {
195 | selText, err := sel.Html()
196 | if err != nil {
197 | return
198 | }
199 |
200 | if !strings.Contains(selText, `rel="theater"`) {
201 | return
202 | }
203 |
204 | hiddenElemText, found = selText, true
205 | })
206 |
207 | if !found {
208 | return
209 | }
210 |
211 | uncommented := hiddenElemText[4 : len(hiddenElemText)-3]
212 | hiddenElem, err := goquery.NewDocumentFromReader(strings.NewReader(uncommented))
213 | if err != nil {
214 | return
215 | }
216 |
217 | hiddenElem.Find(`a[rel="theater"]`).Each(func(_ int, sel *goquery.Selection) {
218 | media, exists := sel.Attr("data-ploi")
219 | if !exists {
220 | return
221 | }
222 |
223 | mediaURL, err := url.Parse(media)
224 | if err != nil {
225 | return
226 | }
227 |
228 | id := ""
229 | if pathParts := strings.Split(mediaURL.Path, "_"); len(pathParts) > 2 {
230 | id = pathParts[2]
231 | }
232 |
233 | ext := filepath.Ext(mediaURL.Path)
234 | if len(ext) > 0 {
235 | // cut the dot
236 | ext = ext[1:]
237 | }
238 |
239 | items = append(items, service.Item{
240 | Meta: map[string]string{
241 | "id": id,
242 | "author": author,
243 | "description": description,
244 | "type": "unknown",
245 | "ext": ext,
246 | "downloadURL": media,
247 | },
248 | DefaultName: "%[author]-%[id].%[ext]",
249 | })
250 | })
251 | }()
252 |
253 | uniqueItems := []service.Item{}
254 | itemIterator:
255 | for _, item := range items {
256 | for _, uniqueItem := range uniqueItems {
257 | if item.Meta["id"] == uniqueItem.Meta["id"] {
258 | continue itemIterator
259 | }
260 | }
261 |
262 | uniqueItems = append(uniqueItems, item)
263 | }
264 |
265 | return uniqueItems, nil
266 | }
267 |
268 | func (i FacebookIterator) HasEnded() bool {
269 | return i.end
270 | }
271 |
--------------------------------------------------------------------------------
/service/facebook/facebook_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package facebook
18 |
19 | import (
20 | "flag"
21 | "strings"
22 | "testing"
23 |
24 | "github.com/kylelemons/godebug/pretty"
25 | "github.com/mlvzk/piko/service"
26 | "github.com/mlvzk/piko/service/testutil"
27 | )
28 |
29 | const base = "https://facebook.com"
30 |
31 | var update = flag.Bool("update", false, "update .golden files")
32 |
33 | func TestIsValidTarget(t *testing.T) {
34 | tests := map[string]bool{
35 | "https://www.facebook.com/Shiba.Zero.Mika/videos/414355892680582": true,
36 | "https://www.facebook.com/Shiba.Zero.Mika": true,
37 | "facebook.com/Shiba.Zero.Mika/videos/414355892680582": true,
38 | "https://twitter.com/": false,
39 | }
40 |
41 | for target, expected := range tests {
42 | if (Facebook{}).IsValidTarget(target) != expected {
43 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
44 | }
45 | }
46 | }
47 | func TestIteratorNext(t *testing.T) {
48 | ts := testutil.CacheHttpRequest(t, base, *update)
49 | defer ts.Close()
50 |
51 | iterator := FacebookIterator{
52 | url: ts.URL + "/Shiba.Zero.Mika/videos/414355892680582",
53 | }
54 |
55 | items, err := iterator.Next()
56 | if err != nil {
57 | t.Fatalf("iterator.Next() error: %v", err)
58 | }
59 |
60 | if len(items) < 1 {
61 | t.Fatalf("Items array is empty")
62 | }
63 |
64 | for _, item := range items {
65 | item.Meta["id"] = "ignore"
66 | if !strings.Contains(item.Meta["downloadURL"], "fbcdn.net") {
67 | t.Fatalf("Incorrect downloadURL: %s", item.Meta["downloadURL"])
68 | }
69 | item.Meta["downloadURL"] = "ignore"
70 | }
71 |
72 | expected := []service.Item{
73 | {
74 | Meta: map[string]string{
75 | "id": "ignore",
76 | "author": "Shiba Inu Zero.Mika",
77 | "description": "早晨啊🌼今早傻波在睡夢中又滾了下床😅之後起身扮作若無其事地再上床睡😂\n#柴犬 #shiba #zeromika #shibazeromika",
78 | "ext": "mp4",
79 | "type": "video",
80 | "downloadURL": "ignore",
81 | },
82 | DefaultName: "%[author]-%[id].%[ext]",
83 | },
84 | }
85 |
86 | if diff := pretty.Compare(items, expected); diff != "" {
87 | t.Errorf("%s diff:\n%s", t.Name(), diff)
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/service/fourchan/fourchan.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package fourchan
18 |
19 | import (
20 | "fmt"
21 | "io"
22 | "net/http"
23 | "strings"
24 |
25 | "github.com/PuerkitoBio/goquery"
26 | "github.com/mlvzk/piko/service"
27 | )
28 |
29 | type Fourchan struct{}
30 | type FourchanIterator struct {
31 | url string
32 | page int
33 | end bool
34 | }
35 |
36 | func New() Fourchan {
37 | return Fourchan{}
38 | }
39 |
40 | type output struct {
41 | io.ReadCloser
42 | length uint64
43 | }
44 |
45 | func (o output) Size() uint64 {
46 | return o.length
47 | }
48 |
49 | func (s Fourchan) IsValidTarget(target string) bool {
50 | return strings.Contains(target, "4chan.org/") || strings.Contains(target, "4channel.org/")
51 | }
52 |
53 | func (s Fourchan) FetchItems(target string) (service.ServiceIterator, error) {
54 | return &FourchanIterator{
55 | url: target,
56 | page: 1,
57 | end: false,
58 | }, nil
59 | }
60 |
61 | func (s Fourchan) Download(meta, options map[string]string) (io.Reader, error) {
62 | url := meta["imgURL"]
63 | if options["thumbnail"] == "yes" {
64 | url = meta["thumbnailURL"]
65 | }
66 |
67 | resp, err := http.Get(url)
68 | if err != nil {
69 | return nil, err
70 | }
71 |
72 | if resp.ContentLength == -1 {
73 | return resp.Body, nil
74 | }
75 |
76 | return output{
77 | ReadCloser: resp.Body,
78 | length: uint64(resp.ContentLength),
79 | }, nil
80 | }
81 |
82 | func (i *FourchanIterator) Next() ([]service.Item, error) {
83 | i.end = true
84 |
85 | resp, err := http.Get(i.url)
86 | if err != nil {
87 | return nil, err
88 | }
89 | defer resp.Body.Close()
90 | if resp.StatusCode != 200 {
91 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", i.url, resp.StatusCode)
92 | }
93 |
94 | doc, err := goquery.NewDocumentFromReader(resp.Body)
95 | if err != nil {
96 | return nil, err
97 | }
98 |
99 | items := []service.Item{}
100 | doc.Find("div.file").Each(func(_ int, sel *goquery.Selection) {
101 | imgURL, iuExists := sel.Find("a.fileThumb").Attr("href")
102 | if !iuExists {
103 | return
104 | }
105 | if imgURL[0] == '/' {
106 | imgURL = "https:" + imgURL
107 | }
108 |
109 | thumbnailURL, _ := sel.Find("a.fileThumb img").Attr("src")
110 | if thumbnailURL[0] == '/' {
111 | thumbnailURL = "https:" + thumbnailURL
112 | }
113 |
114 | titleSel := sel.Find("div.fileText a")
115 | title, titleExists := titleSel.Attr("title")
116 | if !titleExists || strings.TrimSpace(title) == "" {
117 | title = titleSel.Text()
118 | }
119 |
120 | dotParts := strings.Split(imgURL, ".")
121 | ext := dotParts[len(dotParts)-1]
122 |
123 | slashParts := strings.Split(imgURL, "/")
124 | lastSlashDotParts := strings.Split(slashParts[len(slashParts)-1], ".")
125 | id := lastSlashDotParts[0]
126 |
127 | items = append(items, service.Item{
128 | Meta: map[string]string{
129 | "title": title,
130 | "imgURL": imgURL,
131 | "id": id,
132 | "ext": ext,
133 | "thumbnailURL": thumbnailURL,
134 | },
135 | DefaultName: "%[title]",
136 | AvailableOptions: map[string][]string{
137 | "thumbnail": {"yes", "no"},
138 | },
139 | DefaultOptions: map[string]string{
140 | "thumbnail": "no",
141 | },
142 | })
143 | })
144 |
145 | return items, nil
146 | }
147 |
148 | func (i FourchanIterator) HasEnded() bool {
149 | return i.end
150 | }
151 |
--------------------------------------------------------------------------------
/service/fourchan/fourchan_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package fourchan
18 |
19 | import (
20 | "flag"
21 | "testing"
22 |
23 | "github.com/kylelemons/godebug/pretty"
24 | "github.com/mlvzk/piko/service"
25 | "github.com/mlvzk/piko/service/testutil"
26 | )
27 |
28 | const base = "https://boards.4channel.org"
29 |
30 | var update = flag.Bool("update", false, "update .golden files")
31 |
32 | func TestFourchanIsValidTarget(t *testing.T) {
33 | tests := map[string]bool{
34 | "https://boards.4channel.org/g/": true,
35 | "boards.4channel.org/g/": true,
36 | "https://boards.4channel.org/g/thread/70377765/hpg-esg-headphone-general": true,
37 | "https://boards.4chan.org/pol/": true,
38 | "https://imgur.com/": false,
39 | }
40 |
41 | for target, expected := range tests {
42 | if (Fourchan{}).IsValidTarget(target) != expected {
43 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
44 | }
45 | }
46 | }
47 |
48 | func TestIteratorNext(t *testing.T) {
49 | ts := testutil.CacheHttpRequest(t, base, *update)
50 | defer ts.Close()
51 |
52 | iterator := FourchanIterator{
53 | url: ts.URL + "/vip/thread/88504",
54 | }
55 |
56 | items, err := iterator.Next()
57 | if err != nil {
58 | t.Fatalf("iterator.Next() error: %v", err)
59 | }
60 |
61 | expected := []service.Item{
62 | {
63 | Meta: map[string]string{
64 | "title": "F.png",
65 | "imgURL": "https://is2.4chan.org/vip/1546227263937.png",
66 | "id": "1546227263937",
67 | "ext": "png",
68 | "thumbnailURL": "https://i.4cdn.org/vip/1546227263937s.jpg",
69 | },
70 | DefaultName: "%[title]",
71 | AvailableOptions: map[string][]string{
72 | "thumbnail": {"yes", "no"},
73 | },
74 | DefaultOptions: map[string]string{
75 | "thumbnail": "no",
76 | },
77 | },
78 | {
79 | Meta: map[string]string{
80 | "title": "1545804746249.jpg",
81 | "imgURL": "https://is2.4chan.org/vip/1546318308248.jpg",
82 | "id": "1546318308248",
83 | "ext": "jpg",
84 | "thumbnailURL": "https://i.4cdn.org/vip/1546318308248s.jpg",
85 | },
86 | DefaultName: "%[title]",
87 | AvailableOptions: map[string][]string{
88 | "thumbnail": {"yes", "no"},
89 | },
90 | DefaultOptions: map[string]string{
91 | "thumbnail": "no",
92 | },
93 | },
94 | {
95 | Meta: map[string]string{
96 | "title": "tegaki.png",
97 | "imgURL": "https://is2.4chan.org/vip/1549849384199.png",
98 | "id": "1549849384199",
99 | "ext": "png",
100 | "thumbnailURL": "https://i.4cdn.org/vip/1549849384199s.jpg",
101 | },
102 | DefaultName: "%[title]",
103 | AvailableOptions: map[string][]string{
104 | "thumbnail": {"yes", "no"},
105 | },
106 | DefaultOptions: map[string]string{
107 | "thumbnail": "no",
108 | },
109 | },
110 | }
111 |
112 | if diff := pretty.Compare(items, expected); diff != "" {
113 | t.Errorf("%s diff:\n%s", t.Name(), diff)
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/service/fourchan/testdata/TestIteratorNext-resp.golden:
--------------------------------------------------------------------------------
1 | /vip/ - F - Very Important Posts - 4chan
All trademarks and copyrights on this page are owned by their respective parties. Images uploaded are the responsibility of the Poster. Comments are owned by the Poster.
--------------------------------------------------------------------------------
/service/imgur/imgur.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package imgur
18 |
19 | import (
20 | "fmt"
21 | "io"
22 | "net/http"
23 | "strings"
24 |
25 | "github.com/PuerkitoBio/goquery"
26 | "github.com/mlvzk/piko/service"
27 | )
28 |
29 | type Imgur struct{}
30 | type ImgurIterator struct {
31 | url string
32 | page int
33 | end bool
34 | }
35 |
36 | func New() Imgur {
37 | return Imgur{}
38 | }
39 |
40 | type output struct {
41 | io.ReadCloser
42 | length uint64
43 | }
44 |
45 | func (o output) Size() uint64 {
46 | return o.length
47 | }
48 |
49 | func (s Imgur) IsValidTarget(target string) bool {
50 | return strings.Contains(target, "imgur.com/")
51 | }
52 |
53 | func (s Imgur) FetchItems(target string) (service.ServiceIterator, error) {
54 | return &ImgurIterator{
55 | url: target,
56 | page: 1,
57 | end: false,
58 | }, nil
59 | }
60 |
61 | func (s Imgur) Download(meta, options map[string]string) (io.Reader, error) {
62 | resp, err := http.Get(fmt.Sprintf("https://i.imgur.com/%s.%s", meta["id"], meta["ext"]))
63 | if err != nil {
64 | return nil, err
65 | }
66 |
67 | if resp.ContentLength == -1 {
68 | return resp.Body, nil
69 | }
70 |
71 | return output{
72 | ReadCloser: resp.Body,
73 | length: uint64(resp.ContentLength),
74 | }, nil
75 | }
76 |
77 | func (i *ImgurIterator) Next() ([]service.Item, error) {
78 | i.end = true
79 |
80 | resp, err := http.Get(i.url)
81 | if err != nil {
82 | return nil, err
83 | }
84 | defer resp.Body.Close()
85 | if resp.StatusCode != 200 {
86 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", i.url, resp.StatusCode)
87 | }
88 |
89 | doc, err := goquery.NewDocumentFromReader(resp.Body)
90 | if err != nil {
91 | return nil, err
92 | }
93 |
94 | albumTitle := doc.Find("div.post-title-container h1").Text()
95 |
96 | items := []service.Item{}
97 | doc.Find("div.post-images div.post-image-container").Each(func(_ int, sel *goquery.Selection) {
98 | itemType, itExists := sel.Attr("itemtype")
99 | id, _ := sel.Attr("id")
100 |
101 | ext := "png"
102 | if itExists && strings.Contains(itemType, "VideoObject") {
103 | ext = "mp4"
104 | }
105 |
106 | // TODO: take src from meta[@contentURL] if available
107 | items = append(items, service.Item{
108 | Meta: map[string]string{
109 | "id": id,
110 | "ext": ext,
111 | "itemType": itemType,
112 | "albumTitle": albumTitle,
113 | },
114 | DefaultName: "%[id].%[ext]",
115 | })
116 | })
117 |
118 | return items, nil
119 | }
120 |
121 | func (i ImgurIterator) HasEnded() bool {
122 | return i.end
123 | }
124 |
--------------------------------------------------------------------------------
/service/imgur/imgur_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package imgur
18 |
19 | import (
20 | "flag"
21 | "testing"
22 |
23 | "github.com/kylelemons/godebug/pretty"
24 | "github.com/mlvzk/piko/service"
25 | "github.com/mlvzk/piko/service/testutil"
26 | )
27 |
28 | const base = "https://imgur.com"
29 |
30 | var update = flag.Bool("update", false, "update .golden files")
31 |
32 | func TestIsValidTarget(t *testing.T) {
33 | tests := map[string]bool{
34 | "https://imgur.com/gallery/kgIfZrm": true,
35 | "imgur.com/gallery/kgIfZrm": true,
36 | "https://imgur.com/t/article13/y2Vp0nZ": true,
37 | "https://youtube.com/": false,
38 | }
39 |
40 | for target, expected := range tests {
41 | if (Imgur{}).IsValidTarget(target) != expected {
42 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
43 | }
44 | }
45 | }
46 |
47 | func TestIteratorNext(t *testing.T) {
48 | ts := testutil.CacheHttpRequest(t, base, *update)
49 | defer ts.Close()
50 |
51 | iterator := ImgurIterator{
52 | url: ts.URL + "/t/article13/EfY6CxU",
53 | }
54 |
55 | items, err := iterator.Next()
56 | if err != nil {
57 | t.Fatalf("iterator.Next() error: %v", err)
58 | }
59 |
60 | expected := []service.Item{
61 | {
62 | Meta: map[string]string{
63 | "id": "o2nusiZ",
64 | "ext": "png",
65 | "itemType": "http://schema.org/ImageObject",
66 | "albumTitle": "Some advice for those of you in the EU",
67 | },
68 | DefaultName: "%[id].%[ext]",
69 | },
70 | }
71 |
72 | if diff := pretty.Compare(items, expected); diff != "" {
73 | t.Errorf("%s diff:\n%s", t.Name(), diff)
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/service/instagram/instagram.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package instagram
18 |
19 | import (
20 | "errors"
21 | "fmt"
22 | "io"
23 | "net/http"
24 | "strings"
25 |
26 | "github.com/PuerkitoBio/goquery"
27 | "github.com/mlvzk/piko/service"
28 | )
29 |
30 | type schema struct {
31 | Context string `json:"@context"`
32 | Type string `json:"@type"`
33 | Caption string `json:"caption"`
34 | RepresentativeOfPage string `json:"representativeOfPage"`
35 | UploadDate string `json:"uploadDate"`
36 | Author struct {
37 | Type string `json:"@type"`
38 | AlternateName string `json:"alternateName"`
39 | MainEntityofPage struct {
40 | Type string `json:"@type"`
41 | ID string `json:"@id"`
42 | } `json:"mainEntityofPage"`
43 | } `json:"author"`
44 | Comment []struct {
45 | Type string `json:"@type"`
46 | Text string `json:"text"`
47 | Author struct {
48 | Type string `json:"@type"`
49 | AlternateName string `json:"alternateName"`
50 | MainEntityofPage struct {
51 | Type string `json:"@type"`
52 | ID string `json:"@id"`
53 | } `json:"mainEntityofPage"`
54 | } `json:"author"`
55 | } `json:"comment"`
56 | CommentCount string `json:"commentCount"`
57 | InteractionStatistic struct {
58 | Type string `json:"@type"`
59 | InteractionType struct {
60 | Type string `json:"@type"`
61 | } `json:"interactionType"`
62 | UserInteractionCount string `json:"userInteractionCount"`
63 | } `json:"interactionStatistic"`
64 | MainEntityofPage struct {
65 | Type string `json:"@type"`
66 | ID string `json:"@id"`
67 | } `json:"mainEntityofPage"`
68 | Description string `json:"description"`
69 | Name string `json:"name"`
70 | }
71 |
72 | type Instagram struct{}
73 | type InstagramIterator struct {
74 | url string
75 | end bool
76 | }
77 |
78 | func New() Instagram {
79 | return Instagram{}
80 | }
81 |
82 | type output struct {
83 | io.ReadCloser
84 | length uint64
85 | }
86 |
87 | func (o output) Size() uint64 {
88 | return o.length
89 | }
90 |
91 | func (s Instagram) IsValidTarget(target string) bool {
92 | return strings.Contains(target, "instagram.com/")
93 | }
94 |
95 | func (s Instagram) FetchItems(target string) (service.ServiceIterator, error) {
96 | return &InstagramIterator{
97 | url: target,
98 | }, nil
99 | }
100 |
101 | func (s Instagram) Download(meta, options map[string]string) (io.Reader, error) {
102 | resp, err := http.Get(meta["imgURL"])
103 | if err != nil {
104 | return nil, err
105 | }
106 |
107 | if resp.ContentLength == -1 {
108 | return resp.Body, nil
109 | }
110 |
111 | return output{
112 | ReadCloser: resp.Body,
113 | length: uint64(resp.ContentLength),
114 | }, nil
115 | }
116 |
117 | func (i *InstagramIterator) Next() ([]service.Item, error) {
118 | i.end = true
119 |
120 | resp, err := http.Get(i.url)
121 | if err != nil {
122 | return nil, err
123 | }
124 | defer resp.Body.Close()
125 | if resp.StatusCode != 200 {
126 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", i.url, resp.StatusCode)
127 | }
128 |
129 | doc, err := goquery.NewDocumentFromReader(resp.Body)
130 | if err != nil {
131 | return nil, err
132 | }
133 |
134 | if strings.Contains(i.url, "/p/") {
135 | // single image
136 | urlParts := strings.Split(i.url, "/p/")
137 | id := urlParts[1][:len(urlParts[1])-1]
138 |
139 | title := doc.Find(`title`).Text()
140 | imgURL, imgExists := doc.Find(`meta[property="og:image"]`).Attr("content")
141 | if !imgExists {
142 | return nil, errors.New("Couldn't find the image url in meta tags")
143 | }
144 |
145 | var author string
146 | canonicalURL, hasCanonical := doc.Find(`link[rel="canonical"]`).Attr("href")
147 | if hasCanonical {
148 | canonicalParts := strings.Split(canonicalURL, "/")
149 | if len(canonicalParts) > 3 {
150 | author = canonicalParts[3]
151 | }
152 | }
153 |
154 | var caption string
155 | titleQuoteParts := strings.Split(title, "“")
156 | if len(titleQuoteParts) != 0 {
157 | caption = strings.Split(titleQuoteParts[len(titleQuoteParts)-1], "”")[0]
158 | }
159 |
160 | return []service.Item{
161 | {
162 | Meta: map[string]string{
163 | "imgURL": imgURL,
164 | "caption": caption,
165 | "author": author,
166 | "id": id,
167 | "ext": "jpg",
168 | },
169 | DefaultName: "%[author]_%[id].%[ext]",
170 | },
171 | }, nil
172 | }
173 |
174 | return []service.Item{}, nil
175 | }
176 |
177 | func (i InstagramIterator) HasEnded() bool {
178 | return i.end
179 | }
180 |
--------------------------------------------------------------------------------
/service/instagram/instagram_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package instagram
18 |
19 | import (
20 | "flag"
21 | "strings"
22 | "testing"
23 |
24 | "github.com/kylelemons/godebug/pretty"
25 | "github.com/mlvzk/piko/service"
26 | "github.com/mlvzk/piko/service/testutil"
27 | )
28 |
29 | const base = "https://www.instagram.com"
30 |
31 | var update = flag.Bool("update", false, "update .golden files")
32 |
33 | func TestIsValidTarget(t *testing.T) {
34 | tests := map[string]bool{
35 | "https://www.instagram.com/explore/tags/cat/": true,
36 | "instagram.com/explore/tags/cat/": true,
37 | "https://www.instagram.com/p/Bv3X1rVBWm5/": true,
38 | "https://www.instagram.com/newding2/?hl=en": true,
39 | "https://youtube.com/": false,
40 | }
41 |
42 | for target, expected := range tests {
43 | if (Instagram{}).IsValidTarget(target) != expected {
44 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
45 | }
46 | }
47 | }
48 |
49 | func TestIteratorNext(t *testing.T) {
50 | ts := testutil.CacheHttpRequest(t, base, *update)
51 | defer ts.Close()
52 |
53 | iterator := InstagramIterator{
54 | url: ts.URL + "/p/BsOGulcndj-/",
55 | }
56 |
57 | items, err := iterator.Next()
58 | if err != nil {
59 | t.Fatalf("iterator.Next() error: %v", err)
60 | }
61 |
62 | if len(items) < 1 {
63 | t.Fatalf("Items array is empty")
64 | }
65 |
66 | if correctURL := strings.Contains(items[0].Meta["imgURL"], "cdninstagram.com"); !correctURL {
67 | t.Fatalf("Incorrect imgURL")
68 | }
69 | items[0].Meta["imgURL"] = "ignore"
70 |
71 | expected := []service.Item{
72 | {
73 | Meta: map[string]string{
74 | "imgURL": "ignore",
75 | "caption": "Let’s set a world record together and get the most liked post on Instagram. Beating the current world record held by Kylie Jenner (18…",
76 | "author": "world_record_egg",
77 | "id": "BsOGulcndj-",
78 | "ext": "jpg",
79 | },
80 | DefaultName: "%[author]_%[id].%[ext]",
81 | },
82 | }
83 |
84 | if diff := pretty.Compare(items, expected); diff != "" {
85 | t.Errorf("%s diff:\n%s", t.Name(), diff)
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/service/instagram/testdata/TestIteratorNext-resp.golden:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | EGG GANG 🌍 on Instagram: “Let’s set a world record together and get the most liked post on Instagram. Beating the current world record held by Kylie Jenner (18…”
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
36 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
327 |
328 |
329 |
--------------------------------------------------------------------------------
/service/soundcloud/soundcloud.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package soundcloud
18 |
19 | import (
20 | "encoding/json"
21 | "errors"
22 | "fmt"
23 | "io"
24 | "io/ioutil"
25 | "net/http"
26 | "net/url"
27 | "strconv"
28 | "strings"
29 |
30 | "github.com/mlvzk/piko/service"
31 | )
32 |
33 | type trackData struct {
34 | CommentCount int `json:"comment_count"`
35 | Downloadable bool `json:"downloadable"`
36 | CreatedAt string `json:"created_at"`
37 | Description string `json:"description"`
38 | OriginalContentSize int `json:"original_content_size"`
39 | Title string `json:"title"`
40 | Duration int `json:"duration"`
41 | OriginalFormat string `json:"original_format"`
42 | ArtworkURL string `json:"artwork_url"`
43 | Streamable bool `json:"streamable"`
44 | TagList string `json:"tag_list"`
45 | Genre string `json:"genre"`
46 | DownloadURL string `json:"download_url"`
47 | ID int `json:"id"`
48 | State string `json:"state"`
49 | RepostsCount int `json:"reposts_count"`
50 | LastModified string `json:"last_modified"`
51 | Commentable bool `json:"commentable"`
52 | Policy string `json:"policy"`
53 | FavoritingsCount int `json:"favoritings_count"`
54 | Kind string `json:"kind"`
55 | Sharing string `json:"sharing"`
56 | URI string `json:"uri"`
57 | AttachmentsURI string `json:"attachments_uri"`
58 | DownloadCount int `json:"download_count"`
59 | License string `json:"license"`
60 | UserID int `json:"user_id"`
61 | EmbeddableBy string `json:"embeddable_by"`
62 | MonetizationModel string `json:"monetization_model"`
63 | WaveformURL string `json:"waveform_url"`
64 | Permalink string `json:"permalink"`
65 | PermalinkURL string `json:"permalink_url"`
66 | User struct {
67 | ID int `json:"id"`
68 | Kind string `json:"kind"`
69 | Permalink string `json:"permalink"`
70 | Username string `json:"username"`
71 | LastModified string `json:"last_modified"`
72 | URI string `json:"uri"`
73 | PermalinkURL string `json:"permalink_url"`
74 | AvatarURL string `json:"avatar_url"`
75 | } `json:"user"`
76 | StreamURL string `json:"stream_url"`
77 | PlaybackCount int `json:"playback_count"`
78 | }
79 |
80 | type Soundcloud struct {
81 | clientID string
82 | }
83 | type SoundcloudIterator struct {
84 | clientID string
85 | baseApiURL string
86 | url string
87 | end bool
88 | }
89 |
90 | func New(clientID string) Soundcloud {
91 | return Soundcloud{
92 | clientID: clientID,
93 | }
94 | }
95 |
96 | type output struct {
97 | io.ReadCloser
98 | length uint64
99 | }
100 |
101 | func (o output) Size() uint64 {
102 | return o.length
103 | }
104 |
105 | func (s Soundcloud) IsValidTarget(target string) bool {
106 | return strings.Contains(target, "soundcloud.com/")
107 | }
108 |
109 | func (s Soundcloud) FetchItems(target string) (service.ServiceIterator, error) {
110 | return &SoundcloudIterator{
111 | url: target,
112 | baseApiURL: "https://api.soundcloud.com",
113 | clientID: s.clientID,
114 | }, nil
115 | }
116 |
117 | func (s Soundcloud) Download(meta, options map[string]string) (io.Reader, error) {
118 | downloadURL, downloadExists := meta["_downloadURL"]
119 | if !downloadExists {
120 | return nil, errors.New("Download URL doesn't exist")
121 | }
122 |
123 | resp, err := http.Get(downloadURL + "?client_id=" + s.clientID)
124 | if err != nil {
125 | return nil, err
126 | }
127 | if resp.StatusCode != 200 {
128 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", downloadURL, resp.StatusCode)
129 | }
130 |
131 | contentDisp := resp.Header.Get("Content-Disposition")
132 | if strings.Contains(contentDisp, "filename=") {
133 | dotParts := strings.Split(contentDisp, ".")
134 | lastPart := dotParts[len(dotParts)-1]
135 | meta["ext"] = lastPart[:len(lastPart)-1]
136 | } else {
137 | dotParts := strings.Split(resp.Request.URL.Path, ".")
138 | meta["ext"] = dotParts[len(dotParts)-1]
139 | }
140 |
141 | if resp.ContentLength == -1 {
142 | return resp.Body, nil
143 | }
144 |
145 | return output{
146 | ReadCloser: resp.Body,
147 | length: uint64(resp.ContentLength),
148 | }, nil
149 | }
150 |
151 | func (i *SoundcloudIterator) Next() ([]service.Item, error) {
152 | i.end = true
153 |
154 | u, err := makeResolveUrl(i.baseApiURL, i.url, i.clientID)
155 | if err != nil {
156 | return nil, err
157 | }
158 |
159 | resp, err := http.Get(u)
160 | if err != nil {
161 | return nil, err
162 | }
163 | defer resp.Body.Close()
164 | if resp.StatusCode != 200 {
165 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", i.url, resp.StatusCode)
166 | }
167 |
168 | respData, err := ioutil.ReadAll(resp.Body)
169 | if err != nil {
170 | return nil, err
171 | }
172 |
173 | trackResp := trackData{}
174 | json.Unmarshal(respData, &trackResp)
175 |
176 | downloadURL := ""
177 | if trackResp.Downloadable {
178 | downloadURL = trackResp.DownloadURL
179 | } else if trackResp.Streamable {
180 | downloadURL = trackResp.StreamURL
181 | } else {
182 | return nil, errors.New("Track is neither downloadable or streamable")
183 | }
184 |
185 | return []service.Item{
186 | {
187 | Meta: map[string]string{
188 | "id": strconv.Itoa(trackResp.ID),
189 | "title": trackResp.Title,
190 | "username": trackResp.User.Username,
191 | "playCount": strconv.Itoa(trackResp.PlaybackCount),
192 | "duration": strconv.Itoa(trackResp.Duration),
193 | "createdAt": trackResp.CreatedAt,
194 | "ext": "mp3",
195 | "_downloadURL": downloadURL,
196 | },
197 | DefaultName: "%[title].%[ext]",
198 | },
199 | }, nil
200 | }
201 |
202 | func (i SoundcloudIterator) HasEnded() bool {
203 | return i.end
204 | }
205 |
206 | func makeResolveUrl(baseApiURL, targetURL, client_id string) (string, error) {
207 | u, err := url.Parse(baseApiURL)
208 | if err != nil {
209 | return "", err
210 | }
211 |
212 | u.Path = "resolve"
213 |
214 | v := url.Values{}
215 | v.Set("client_id", client_id)
216 | v.Set("url", targetURL)
217 | u.RawQuery = v.Encode()
218 |
219 | return u.String(), nil
220 | }
221 |
--------------------------------------------------------------------------------
/service/soundcloud/soundcloud_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package soundcloud
18 |
19 | import (
20 | "flag"
21 | "testing"
22 |
23 | "github.com/kylelemons/godebug/pretty"
24 | "github.com/mlvzk/piko/service"
25 | "github.com/mlvzk/piko/service/testutil"
26 | )
27 |
28 | const baseApiURL = "https://api.soundcloud.com"
29 |
30 | var update = flag.Bool("update", false, "update .golden files")
31 |
32 | func TestIsValidTarget(t *testing.T) {
33 | tests := map[string]bool{
34 | "https://soundcloud.com/musicpromouser/mac-miller-ok-ft-tyler-the-creator": true,
35 | "https://soundcloud.com/": true,
36 | "https://soundcloud.com/search?q=test": true,
37 | "https://soundcloud.com/fadermedia": true,
38 | "https://instagram.com/": false,
39 | }
40 |
41 | for target, expected := range tests {
42 | if (Soundcloud{}).IsValidTarget(target) != expected {
43 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
44 | }
45 | }
46 | }
47 |
48 | func TestIteratorNext(t *testing.T) {
49 | ts := testutil.CacheHttpRequest(t, baseApiURL, *update)
50 | defer ts.Close()
51 |
52 | iterator := SoundcloudIterator{
53 | url: "https://soundcloud.com/ishaan-bhagwakar/oldie-ofwgkta",
54 | baseApiURL: ts.URL,
55 | clientID: "a3e059563d7fd3372b49b37f00a00bcf",
56 | }
57 |
58 | items, err := iterator.Next()
59 | if err != nil {
60 | t.Fatalf("iterator.Next() error: %v", err)
61 | }
62 |
63 | for _, item := range items {
64 | item.Meta["playCount"] = "ignore"
65 | item.Meta["_downloadURL"] = "ignore"
66 | }
67 |
68 | expected := []service.Item{
69 | {
70 | Meta: map[string]string{
71 | "id": "224754696",
72 | "title": "Oldie - OFWGKTA",
73 | "username": "Ishaan Bhagwakar",
74 | "createdAt": "2015/09/20 17:32:13 +0000",
75 | "duration": "636453",
76 | "playCount": "ignore",
77 | "ext": "mp3",
78 | "_downloadURL": "ignore",
79 | },
80 | DefaultName: "%[title].%[ext]",
81 | },
82 | }
83 |
84 | if diff := pretty.Compare(items, expected); diff != "" {
85 | t.Errorf("%s diff:\n%s", t.Name(), diff)
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/service/soundcloud/testdata/TestIteratorNext-resp.golden:
--------------------------------------------------------------------------------
1 | {"comment_count":26,"downloadable":false,"release":null,"created_at":"2015/09/20 17:32:13 +0000","description":"classic track from 2012, hopefully they get back together\n\nFOLLOW ME FOR GOOD MUSIC ","original_content_size":10182431,"title":"Oldie - OFWGKTA","track_type":null,"duration":636453,"video_url":null,"original_format":"mp3","artwork_url":"https://i1.sndcdn.com/artworks-000130152461-7bwtm3-large.jpg","streamable":true,"tag_list":"","release_month":null,"genre":"tbt","release_day":null,"download_url":"https://api.soundcloud.com/tracks/224754696/download","id":224754696,"state":"finished","reposts_count":642,"last_modified":"2019/01/27 08:18:06 +0000","label_name":null,"commentable":true,"bpm":null,"policy":"ALLOW","favoritings_count":8740,"kind":"track","purchase_url":null,"release_year":null,"key_signature":null,"isrc":null,"sharing":"public","uri":"https://api.soundcloud.com/tracks/224754696","attachments_uri":"https://api.soundcloud.com/tracks/224754696/attachments","download_count":0,"license":"all-rights-reserved","purchase_title":null,"user_id":51011571,"embeddable_by":"all","monetization_model":"NOT_APPLICABLE","waveform_url":"https://wave.sndcdn.com/sAefRYOniH6L_m.png","permalink":"oldie-ofwgkta","permalink_url":"https://soundcloud.com/ishaan-bhagwakar/oldie-ofwgkta","user":{"id":51011571,"kind":"user","permalink":"ishaan-bhagwakar","username":"Ishaan Bhagwakar","last_modified":"2017/06/01 01:56:00 +0000","uri":"https://api.soundcloud.com/users/51011571","permalink_url":"http://soundcloud.com/ishaan-bhagwakar","avatar_url":"https://i1.sndcdn.com/avatars-000067469521-lcmw8k-large.jpg"},"label_id":null,"stream_url":"https://api.soundcloud.com/tracks/224754696/stream","playback_count":530868}
--------------------------------------------------------------------------------
/service/testutil/testutil.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package testutil
18 |
19 | import (
20 | "io"
21 | "net/http"
22 | "net/http/httptest"
23 | "os"
24 | "path/filepath"
25 | "strings"
26 | "testing"
27 | )
28 |
29 | func CacheHttpRequest(t *testing.T, base string, update bool) *httptest.Server {
30 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
31 | golden := filepath.Join("testdata", strings.Replace(t.Name(), "/", "_", -1)+"-resp.golden")
32 |
33 | if update {
34 | req, err := http.NewRequest("GET", base+r.URL.Path+"?"+r.URL.RawQuery, nil)
35 | if err != nil {
36 | t.Fatalf("Error creating new request: %v", err)
37 | }
38 | req.Header = r.Header
39 | req.Header.Del("accept-encoding")
40 |
41 | resp, err := http.DefaultClient.Do(req)
42 | if err != nil {
43 | t.Fatalf("Error sending http request: %v", err)
44 | }
45 | defer resp.Body.Close()
46 |
47 | file, err := os.Create(golden)
48 | if err != nil {
49 | t.Fatalf("Error creating golden file: %v", err)
50 | }
51 |
52 | io.Copy(file, resp.Body)
53 | file.Close()
54 | }
55 |
56 | goldenFile, err := os.Open(golden)
57 | if err != nil {
58 | t.Fatalf("Couldn't open the golden file: %v", err)
59 | }
60 | io.Copy(w, goldenFile)
61 | }))
62 |
63 | return ts
64 | }
65 |
--------------------------------------------------------------------------------
/service/twitter/twitter.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package twitter
18 |
19 | import (
20 | "encoding/json"
21 | "errors"
22 | "fmt"
23 | "io"
24 | "io/ioutil"
25 | "net/http"
26 | "net/url"
27 | "strconv"
28 | "strings"
29 | "time"
30 |
31 | "github.com/PuerkitoBio/goquery"
32 | "github.com/mlvzk/piko/service"
33 | )
34 |
35 | type videoTweet struct {
36 | Track struct {
37 | ContentType string `json:"contentType"`
38 | PublisherID string `json:"publisherId"`
39 | ContentID string `json:"contentId"`
40 | DurationMs int `json:"durationMs"`
41 | PlaybackURL string `json:"playbackUrl"`
42 | PlaybackType string `json:"playbackType"`
43 | ExpandedURL string `json:"expandedUrl"`
44 | ShouldLoop bool `json:"shouldLoop"`
45 | ViewCount string `json:"viewCount"`
46 | IsEventGeoblocked bool `json:"isEventGeoblocked"`
47 | Is360 bool `json:"is360"`
48 | } `json:"track"`
49 | }
50 |
51 | type output struct {
52 | io.ReadCloser
53 | length uint64
54 | }
55 |
56 | func (o output) Size() uint64 {
57 | return o.length
58 | }
59 |
60 | type Twitter struct {
61 | key string
62 | }
63 | type TwitterIterator struct {
64 | url string
65 | end bool
66 | }
67 |
68 | func New(apiKey string) Twitter {
69 | return Twitter{
70 | key: apiKey,
71 | }
72 | }
73 |
74 | func (s Twitter) IsValidTarget(target string) bool {
75 | return strings.Contains(target, "twitter.com/")
76 | }
77 |
78 | func (s Twitter) FetchItems(target string) (service.ServiceIterator, error) {
79 | return &TwitterIterator{
80 | url: target,
81 | }, nil
82 | }
83 |
84 | func (s Twitter) Download(meta, options map[string]string) (io.Reader, error) {
85 | downloadURL, hasDownloadURL := meta["downloadURL"]
86 |
87 | if !hasDownloadURL {
88 | return nil, errors.New("Missing downloadURL")
89 | }
90 |
91 | if meta["type"] == "image" {
92 | resp, err := http.Get(meta["downloadURL"])
93 | if err != nil {
94 | return nil, err
95 | }
96 | if resp.StatusCode != 200 {
97 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", downloadURL, resp.StatusCode)
98 | }
99 |
100 | return resp.Body, nil
101 | } else if meta["type"] == "video" {
102 | configURL := fmt.Sprintf("https://api.twitter.com/1.1/videos/tweet/config/%s.json", meta["id"])
103 | configReq, err := http.NewRequest("GET", configURL, nil)
104 | if err != nil {
105 | return nil, err
106 | }
107 | configReq.Header.Add("Authorization", "Bearer "+s.key)
108 |
109 | playbackURLStr := ""
110 | // retry 4 times, api calls sometimes fail
111 | for i := 0; i < 4; i++ {
112 | configRes, err := http.DefaultClient.Do(configReq)
113 | if err != nil {
114 | return nil, err
115 | }
116 | defer configRes.Body.Close()
117 |
118 | configBytes, err := ioutil.ReadAll(configRes.Body)
119 | if err != nil {
120 | return nil, err
121 | }
122 |
123 | config := videoTweet{}
124 | json.Unmarshal(configBytes, &config)
125 |
126 | if config.Track.PlaybackURL != "" {
127 | playbackURLStr = config.Track.PlaybackURL
128 | break
129 | }
130 |
131 | time.Sleep(time.Millisecond * 500)
132 | }
133 |
134 | if playbackURLStr == "" {
135 | return nil, errors.New("Couldn't get playbackURL")
136 | }
137 |
138 | playbackRes, err := http.Get(playbackURLStr)
139 | if err != nil {
140 | return nil, err
141 | }
142 |
143 | if strings.Contains(playbackURLStr, ".m3u8") {
144 | defer playbackRes.Body.Close()
145 | contentBytes, err := ioutil.ReadAll(playbackRes.Body)
146 | if err != nil {
147 | return nil, err
148 | }
149 | playbackURL, err := url.Parse(playbackURLStr)
150 | if err != nil {
151 | return nil, err
152 | }
153 | playbackBase := playbackURL.Scheme + "://" + playbackURL.Host
154 | bestContent, err := getBestM3u8(playbackBase, string(contentBytes))
155 | if err != nil {
156 | return nil, err
157 | }
158 |
159 | pipeReader, pipeWriter := io.Pipe()
160 | go m3u8ToMpeg(bestContent, pipeWriter)
161 |
162 | meta["ext"] = "mp4"
163 | return pipeReader, nil
164 | }
165 |
166 | if playbackRes.ContentLength == -1 {
167 | return playbackRes.Body, nil
168 | }
169 |
170 | return output{
171 | ReadCloser: playbackRes.Body,
172 | length: uint64(playbackRes.ContentLength),
173 | }, nil
174 | }
175 |
176 | return nil, errors.New("Unsupported type")
177 | }
178 |
179 | func (i *TwitterIterator) Next() ([]service.Item, error) {
180 | i.end = true
181 |
182 | resp, err := http.Get(i.url)
183 | if err != nil {
184 | return nil, err
185 | }
186 | defer resp.Body.Close()
187 | if resp.StatusCode != 200 {
188 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", i.url, resp.StatusCode)
189 | }
190 |
191 | urlParsed, err := url.Parse(i.url)
192 | if err != nil {
193 | return nil, err
194 | }
195 | author := ""
196 | pathParts := strings.Split(urlParsed.Path, "/")
197 | if len(pathParts) >= 1 {
198 | author = pathParts[1]
199 | }
200 | id := pathParts[len(pathParts)-1]
201 |
202 | doc, err := goquery.NewDocumentFromReader(resp.Body)
203 | if err != nil {
204 | return nil, err
205 | }
206 |
207 | description, hasDescription := doc.Find(`meta[property="og:description"]`).Attr("content")
208 | if hasDescription {
209 | runes := []rune(description)
210 | description = string(runes[1 : len(runes)-1])
211 | }
212 |
213 | items := []service.Item{}
214 |
215 | doc.Find(`meta[property="og:video:url"]`).Each(func(index int, videoSel *goquery.Selection) {
216 | videoURL, hasVideo := videoSel.Attr("content")
217 | if !hasVideo {
218 | return
219 | }
220 |
221 | items = append(items, service.Item{
222 | Meta: map[string]string{
223 | "index": strconv.Itoa(index),
224 | "author": author,
225 | "id": id,
226 | "description": description,
227 | "ext": "mp4",
228 | "type": "video",
229 | "downloadURL": videoURL,
230 | },
231 | DefaultName: "%[author]-%[id]-%[index].%[ext]",
232 | })
233 | })
234 |
235 | doc.Find(`meta[property="og:image"]`).Each(func(index int, imageSel *goquery.Selection) {
236 | imageURL, hasImage := imageSel.Attr("content")
237 | if !hasImage || len(imageURL) == 0 {
238 | return
239 | }
240 |
241 | items = append(items, service.Item{
242 | Meta: map[string]string{
243 | "index": strconv.Itoa(index),
244 | "author": author,
245 | "id": id,
246 | "description": description,
247 | "ext": "jpg",
248 | "type": "image",
249 | "downloadURL": imageURL,
250 | },
251 | DefaultName: "%[author]-%[id]-%[index].%[ext]",
252 | })
253 | })
254 |
255 | return items, nil
256 | }
257 |
258 | func (i TwitterIterator) HasEnded() bool {
259 | return i.end
260 | }
261 |
--------------------------------------------------------------------------------
/service/twitter/twitter_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package twitter
18 |
19 | import (
20 | "flag"
21 | "strings"
22 | "testing"
23 |
24 | "github.com/kylelemons/godebug/pretty"
25 | "github.com/mlvzk/piko/service"
26 | "github.com/mlvzk/piko/service/testutil"
27 | )
28 |
29 | const base = "https://twitter.com"
30 |
31 | var update = flag.Bool("update", false, "update .golden files")
32 |
33 | func TestIsValidTarget(t *testing.T) {
34 | tests := map[string]bool{
35 | "https://twitter.com/golang": true,
36 | "https://twitter.com/golang/status/1116531752602951681": true,
37 | "twitter.com/golang/status/1116531752602951681": true,
38 | "https://soundcloud.com/": false,
39 | }
40 |
41 | for target, expected := range tests {
42 | if (Twitter{}).IsValidTarget(target) != expected {
43 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
44 | }
45 | }
46 | }
47 |
48 | func TestIteratorNext(t *testing.T) {
49 | ts := testutil.CacheHttpRequest(t, base, *update)
50 | defer ts.Close()
51 |
52 | iterator := TwitterIterator{
53 | url: ts.URL + "/golang/status/1106303553474301955",
54 | }
55 |
56 | items, err := iterator.Next()
57 | if err != nil {
58 | t.Fatalf("iterator.Next() error: %v", err)
59 | }
60 |
61 | if len(items) < 1 {
62 | t.Fatalf("Items array is empty")
63 | }
64 |
65 | if !strings.Contains(items[0].Meta["downloadURL"], "pbs.twimg.com") {
66 | t.Fatalf("Incorrect downloadURL")
67 | }
68 | items[0].Meta["downloadURL"] = "ignore"
69 |
70 | expected := []service.Item{
71 | {
72 | Meta: map[string]string{
73 | "downloadURL": "ignore",
74 | "index": "0",
75 | "id": "1106303553474301955",
76 | "author": "golang",
77 | "description": "🎉 Go 1.12.1 and 1.11.6 are released!\n\n🗣 Announcement: https://t.co/PAttJybffj\n\nHappy Pi day! 🥧\n\n#golang",
78 | "type": "image",
79 | "ext": "jpg",
80 | },
81 | DefaultName: "%[author]-%[id]-%[index].%[ext]",
82 | },
83 | }
84 |
85 | if diff := pretty.Compare(items, expected); diff != "" {
86 | t.Errorf("%s diff:\n%s", t.Name(), diff)
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/service/twitter/utils.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package twitter
18 |
19 | import (
20 | "errors"
21 | "io"
22 | "io/ioutil"
23 | "net/http"
24 | "strings"
25 | )
26 |
27 | func getBestM3u8(baseURL, content string) (string, error) {
28 | lines := strings.Split(content, "\n")
29 | best, found := "", false
30 |
31 | for i := len(lines) - 1; i >= 0; i-- {
32 | if strings.Contains(lines[i], ".m3u8") {
33 | best, found = lines[i], true
34 | break
35 | }
36 | }
37 |
38 | if !found {
39 | return "", errors.New("Couldn't find the best m3u8")
40 | }
41 |
42 | res, err := http.Get(baseURL + best)
43 | if err != nil {
44 | return "", err
45 | }
46 | defer res.Body.Close()
47 |
48 | bytes, err := ioutil.ReadAll(res.Body)
49 | if err != nil {
50 | return "", err
51 | }
52 |
53 | contentLines := strings.Split(string(bytes), "\n")
54 | for i := range contentLines {
55 | if len(contentLines[i]) > 0 && contentLines[i][0] == '/' {
56 | contentLines[i] = baseURL + contentLines[i]
57 | }
58 | }
59 |
60 | return strings.Join(contentLines, "\n"), nil
61 | }
62 |
63 | func m3u8ToMpeg(content string, writer io.WriteCloser) error {
64 | defer writer.Close()
65 | lines := strings.Split(content, "\n")
66 |
67 | for _, line := range lines {
68 | if len(line) < 4 || line[0:4] != "http" {
69 | continue
70 | }
71 |
72 | res, err := http.Get(line)
73 | if err != nil {
74 | return err
75 | }
76 | io.Copy(writer, res.Body)
77 | res.Body.Close()
78 | }
79 |
80 | return nil
81 | }
82 |
--------------------------------------------------------------------------------
/service/utils.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package service
18 |
19 | import (
20 | "fmt"
21 | "io"
22 | "net/http"
23 | )
24 |
25 | func FetchContentLength(url string) (int64, error) {
26 | req, err := http.NewRequest("GET", url, nil)
27 | if err != nil {
28 | return -1, err
29 | }
30 |
31 | res, err := http.DefaultClient.Do(req)
32 | if err != nil {
33 | return -1, err
34 | }
35 | res.Body.Close()
36 |
37 | return res.ContentLength, nil
38 | }
39 |
40 | func DownloadByChunks(url string, chunkSize uint64, writer io.WriteCloser) error {
41 | var pos uint64
42 |
43 | for {
44 | start := pos
45 | end := pos + chunkSize - 1
46 | pos += chunkSize
47 |
48 | req, err := http.NewRequest("GET", url, nil)
49 | if err != nil {
50 | return err
51 | }
52 | req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, end))
53 |
54 | res, err := http.DefaultClient.Do(req)
55 | if err != nil {
56 | return err
57 | }
58 |
59 | n, err := io.Copy(writer, res.Body)
60 | if err != nil {
61 | return err
62 | }
63 | res.Body.Close()
64 |
65 | if uint64(n) < chunkSize {
66 | break
67 | }
68 | }
69 |
70 | writer.Close()
71 |
72 | return nil
73 | }
74 |
--------------------------------------------------------------------------------
/service/youtube/youtube.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package youtube
18 |
19 | import (
20 | "encoding/json"
21 | "errors"
22 | "fmt"
23 | "io"
24 | "io/ioutil"
25 | "net/http"
26 | "net/url"
27 | "os/exec"
28 | "regexp"
29 | "strconv"
30 | "strings"
31 |
32 | "github.com/PuerkitoBio/goquery"
33 | "github.com/mlvzk/piko/service"
34 | "github.com/mlvzk/piko/service/youtube/ytdl"
35 | )
36 |
37 | type Youtube struct{}
38 | type YoutubeIterator struct {
39 | urls []string
40 | }
41 |
42 | func New() Youtube {
43 | return Youtube{}
44 | }
45 |
46 | // youtubeConfig is a partial structure for deserializing youtube's json config
47 | type youtubeConfig struct {
48 | Args struct {
49 | PlayerResponseStr string `json:"player_response"`
50 | URLEncodedFmtStreamMap string `json:"url_encoded_fmt_stream_map"`
51 | AdaptiveFmts string `json:"adaptive_fmts"`
52 | } `json:"args"`
53 | Assets struct {
54 | JS string `json:"js"`
55 | } `json:"assets"`
56 | }
57 |
58 | type playerResponse struct {
59 | VideoDetails struct {
60 | Title string `json:"title"`
61 | Author string `json:"author"`
62 | } `json:"videoDetails"`
63 | }
64 |
65 | type output struct {
66 | io.ReadCloser
67 | length uint64
68 | }
69 |
70 | func (o output) Size() uint64 {
71 | return o.length
72 | }
73 |
74 | func (s Youtube) IsValidTarget(target string) bool {
75 | return strings.Contains(target, "youtube.com/") || strings.Contains(target, "youtu.be/")
76 | }
77 |
78 | func (s Youtube) FetchItems(target string) (service.ServiceIterator, error) {
79 | if strings.Contains(target, "/playlist") {
80 | urls, err := s.extractURLs(target)
81 | if err != nil {
82 | return nil, err
83 | }
84 |
85 | return &YoutubeIterator{
86 | urls: urls,
87 | }, nil
88 | }
89 |
90 | return &YoutubeIterator{
91 | urls: []string{target},
92 | }, nil
93 | }
94 |
95 | func (s Youtube) Download(meta, options map[string]string) (io.Reader, error) {
96 | quality := options["quality"]
97 | useFfmpeg := options["useFfmpeg"] == "yes"
98 | onlyAudio := options["onlyAudio"] == "yes"
99 |
100 | ytConfig := youtubeConfig{}
101 | json.Unmarshal([]byte(meta["_ytConfig"]), &ytConfig)
102 |
103 | formats := getFormats(ytConfig.Args.AdaptiveFmts, ytConfig.Args.URLEncodedFmtStreamMap)
104 |
105 | if _, err := exec.LookPath("ffmpeg"); (err != nil || !useFfmpeg) && !onlyAudio {
106 | // no ffmpeg, fallbacking to format with both audio and video
107 | video := findBestVideoAudio(formats)
108 | videoURL, err := ytdl.GetDownloadURL(video.Meta, ytConfig.Assets.JS)
109 | if err != nil {
110 | return nil, err
111 | }
112 |
113 | videoStream, videoStreamWriter := io.Pipe()
114 | go service.DownloadByChunks(videoURL.String(), 0xFFFFF, videoStreamWriter)
115 |
116 | // this is bad, in order for this to work file name needs to be formatted after Download is called
117 | meta["ext"] = video.Extension
118 |
119 | var videoLength int64
120 | videoLengthMeta, hasLength := video.Meta["clen"]
121 | if hasLength {
122 | videoLength, err = strconv.ParseInt(videoLengthMeta.(string), 10, 64)
123 | if err != nil {
124 | hasLength = false
125 | }
126 | }
127 | if !hasLength {
128 | videoLength, err = service.FetchContentLength(videoURL.String())
129 | if err != nil {
130 | hasLength = false
131 | } else {
132 | hasLength = true
133 | }
134 | }
135 |
136 | if hasLength && videoLength != -1 {
137 | return &output{
138 | ReadCloser: videoStream,
139 | length: uint64(videoLength),
140 | }, nil
141 | }
142 |
143 | return videoStream, nil
144 | }
145 |
146 | audioFormat, audioFormatFound := findBestAudio(formats)
147 | var (
148 | videoFormat ytdl.Format
149 | videoFormatFound bool
150 | )
151 | switch quality {
152 | case "worst", "low":
153 | videoFormat, videoFormatFound = findWorstVideo(formats)
154 | case "best", "high":
155 | videoFormat, videoFormatFound = findBestVideo(formats)
156 | default:
157 | videoFormat, videoFormatFound = findMediumVideo(formats)
158 | if !videoFormatFound {
159 | // fallback to best if medium not found
160 | videoFormat, videoFormatFound = findBestVideo(formats)
161 | }
162 | }
163 |
164 | if !audioFormatFound || !videoFormatFound {
165 | return nil, errors.New("Couldn't find either audio or video format")
166 | }
167 |
168 | // not actual length, but should be close
169 | audioLengthMeta, hasAudioLength := audioFormat.Meta["clen"]
170 | audioLength, _ := strconv.ParseInt(audioLengthMeta.(string), 10, 64)
171 |
172 | audioURL, err := ytdl.GetDownloadURL(audioFormat.Meta, ytConfig.Assets.JS)
173 | if err != nil {
174 | return nil, err
175 | }
176 |
177 | if onlyAudio {
178 | meta["ext"] = audioFormat.Extension
179 | audioStream, audioStreamWriter := io.Pipe()
180 | go service.DownloadByChunks(audioURL.String(), 0xFFFFF, audioStreamWriter)
181 |
182 | return output{
183 | ReadCloser: audioStream,
184 | length: uint64(audioLength),
185 | }, nil
186 | }
187 |
188 | videoLengthMeta, hasVideoLength := videoFormat.Meta["clen"]
189 | videoLength, _ := strconv.ParseInt(videoLengthMeta.(string), 10, 64)
190 |
191 | tmpAudioFile, err := ioutil.TempFile("", "audio*."+audioFormat.Extension)
192 | if err != nil {
193 | return nil, err
194 | }
195 |
196 | audioStream, audioStreamWriter := io.Pipe()
197 | go io.Copy(tmpAudioFile, audioStream)
198 | err = service.DownloadByChunks(audioURL.String(), 0xFFFFF, audioStreamWriter)
199 | if err != nil {
200 | return nil, err
201 | }
202 | io.Copy(tmpAudioFile, audioStream)
203 | tmpAudioFile.Close()
204 | audioStream.Close()
205 |
206 | videoURL, err := ytdl.GetDownloadURL(videoFormat.Meta, ytConfig.Assets.JS)
207 | if err != nil {
208 | return nil, err
209 | }
210 | videoStream, videoStreamWriter := io.Pipe()
211 | // download by 10MB chunks to avoid throttling
212 | go service.DownloadByChunks(videoURL.String(), 0xFFFFF, videoStreamWriter)
213 |
214 | cmd := exec.Command("ffmpeg", "-i", tmpAudioFile.Name(), "-i", "-", "-c", "copy", "-f", "matroska", "-")
215 | cmd.Stdin = videoStream
216 | stdout, err := cmd.StdoutPipe()
217 | if err != nil {
218 | return nil, err
219 | }
220 |
221 | go cmd.Run()
222 |
223 | if (hasAudioLength && hasVideoLength) || hasVideoLength {
224 | return output{
225 | ReadCloser: stdout,
226 | // length is not exact, but close enough
227 | length: uint64(audioLength + videoLength),
228 | }, nil
229 | }
230 |
231 | return stdout, nil
232 | }
233 |
234 | var extractURLsRegex = regexp.MustCompile(`"url":"/watch\?v=([A-Za-z0-9_\-]{11})`)
235 |
236 | func (s Youtube) extractURLs(target string) ([]string, error) {
237 | req, err := http.NewRequest("GET", target, nil)
238 | if err != nil {
239 | return nil, err
240 | }
241 | req.Header.Set("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36")
242 |
243 | resp, err := http.DefaultClient.Do(req)
244 | if err != nil {
245 | return nil, err
246 | }
247 | body, err := ioutil.ReadAll(resp.Body)
248 | if err != nil {
249 | return nil, err
250 | }
251 |
252 | urls := map[string]struct{}{}
253 | matches := extractURLsRegex.FindAllStringSubmatch(string(body), -1)
254 | for _, m := range matches {
255 | if len(m) < 2 {
256 | continue
257 | }
258 |
259 | urls["https://www.youtube.com/watch?v="+m[1]] = struct{}{}
260 | }
261 |
262 | links := make([]string, 0, len(urls))
263 | for u := range urls {
264 | links = append(links, u)
265 | }
266 |
267 | return links, nil
268 | }
269 |
270 | var ytConfigRegexp = regexp.MustCompile(`ytplayer\.config = (.*?);ytplayer\.load = function()`)
271 |
272 | func (i *YoutubeIterator) Next() ([]service.Item, error) {
273 | if len(i.urls) == 0 {
274 | return nil, nil
275 | }
276 |
277 | u := i.urls[0]
278 | i.urls = i.urls[1:]
279 |
280 | resp, err := http.Get(u)
281 | if err != nil {
282 | return nil, err
283 | }
284 | defer resp.Body.Close()
285 | if resp.StatusCode != 200 {
286 | return nil, fmt.Errorf("GET %v returned a wrong status code - %v", u, resp.StatusCode)
287 | }
288 |
289 | doc, err := goquery.NewDocumentFromReader(resp.Body)
290 | if err != nil {
291 | return nil, err
292 | }
293 |
294 | // TODO: download all videos from playlists/channels
295 | // this only downloads the main video from the url
296 | ytMatches := ytConfigRegexp.FindStringSubmatch(doc.Find("script").Text())
297 | if len(ytMatches) < 2 {
298 | return nil, errors.New("Could not match youtube's json config for url: " + u + " ; The video is probably not available")
299 | }
300 | ytConfigStr := ytMatches[1]
301 | ytConfig := youtubeConfig{}
302 | json.Unmarshal([]byte(ytConfigStr), &ytConfig)
303 |
304 | ytPlayer := playerResponse{}
305 | json.Unmarshal([]byte(ytConfig.Args.PlayerResponseStr), &ytPlayer)
306 |
307 | item := service.Item{
308 | Meta: map[string]string{
309 | "title": ytPlayer.VideoDetails.Title,
310 | "author": ytPlayer.VideoDetails.Author,
311 | "ext": "mkv",
312 | "_ytConfig": ytMatches[1],
313 | },
314 | DefaultName: "%[title].%[ext]",
315 | AvailableOptions: map[string]([]string){
316 | "quality": []string{"best", "medium", "worst"},
317 | "useFfmpeg": []string{"yes", "no"},
318 | "onlyAudio": []string{"yes", "no"},
319 | },
320 | DefaultOptions: map[string]string{
321 | "quality": "medium",
322 | "useFfmpeg": "yes",
323 | "onlyAudio": "no",
324 | },
325 | }
326 |
327 | return []service.Item{item}, nil
328 | }
329 |
330 | func (i YoutubeIterator) HasEnded() bool {
331 | return len(i.urls) == 0
332 | }
333 |
334 | func findBestVideoAudio(formats []ytdl.Format) ytdl.Format {
335 | var best ytdl.Format
336 |
337 | for _, f := range formats {
338 | if f.AudioEncoding == "" || f.VideoEncoding == "" {
339 | continue
340 | }
341 |
342 | if f.AudioBitrate > best.AudioBitrate {
343 | best = f
344 | }
345 | }
346 |
347 | return best
348 | }
349 |
350 | func findBestAudio(formats []ytdl.Format) (audio ytdl.Format, found bool) {
351 | for _, f := range formats {
352 | if f.AudioEncoding == "" || f.VideoEncoding != "" {
353 | continue
354 | }
355 |
356 | if f.AudioBitrate > audio.AudioBitrate {
357 | audio = f
358 | found = true
359 | }
360 | }
361 |
362 | return
363 | }
364 |
365 | func _findVideo(formats []ytdl.Format, best bool) (video ytdl.Format, found bool) {
366 | var currBitrate int
367 |
368 | for _, f := range formats {
369 | if f.VideoEncoding == "" || f.AudioEncoding != "" {
370 | continue
371 | }
372 |
373 | bitrateI, exists := f.Meta["bitrate"]
374 | if !exists {
375 | continue
376 | }
377 | bitrate, err := strconv.Atoi(bitrateI.(string))
378 | if err != nil {
379 | continue
380 | }
381 |
382 | if (best && bitrate > currBitrate) ||
383 | (!best && (bitrate < currBitrate || !found)) {
384 | video = f
385 | currBitrate = bitrate
386 | found = true
387 | }
388 | }
389 |
390 | return
391 | }
392 |
393 | func findBestVideo(formats []ytdl.Format) (video ytdl.Format, found bool) {
394 | return _findVideo(formats, true)
395 | }
396 |
397 | func findWorstVideo(formats []ytdl.Format) (video ytdl.Format, found bool) {
398 | return _findVideo(formats, false)
399 | }
400 |
401 | // medium is 1080p, 720p or 480p
402 | // prefers 1080p > 720p > 480p
403 | func findMediumVideo(formats []ytdl.Format) (video ytdl.Format, found bool) {
404 | var currBitrate int
405 | allowedRes := map[string]bool{"1080p": true, "720p": true, "480p": true}
406 |
407 | for _, f := range formats {
408 | if f.VideoEncoding == "" || f.AudioEncoding != "" {
409 | continue
410 | }
411 |
412 | bitrateI, exists := f.Meta["bitrate"]
413 | if !exists {
414 | continue
415 | }
416 | bitrate, err := strconv.Atoi(bitrateI.(string))
417 | if err != nil {
418 | continue
419 | }
420 |
421 | allowed, inRange := allowedRes[f.Resolution]
422 | if inRange && allowed && bitrate > currBitrate {
423 | video = f
424 | currBitrate = bitrate
425 | found = true
426 | }
427 | }
428 |
429 | return
430 | }
431 |
432 | func getFormats(strs ...string) []ytdl.Format {
433 | var formats []ytdl.Format
434 |
435 | for _, str := range strs {
436 | formatStrs := strings.Split(str, ",")
437 |
438 | for _, formatStr := range formatStrs {
439 | query, err := url.ParseQuery(formatStr)
440 | if err != nil {
441 | continue
442 | }
443 |
444 | itag, err := strconv.Atoi(query.Get("itag"))
445 | if err != nil || itag <= 0 {
446 | continue
447 | }
448 |
449 | format, _ := ytdl.NewFormat(itag)
450 | format.Meta = make(map[string]interface{})
451 | if strings.HasPrefix(query.Get("conn"), "rtmp") {
452 | format.Meta["rtmp"] = true
453 | }
454 | for k, v := range query {
455 | if len(v) == 1 {
456 | format.Meta[k] = v[0]
457 | } else {
458 | format.Meta[k] = v
459 | }
460 | }
461 | formats = append(formats, format)
462 | }
463 | }
464 |
465 | return formats
466 | }
467 |
--------------------------------------------------------------------------------
/service/youtube/youtube_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package youtube
18 |
19 | import (
20 | "flag"
21 | "sort"
22 | "testing"
23 |
24 | "github.com/kylelemons/godebug/pretty"
25 | "github.com/mlvzk/piko/service"
26 | "github.com/mlvzk/piko/service/testutil"
27 | )
28 |
29 | const base = "https://www.youtube.com"
30 |
31 | var update = flag.Bool("update", false, "update .golden files")
32 |
33 | func TestIsValidTarget(t *testing.T) {
34 | tests := map[string]bool{
35 | "https://www.youtube.com/watch?v=HOK0uF-Z0xM": true,
36 | "https://youtube.com/watch?v=HOK0uF-Z0xM": true,
37 | "youtube.com/watch?v=HOK0uF-Z0xM": true,
38 | "https://youtu.be/HOK0uF-Z0xM": true,
39 | "https://imgur.com/": false,
40 | }
41 |
42 | for target, expected := range tests {
43 | if (Youtube{}).IsValidTarget(target) != expected {
44 | t.Errorf("Invalid result, target: %v, expected: %v", target, expected)
45 | }
46 | }
47 | }
48 |
49 | func TestIteratorNext(t *testing.T) {
50 | ts := testutil.CacheHttpRequest(t, base, *update)
51 | defer ts.Close()
52 |
53 | iterator := YoutubeIterator{
54 | urls: []string{ts.URL + "/watch?v=Q8Tiz6INF7I"},
55 | }
56 |
57 | items, err := iterator.Next()
58 | if err != nil {
59 | t.Fatalf("iterator.Next() error: %v", err)
60 | }
61 |
62 | if len(items) < 1 {
63 | t.Fatalf("Items array is empty")
64 | }
65 |
66 | for k := range items[0].Meta {
67 | if k[0] == '_' {
68 | items[0].Meta[k] = "ignore"
69 | }
70 | }
71 |
72 | expected := []service.Item{
73 | {
74 | Meta: map[string]string{
75 | "_ytConfig": "ignore",
76 | "author": "Andres Trevino",
77 | "title": "Hit the road Jack!",
78 | "ext": "mkv",
79 | },
80 | DefaultName: "%[title].%[ext]",
81 | AvailableOptions: map[string]([]string){
82 | "quality": []string{"best", "medium", "worst"},
83 | "useFfmpeg": []string{"yes", "no"},
84 | "onlyAudio": []string{"yes", "no"},
85 | },
86 | DefaultOptions: map[string]string{
87 | "quality": "medium",
88 | "useFfmpeg": "yes",
89 | "onlyAudio": "no",
90 | },
91 | },
92 | }
93 |
94 | if diff := pretty.Compare(items, expected); diff != "" {
95 | t.Errorf("%s diff:\n%s", t.Name(), diff)
96 | }
97 | }
98 |
99 | func TestExtractURLs(t *testing.T) {
100 | tests := map[string]struct {
101 | target string
102 | wantLinks []string
103 | wantErr bool
104 | }{
105 | "short playlist": {
106 | target: "/playlist?list=PLE2y3n8EQ6Vwnua4Dhfm6lI9zoy2IzDOD",
107 | wantLinks: []string{
108 | "https://www.youtube.com/watch?v=ycxZPpmaxxs",
109 | "https://www.youtube.com/watch?v=bALlZMHqy3g",
110 | "https://www.youtube.com/watch?v=UsSf9zFfgsU",
111 | "https://www.youtube.com/watch?v=SLAmSo0Gd4U",
112 | "https://www.youtube.com/watch?v=6VhhqeF3V_4",
113 | "https://www.youtube.com/watch?v=DIy1YnhvRLY",
114 | "https://www.youtube.com/watch?v=2nHoZZ-P8bI",
115 | "https://www.youtube.com/watch?v=Gibx29aqtcM",
116 | "https://www.youtube.com/watch?v=7AD9r0uUASE",
117 | "https://www.youtube.com/watch?v=ICqjNK1bDqw",
118 | "https://www.youtube.com/watch?v=wUq416NjXbk",
119 | "https://www.youtube.com/watch?v=Usfizu445gE",
120 | "https://www.youtube.com/watch?v=NnGrlVDO8yU",
121 | "https://www.youtube.com/watch?v=0b0Xj-eVneY",
122 | "https://www.youtube.com/watch?v=jx9DLjqngVs",
123 | "https://www.youtube.com/watch?v=lS4hEl_7BbI",
124 | },
125 | },
126 | "long playlist over 100 videos": {
127 | target: "/playlist?list=PLvn31dJXvzXpsspKfYLuhqeyRbMtMu0wL",
128 | wantLinks: []string{
129 | "https://www.youtube.com/watch?v=05qcA4KPI0k",
130 | "https://www.youtube.com/watch?v=18uuczwHp78",
131 | "https://www.youtube.com/watch?v=1oZPtbXBn-4",
132 | "https://www.youtube.com/watch?v=1t-gK-9EIq4",
133 | "https://www.youtube.com/watch?v=2e6OoeY1X8c",
134 | "https://www.youtube.com/watch?v=3yg0-e5ZFqY",
135 | "https://www.youtube.com/watch?v=4dBtfeoXM8I",
136 | "https://www.youtube.com/watch?v=6GfkQOhXg_M",
137 | "https://www.youtube.com/watch?v=75nKyI2FFFI",
138 | "https://www.youtube.com/watch?v=7dgrMSTalZ0",
139 | "https://www.youtube.com/watch?v=8Bv802FwvCY",
140 | "https://www.youtube.com/watch?v=8yn3ViE6mhY",
141 | "https://www.youtube.com/watch?v=9Y5eqpVQ1p8",
142 | "https://www.youtube.com/watch?v=9pt7EWFF_T8",
143 | "https://www.youtube.com/watch?v=AZRGPg5laDU",
144 | "https://www.youtube.com/watch?v=A_p-myZaodg",
145 | "https://www.youtube.com/watch?v=BOrnC3LQeLs",
146 | "https://www.youtube.com/watch?v=B_geuq76Cig",
147 | "https://www.youtube.com/watch?v=BaBR--4bw08",
148 | "https://www.youtube.com/watch?v=C4kVQnZhHmg",
149 | "https://www.youtube.com/watch?v=CnA0ft6DMpM",
150 | "https://www.youtube.com/watch?v=DRPi0XXmc-I",
151 | "https://www.youtube.com/watch?v=FTdcEoBLEuk",
152 | "https://www.youtube.com/watch?v=FWRfpC8s6XU",
153 | "https://www.youtube.com/watch?v=Fy7FzXLin7o",
154 | "https://www.youtube.com/watch?v=GrC_yuzO-Ss",
155 | "https://www.youtube.com/watch?v=HBBFufxHj3M",
156 | "https://www.youtube.com/watch?v=IUWYPbe96jE",
157 | "https://www.youtube.com/watch?v=I_O37cE1j64",
158 | "https://www.youtube.com/watch?v=IsvfofcIE1Q",
159 | "https://www.youtube.com/watch?v=JIrm0dHbCDU",
160 | "https://www.youtube.com/watch?v=JPb-59BPHAk",
161 | "https://www.youtube.com/watch?v=KANBeat7FFA",
162 | "https://www.youtube.com/watch?v=KCRDQ2qwnds",
163 | "https://www.youtube.com/watch?v=KEoU0pgnFNc",
164 | "https://www.youtube.com/watch?v=M2VBmHOYpV8",
165 | "https://www.youtube.com/watch?v=MzGnX-MbYE4",
166 | "https://www.youtube.com/watch?v=NRJh_r1LiqY",
167 | "https://www.youtube.com/watch?v=NihMVuspKQw",
168 | "https://www.youtube.com/watch?v=OL8Wqe-QWM8",
169 | "https://www.youtube.com/watch?v=OSjxK1SrCWk",
170 | "https://www.youtube.com/watch?v=SsKyxkfj8ak",
171 | "https://www.youtube.com/watch?v=TPqLJJfrVVY",
172 | "https://www.youtube.com/watch?v=U9vfK_bl4o8",
173 | "https://www.youtube.com/watch?v=UgTl4wLlMGI",
174 | "https://www.youtube.com/watch?v=V7GCrTFCXYo",
175 | "https://www.youtube.com/watch?v=VEAuMiKqP-4",
176 | "https://www.youtube.com/watch?v=VkqXIpl7a2w",
177 | "https://www.youtube.com/watch?v=WAXfhWUFIGA",
178 | "https://www.youtube.com/watch?v=WWJem7RuBpc",
179 | "https://www.youtube.com/watch?v=XWK7QLvuI-I",
180 | "https://www.youtube.com/watch?v=XkB4COqwcW4",
181 | "https://www.youtube.com/watch?v=Z3U8I0Bktb4",
182 | "https://www.youtube.com/watch?v=Z62fegq1gkk",
183 | "https://www.youtube.com/watch?v=ZUWvXERYJfk",
184 | "https://www.youtube.com/watch?v=_-QPvffO1gs",
185 | "https://www.youtube.com/watch?v=_1JAwLrQy9k",
186 | "https://www.youtube.com/watch?v=_6FBfAQ-NDE",
187 | "https://www.youtube.com/watch?v=a46z0mS3NPM",
188 | "https://www.youtube.com/watch?v=a8gYRf3aeQc",
189 | "https://www.youtube.com/watch?v=aDgHXiWgKlE",
190 | "https://www.youtube.com/watch?v=aGSKrC7dGcY",
191 | "https://www.youtube.com/watch?v=b1Wvvk4YtmE",
192 | "https://www.youtube.com/watch?v=bt-28iNQnwY",
193 | "https://www.youtube.com/watch?v=cGvZyrhObrg",
194 | "https://www.youtube.com/watch?v=cfzAGk8SlfE",
195 | "https://www.youtube.com/watch?v=dKnjm5SJ5jc",
196 | "https://www.youtube.com/watch?v=du8JSARa1H8",
197 | "https://www.youtube.com/watch?v=ejQ7KxUeItY",
198 | "https://www.youtube.com/watch?v=euBr4iyY_x8",
199 | "https://www.youtube.com/watch?v=f95pB9spuFk",
200 | "https://www.youtube.com/watch?v=fphsbLtrDe8",
201 | "https://www.youtube.com/watch?v=h1mD-_DKHc0",
202 | "https://www.youtube.com/watch?v=hUun8wjHx5Y",
203 | "https://www.youtube.com/watch?v=iDoSbyGBmy4",
204 | "https://www.youtube.com/watch?v=iEH4eqtK8SU",
205 | "https://www.youtube.com/watch?v=iTKJ_itifQg",
206 | "https://www.youtube.com/watch?v=j7EsBK4Mr80",
207 | "https://www.youtube.com/watch?v=jsCR05oKROA",
208 | "https://www.youtube.com/watch?v=kqRGZtGNPW4",
209 | "https://www.youtube.com/watch?v=l35XzUD8GGU",
210 | "https://www.youtube.com/watch?v=lD87Hbm9mrI",
211 | "https://www.youtube.com/watch?v=mU3tlDMI8xw",
212 | "https://www.youtube.com/watch?v=nhZdL4JlnxI",
213 | "https://www.youtube.com/watch?v=oeBTsGkngj8",
214 | "https://www.youtube.com/watch?v=pO0A998XZ5k",
215 | "https://www.youtube.com/watch?v=qU8UfYdKHvs",
216 | "https://www.youtube.com/watch?v=r_0sL_SQYvw",
217 | "https://www.youtube.com/watch?v=rxv9TTmk18o",
218 | "https://www.youtube.com/watch?v=snILjFUkk_A",
219 | "https://www.youtube.com/watch?v=u1xrNaTO1bI",
220 | "https://www.youtube.com/watch?v=up3r4qRWxWE",
221 | "https://www.youtube.com/watch?v=urbmwI8APdo",
222 | "https://www.youtube.com/watch?v=vOtRZOlE0WM",
223 | "https://www.youtube.com/watch?v=vXfbnS_BybQ",
224 | "https://www.youtube.com/watch?v=vyrpRzdvp5U",
225 | "https://www.youtube.com/watch?v=wkKueyJaA0A",
226 | "https://www.youtube.com/watch?v=yaGKZsgA_u0",
227 | "https://www.youtube.com/watch?v=zZeRwuN68VQ",
228 | "https://www.youtube.com/watch?v=zzbzHtdCzlI",
229 | },
230 | },
231 | }
232 | for ttName, tt := range tests {
233 | t.Run(ttName, func(t *testing.T) {
234 | ts := testutil.CacheHttpRequest(t, base, *update)
235 | defer ts.Close()
236 |
237 | gotLinks, err := Youtube{}.extractURLs(ts.URL + tt.target)
238 | if (err != nil) != tt.wantErr {
239 | t.Errorf("Youtube.ExtractURLs() error = %v, wantErr %v", err, tt.wantErr)
240 | return
241 | }
242 | sort.Strings(gotLinks)
243 | sort.Strings(tt.wantLinks)
244 | if diff := pretty.Compare(gotLinks, tt.wantLinks); diff != "" {
245 | t.Errorf("%s diff:\n%s", t.Name(), diff)
246 | }
247 | })
248 | }
249 | }
250 |
--------------------------------------------------------------------------------
/service/youtube/ytdl/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) <2015>
2 |
3 |
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 |
13 |
14 | The above copyright notice and this permission notice shall be included in
15 | all copies or substantial portions of the Software.
16 |
17 |
18 |
19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 | THE SOFTWARE.
--------------------------------------------------------------------------------
/service/youtube/ytdl/README.md:
--------------------------------------------------------------------------------
1 | # ytdl
2 |
3 | strip down fork of https://github.com/rylio/ytdl
--------------------------------------------------------------------------------
/service/youtube/ytdl/format.go:
--------------------------------------------------------------------------------
1 | package ytdl
2 |
3 | // Format is a youtube is a static youtube video format
4 | type Format struct {
5 | Itag int `json:"itag"`
6 | Extension string `json:"extension"`
7 | Resolution string `json:"resolution"`
8 | VideoEncoding string `json:"videoEncoding"`
9 | AudioEncoding string `json:"audioEncoding"`
10 | AudioBitrate int `json:"audioBitrate"`
11 | Meta map[string]interface{}
12 | }
13 |
14 | func NewFormat(itag int) (Format, bool) {
15 | if f, ok := FORMATS[itag]; ok {
16 | f.Meta = make(map[string]interface{})
17 | return f, true
18 | }
19 | return Format{}, false
20 | }
21 |
22 | // FORMATS is a map of all itags and their formats
23 | var FORMATS = map[int]Format{
24 | 5: {
25 | Extension: "flv",
26 | Resolution: "240p",
27 | VideoEncoding: "Sorenson H.283",
28 | AudioEncoding: "mp3",
29 | Itag: 5,
30 | AudioBitrate: 64,
31 | },
32 | 6: {
33 | Extension: "flv",
34 | Resolution: "270p",
35 | VideoEncoding: "Sorenson H.263",
36 | AudioEncoding: "mp3",
37 | Itag: 6,
38 | AudioBitrate: 64,
39 | },
40 | 13: {
41 | Extension: "3gp",
42 | Resolution: "",
43 | VideoEncoding: "MPEG-4 Visual",
44 | AudioEncoding: "aac",
45 | Itag: 13,
46 | AudioBitrate: 0,
47 | },
48 | 17: {
49 | Extension: "3gp",
50 | Resolution: "144p",
51 | VideoEncoding: "MPEG-4 Visual",
52 | AudioEncoding: "aac",
53 | Itag: 17,
54 | AudioBitrate: 24,
55 | },
56 | 18: {
57 | Extension: "mp4",
58 | Resolution: "360p",
59 | VideoEncoding: "H.264",
60 | AudioEncoding: "aac",
61 | Itag: 18,
62 | AudioBitrate: 96,
63 | },
64 | 22: {
65 | Extension: "mp4",
66 | Resolution: "720p",
67 | VideoEncoding: "H.264",
68 | AudioEncoding: "aac",
69 | Itag: 22,
70 | AudioBitrate: 192,
71 | },
72 | 34: {
73 | Extension: "flv",
74 | Resolution: "480p",
75 | VideoEncoding: "H.264",
76 | AudioEncoding: "aac",
77 | Itag: 34,
78 | AudioBitrate: 128,
79 | },
80 | 35: {
81 | Extension: "flv",
82 | Resolution: "360p",
83 | VideoEncoding: "H.264",
84 | AudioEncoding: "aac",
85 | Itag: 35,
86 | AudioBitrate: 128,
87 | },
88 | 36: {
89 | Extension: "3gp",
90 | Resolution: "240p",
91 | VideoEncoding: "MPEG-4 Visual",
92 | AudioEncoding: "aac",
93 | Itag: 36,
94 | AudioBitrate: 36,
95 | },
96 | 37: {
97 | Extension: "mp4",
98 | Resolution: "1080p",
99 | VideoEncoding: "H.264",
100 | AudioEncoding: "aac",
101 | Itag: 37,
102 | AudioBitrate: 192,
103 | },
104 | 38: {
105 | Extension: "mp4",
106 | Resolution: "3072p",
107 | VideoEncoding: "H.264",
108 | AudioEncoding: "aac",
109 | Itag: 38,
110 | AudioBitrate: 192,
111 | },
112 | 43: {
113 | Extension: "webm",
114 | Resolution: "360p",
115 | VideoEncoding: "VP8",
116 | AudioEncoding: "vorbis",
117 | Itag: 43,
118 | AudioBitrate: 128,
119 | },
120 | 44: {
121 | Extension: "webm",
122 | Resolution: "480p",
123 | VideoEncoding: "VP8",
124 | AudioEncoding: "vorbis",
125 | Itag: 44,
126 | AudioBitrate: 128,
127 | },
128 | 45: {
129 | Extension: "webm",
130 | Resolution: "720p",
131 | VideoEncoding: "VP8",
132 | AudioEncoding: "vorbis",
133 | Itag: 45,
134 | AudioBitrate: 192,
135 | },
136 | 46: {
137 | Extension: "webm",
138 | Resolution: "1080p",
139 | VideoEncoding: "VP8",
140 | AudioEncoding: "vorbis",
141 | Itag: 46,
142 | AudioBitrate: 192,
143 | },
144 | 82: {
145 | Extension: "mp4",
146 | Resolution: "360p",
147 | VideoEncoding: "H.264",
148 | Itag: 82,
149 | AudioBitrate: 96,
150 | },
151 | 83: {
152 | Extension: "mp4",
153 | Resolution: "240p",
154 | VideoEncoding: "H.264",
155 | AudioEncoding: "aac",
156 | Itag: 83,
157 | AudioBitrate: 96,
158 | },
159 | 84: {
160 | Extension: "mp4",
161 | Resolution: "720p",
162 | VideoEncoding: "H.264",
163 | AudioEncoding: "aac",
164 | Itag: 84,
165 | AudioBitrate: 192,
166 | },
167 | 85: {
168 | Extension: "mp4",
169 | Resolution: "1080p",
170 | VideoEncoding: "H.264",
171 | AudioEncoding: "aac",
172 | Itag: 85,
173 | AudioBitrate: 192,
174 | },
175 | 100: {
176 | Extension: "webm",
177 | Resolution: "360p",
178 | VideoEncoding: "VP8",
179 | AudioEncoding: "vorbis",
180 | Itag: 100,
181 | AudioBitrate: 128,
182 | },
183 | 101: {
184 | Extension: "webm",
185 | Resolution: "360p",
186 | VideoEncoding: "VP8",
187 | AudioEncoding: "vorbis",
188 | Itag: 101,
189 | AudioBitrate: 192,
190 | },
191 | 102: {
192 | Extension: "webm",
193 | Resolution: "720p",
194 | VideoEncoding: "VP8",
195 | AudioEncoding: "vorbis",
196 | Itag: 102,
197 | AudioBitrate: 192,
198 | },
199 | // DASH (video only)
200 | 133: {
201 | Extension: "mp4",
202 | Resolution: "240p",
203 | VideoEncoding: "H.264",
204 | AudioEncoding: "",
205 | Itag: 133,
206 | AudioBitrate: 0,
207 | },
208 | 134: {
209 | Extension: "mp4",
210 | Resolution: "360p",
211 | VideoEncoding: "H.264",
212 | AudioEncoding: "",
213 | Itag: 134,
214 | AudioBitrate: 0,
215 | },
216 | 135: {
217 | Extension: "mp4",
218 | Resolution: "480p",
219 | VideoEncoding: "H.264",
220 | AudioEncoding: "",
221 | Itag: 135,
222 | AudioBitrate: 0,
223 | },
224 | 136: {
225 | Extension: "mp4",
226 | Resolution: "720p",
227 | VideoEncoding: "H.264",
228 | AudioEncoding: "",
229 | Itag: 136,
230 | AudioBitrate: 0,
231 | },
232 | 137: {
233 | Extension: "mp4",
234 | Resolution: "1080p",
235 | VideoEncoding: "H.264",
236 | AudioEncoding: "",
237 | Itag: 137,
238 | AudioBitrate: 0,
239 | },
240 | 138: {
241 | Extension: "mp4",
242 | Resolution: "2160p",
243 | VideoEncoding: "H.264",
244 | AudioEncoding: "",
245 | Itag: 138,
246 | AudioBitrate: 0,
247 | },
248 | 160: {
249 | Extension: "mp4",
250 | Resolution: "144p",
251 | VideoEncoding: "H.264",
252 | AudioEncoding: "",
253 | Itag: 160,
254 | AudioBitrate: 0,
255 | },
256 | 242: {
257 | Extension: "webm",
258 | Resolution: "240p",
259 | VideoEncoding: "VP9",
260 | AudioEncoding: "",
261 | Itag: 242,
262 | AudioBitrate: 0,
263 | },
264 | 243: {
265 | Extension: "webm",
266 | Resolution: "360p",
267 | VideoEncoding: "VP9",
268 | AudioEncoding: "",
269 | Itag: 243,
270 | AudioBitrate: 0,
271 | },
272 | 244: {
273 | Extension: "webm",
274 | Resolution: "480p",
275 | VideoEncoding: "VP9",
276 | AudioEncoding: "",
277 | Itag: 244,
278 | AudioBitrate: 0,
279 | },
280 | 247: {
281 | Extension: "webm",
282 | Resolution: "720p",
283 | VideoEncoding: "VP9",
284 | AudioEncoding: "",
285 | Itag: 247,
286 | AudioBitrate: 0,
287 | },
288 | 248: {
289 | Extension: "webm",
290 | Resolution: "1080p",
291 | VideoEncoding: "VP9",
292 | AudioEncoding: "",
293 | Itag: 248,
294 | AudioBitrate: 9,
295 | },
296 | 264: {
297 | Extension: "mp4",
298 | Resolution: "1440p",
299 | VideoEncoding: "H.264",
300 | AudioEncoding: "",
301 | Itag: 264,
302 | AudioBitrate: 0,
303 | },
304 | 266: {
305 | Extension: "mp4",
306 | Resolution: "2160p",
307 | VideoEncoding: "H.264",
308 | AudioEncoding: "",
309 | Itag: 266,
310 | AudioBitrate: 0,
311 | },
312 | 271: {
313 | Extension: "webm",
314 | Resolution: "1440p",
315 | VideoEncoding: "VP9",
316 | AudioEncoding: "",
317 | Itag: 271,
318 | AudioBitrate: 0,
319 | },
320 | 272: {
321 | Extension: "webm",
322 | Resolution: "2160p",
323 | VideoEncoding: "VP9",
324 | AudioEncoding: "",
325 | Itag: 272,
326 | AudioBitrate: 0,
327 | },
328 | 278: {
329 | Extension: "webm",
330 | Resolution: "144p",
331 | VideoEncoding: "VP9",
332 | AudioEncoding: "",
333 | Itag: 278,
334 | AudioBitrate: 0,
335 | },
336 | 298: {
337 | Extension: "mp4",
338 | Resolution: "720p",
339 | VideoEncoding: "H.264",
340 | AudioEncoding: "",
341 | Itag: 298,
342 | AudioBitrate: 0,
343 | },
344 | 299: {
345 | Extension: "mp4",
346 | Resolution: "1080p",
347 | VideoEncoding: "H.264",
348 | AudioEncoding: "",
349 | Itag: 299,
350 | AudioBitrate: 0,
351 | },
352 | 302: {
353 | Extension: "webm",
354 | Resolution: "720p",
355 | VideoEncoding: "VP9",
356 | AudioEncoding: "",
357 | Itag: 302,
358 | AudioBitrate: 0,
359 | },
360 | 303: {
361 | Extension: "webm",
362 | Resolution: "1080p",
363 | VideoEncoding: "VP9",
364 | AudioEncoding: "",
365 | Itag: 303,
366 | AudioBitrate: 0,
367 | },
368 | // DASH (audio only)
369 | 139: {
370 | Extension: "mp4",
371 | Resolution: "",
372 | VideoEncoding: "",
373 | AudioEncoding: "aac",
374 | Itag: 139,
375 | AudioBitrate: 48,
376 | },
377 | 140: {
378 | Extension: "mp4",
379 | Resolution: "",
380 | VideoEncoding: "",
381 | AudioEncoding: "aac",
382 | Itag: 140,
383 | AudioBitrate: 128,
384 | },
385 | 141: {
386 | Extension: "mp4",
387 | Resolution: "",
388 | VideoEncoding: "",
389 | AudioEncoding: "aac",
390 | Itag: 141,
391 | AudioBitrate: 256,
392 | },
393 | 171: {
394 | Extension: "webm",
395 | Resolution: "",
396 | VideoEncoding: "",
397 | AudioEncoding: "vorbis",
398 | Itag: 171,
399 | AudioBitrate: 128,
400 | },
401 | 172: {
402 | Extension: "webm",
403 | Resolution: "",
404 | VideoEncoding: "",
405 | AudioEncoding: "vorbis",
406 | Itag: 172,
407 | AudioBitrate: 192,
408 | },
409 | 249: {
410 | Extension: "webm",
411 | Resolution: "",
412 | VideoEncoding: "",
413 | AudioEncoding: "opus",
414 | Itag: 249,
415 | AudioBitrate: 50,
416 | },
417 | 250: {
418 | Extension: "webm",
419 | Resolution: "",
420 | VideoEncoding: "",
421 | AudioEncoding: "opus",
422 | Itag: 250,
423 | AudioBitrate: 70,
424 | },
425 | 251: {
426 | Extension: "webm",
427 | Resolution: "",
428 | VideoEncoding: "",
429 | AudioEncoding: "opus",
430 | Itag: 251,
431 | AudioBitrate: 160,
432 | },
433 | // Live streaming
434 | 92: {
435 | Extension: "ts",
436 | Resolution: "240p",
437 | VideoEncoding: "H.264",
438 | AudioEncoding: "aac",
439 | Itag: 92,
440 | AudioBitrate: 48,
441 | },
442 | 93: {
443 | Extension: "ts",
444 | Resolution: "480p",
445 | VideoEncoding: "H.264",
446 | AudioEncoding: "aac",
447 | Itag: 93,
448 | AudioBitrate: 128,
449 | },
450 | 94: {
451 | Extension: "ts",
452 | Resolution: "720p",
453 | VideoEncoding: "H.264",
454 | AudioEncoding: "aac",
455 | Itag: 94,
456 | AudioBitrate: 128,
457 | },
458 | 95: {
459 | Extension: "ts",
460 | Resolution: "1080p",
461 | VideoEncoding: "H.264",
462 | AudioEncoding: "aac",
463 | Itag: 95,
464 | AudioBitrate: 256,
465 | },
466 | 96: {
467 | Extension: "ts",
468 | Resolution: "720p",
469 | VideoEncoding: "H.264",
470 | AudioEncoding: "aac",
471 | Itag: 96,
472 | AudioBitrate: 256,
473 | },
474 | 120: {
475 | Extension: "flv",
476 | Resolution: "720p",
477 | VideoEncoding: "H.264",
478 | AudioEncoding: "aac",
479 | Itag: 120,
480 | AudioBitrate: 128,
481 | },
482 | 127: {
483 | Extension: "ts",
484 | Resolution: "",
485 | VideoEncoding: "",
486 | AudioEncoding: "aac",
487 | Itag: 127,
488 | AudioBitrate: 96,
489 | },
490 | 128: {
491 | Extension: "ts",
492 | Resolution: "",
493 | VideoEncoding: "",
494 | AudioEncoding: "aac",
495 | Itag: 128,
496 | AudioBitrate: 96,
497 | },
498 | 132: {
499 | Extension: "ts",
500 | Resolution: "240p",
501 | VideoEncoding: "H.264",
502 | AudioEncoding: "aac",
503 | Itag: 132,
504 | AudioBitrate: 48,
505 | },
506 | 151: {
507 | Extension: "ts",
508 | Resolution: "720p",
509 | VideoEncoding: "H.264",
510 | AudioEncoding: "aac",
511 | Itag: 151,
512 | AudioBitrate: 24,
513 | },
514 | }
515 |
--------------------------------------------------------------------------------
/service/youtube/ytdl/signature.go:
--------------------------------------------------------------------------------
1 | package ytdl
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "net/http"
7 | "net/url"
8 | "regexp"
9 | "strconv"
10 | "strings"
11 | )
12 |
13 | func GetDownloadURL(formatMeta map[string]interface{}, htmlPlayerFile string) (*url.URL, error) {
14 | var sig string
15 | if s, ok := formatMeta["s"]; ok && len(s.(string)) > 0 {
16 | tokens, err := getSigTokens(htmlPlayerFile)
17 | if err != nil {
18 | return nil, err
19 | }
20 | sig = decipherTokens(tokens, s.(string))
21 | } else {
22 | if s, ok := formatMeta["sig"]; ok {
23 | sig = s.(string)
24 | }
25 | }
26 | var urlString string
27 | if s, ok := formatMeta["url"]; ok {
28 | urlString = s.(string)
29 | } else if s, ok := formatMeta["stream"]; ok {
30 | if c, ok := formatMeta["conn"]; ok {
31 | urlString = c.(string)
32 | if urlString[len(urlString)-1] != '/' {
33 | urlString += "/"
34 | }
35 | }
36 | urlString += s.(string)
37 | } else {
38 | return nil, fmt.Errorf("Couldn't extract url from format")
39 | }
40 | urlString, err := url.QueryUnescape(urlString)
41 | if err != nil {
42 | return nil, err
43 | }
44 | u, err := url.Parse(urlString)
45 | if err != nil {
46 | return nil, err
47 | }
48 | query := u.Query()
49 | query.Set("ratebypass", "yes")
50 | if len(sig) > 0 {
51 | param := "signature"
52 | if v, ok := formatMeta["sp"].(string); ok && v != "" {
53 | param = v
54 | }
55 | query.Set(param, sig)
56 | }
57 | u.RawQuery = query.Encode()
58 | return u, nil
59 | }
60 |
61 | func decipherTokens(tokens []string, sig string) string {
62 | var pos int
63 | sigSplit := strings.Split(sig, "")
64 | for i, l := 0, len(tokens); i < l; i++ {
65 | tok := tokens[i]
66 | if len(tok) > 1 {
67 | pos, _ = strconv.Atoi(string(tok[1:]))
68 | pos = ^^pos
69 | }
70 | switch string(tok[0]) {
71 | case "r":
72 | reverseStringSlice(sigSplit)
73 | case "w":
74 | s := sigSplit[0]
75 | sigSplit[0] = sigSplit[pos]
76 | sigSplit[pos] = s
77 | case "s":
78 | sigSplit = sigSplit[pos:]
79 | case "p":
80 | sigSplit = sigSplit[pos:]
81 | }
82 | }
83 | return strings.Join(sigSplit, "")
84 | }
85 |
86 | const (
87 | jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
88 | reverseStr = ":function\\(a\\)\\{" +
89 | "(?:return )?a\\.reverse\\(\\)" +
90 | "\\}"
91 | sliceStr = ":function\\(a,b\\)\\{" +
92 | "return a\\.slice\\(b\\)" +
93 | "\\}"
94 | spliceStr = ":function\\(a,b\\)\\{" +
95 | "a\\.splice\\(0,b\\)" +
96 | "\\}"
97 | swapStr = ":function\\(a,b\\)\\{" +
98 | "var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
99 | "\\}"
100 | )
101 |
102 | var actionsObjRegexp = regexp.MustCompile(fmt.Sprintf(
103 | "var (%s)=\\{((?:(?:%s%s|%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, reverseStr, jsvarStr, sliceStr, jsvarStr, spliceStr, jsvarStr, swapStr))
104 |
105 | var actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
106 | "function(?: %s)?\\(a\\)\\{"+
107 | "a=a\\.split\\(\"\"\\);\\s*"+
108 | "((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
109 | "return a\\.join\\(\"\"\\)"+
110 | "\\}", jsvarStr, jsvarStr, jsvarStr))
111 |
112 | var reverseRegexp = regexp.MustCompile(fmt.Sprintf(
113 | "(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
114 | var sliceRegexp = regexp.MustCompile(fmt.Sprintf(
115 | "(?m)(?:^|,)(%s)%s", jsvarStr, sliceStr))
116 | var spliceRegexp = regexp.MustCompile(fmt.Sprintf(
117 | "(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
118 | var swapRegexp = regexp.MustCompile(fmt.Sprintf(
119 | "(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
120 |
121 | func getSigTokens(htmlPlayerFile string) ([]string, error) {
122 | u, _ := url.Parse("https://www.youtube.com/watch")
123 | p, err := url.Parse(htmlPlayerFile)
124 | if err != nil {
125 | return nil, err
126 | }
127 | resp, err := http.Get(u.ResolveReference(p).String())
128 | if err != nil {
129 | return nil, err
130 | }
131 | defer resp.Body.Close()
132 | if resp.StatusCode != 200 {
133 | return nil, fmt.Errorf("Error fetching signature tokens, status code %d", resp.StatusCode)
134 | }
135 | body, err := ioutil.ReadAll(resp.Body)
136 | bodyString := string(body)
137 | if err != nil {
138 | return nil, err
139 | }
140 |
141 | objResult := actionsObjRegexp.FindStringSubmatch(bodyString)
142 | funcResult := actionsFuncRegexp.FindStringSubmatch(bodyString)
143 |
144 | if len(objResult) < 3 || len(funcResult) < 2 {
145 | return nil, fmt.Errorf("Error parsing signature tokens")
146 | }
147 | obj := strings.Replace(objResult[1], "$", "\\$", -1)
148 | objBody := strings.Replace(objResult[2], "$", "\\$", -1)
149 | funcBody := strings.Replace(funcResult[1], "$", "\\$", -1)
150 |
151 | var reverseKey, sliceKey, spliceKey, swapKey string
152 | var result []string
153 |
154 | if result = reverseRegexp.FindStringSubmatch(objBody); len(result) > 1 {
155 | reverseKey = strings.Replace(result[1], "$", "\\$", -1)
156 | }
157 | if result = sliceRegexp.FindStringSubmatch(objBody); len(result) > 1 {
158 | sliceKey = strings.Replace(result[1], "$", "\\$", -1)
159 | }
160 | if result = spliceRegexp.FindStringSubmatch(objBody); len(result) > 1 {
161 | spliceKey = strings.Replace(result[1], "$", "\\$", -1)
162 | }
163 | if result = swapRegexp.FindStringSubmatch(objBody); len(result) > 1 {
164 | swapKey = strings.Replace(result[1], "$", "\\$", -1)
165 | }
166 |
167 | keys := []string{reverseKey, sliceKey, spliceKey, swapKey}
168 | regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s)\\(a,(\\d+)\\)", obj, strings.Join(keys, "|")))
169 | if err != nil {
170 | return nil, err
171 | }
172 | results := regex.FindAllStringSubmatch(funcBody, -1)
173 | var tokens []string
174 | for _, s := range results {
175 | switch s[1] {
176 | case swapKey:
177 | tokens = append(tokens, "w"+s[2])
178 | case reverseKey:
179 | tokens = append(tokens, "r")
180 | case sliceKey:
181 | tokens = append(tokens, "s"+s[2])
182 | case spliceKey:
183 | tokens = append(tokens, "p"+s[2])
184 | }
185 | }
186 | return tokens, nil
187 | }
188 |
--------------------------------------------------------------------------------
/service/youtube/ytdl/utils.go:
--------------------------------------------------------------------------------
1 | package ytdl
2 |
3 | func reverseStringSlice(s []string) {
4 | for i, j := 0, len(s)-1; i < len(s)/2; i, j = i+1, j-1 {
5 | s[i], s[j] = s[j], s[i]
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/utils.go:
--------------------------------------------------------------------------------
1 | // Copyright 2019 mlvzk
2 | // This file is part of the piko library.
3 | //
4 | // The piko library is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU Lesser General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // The piko library is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU Lesser General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU Lesser General Public License
15 | // along with the piko library. If not, see .
16 |
17 | package piko
18 |
19 | import (
20 | "github.com/mlvzk/piko/service"
21 | "github.com/mlvzk/piko/service/facebook"
22 | "github.com/mlvzk/piko/service/fourchan"
23 | "github.com/mlvzk/piko/service/imgur"
24 | "github.com/mlvzk/piko/service/instagram"
25 | "github.com/mlvzk/piko/service/soundcloud"
26 | "github.com/mlvzk/piko/service/twitter"
27 | "github.com/mlvzk/piko/service/youtube"
28 | )
29 |
30 | func GetAllServices() []service.Service {
31 | return []service.Service{
32 | youtube.New(),
33 | imgur.New(),
34 | instagram.New(),
35 | fourchan.New(),
36 | soundcloud.New("a3e059563d7fd3372b49b37f00a00bcf"),
37 | twitter.New("AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE"),
38 | facebook.New(),
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
