├── .gitignore
├── .travis.yml
├── LICENSE
├── common.go
├── extract.go
├── go.mod
├── go.sum
├── handler.go
├── main.go
├── main_test.go
├── models.go
├── readme.md
├── regex.go
├── tistory.go
└── vars.go
/.gitignore:
--------------------------------------------------------------------------------
1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
2 | *.o
3 | *.a
4 | *.so
5 |
6 | # Folders
7 | _obj
8 | _test
9 |
10 | # Architecture specific extensions/prefixes
11 | *.[568vq]
12 | [568vq].out
13 |
14 | *.cgo1.go
15 | *.cgo2.c
16 | _cgo_defun.c
17 | _cgo_gotypes.go
18 | _cgo_export.*
19 |
20 | _testmain.go
21 |
22 | *.exe
23 | *.test
24 | *.prof
25 |
26 | # Output of the go coverage tool, specifically when used with LiteIDE
27 | *.out
28 |
29 | config.ini
30 | crossbuild.bat
31 |
32 | .idea/
33 |
34 | vendor
35 | credentials.json
36 | .DS_Store
37 |
38 | # Exclude subdirectories (things like database, saved images during testing)
39 | /**
40 | # Exclude build command shortcut(s)
41 | *.cmd
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: go
2 | go:
3 | - "1.15"
4 | install:
5 | - env GO111MODULE=on go mod download
6 | - go get github.com/mitchellh/gox
7 | script:
8 | - env GO111MODULE=on gox -output="bin/discord-image-downloader-go-{{.OS}}-{{.Arch}}" -osarch="windows/amd64 linux/amd64 linux/arm"
9 | deploy:
10 | provider: releases
11 | api_key:
12 | secure: BZnS+tH7kCy2Jpzl5ctIIlQqurIlLvJVw5Pn2tkjEnQlnEQZUW7dEI/eejz1Tw1NNjuxdfJRzgkTNI7sRM6VjtuoTzq772TuxncTv4Oy/t/S3zaBthMMl/8Ug+VW2Py/A6LO8YDVpEaIIenyUTsmPG99HJd7jjW+qCh3npfjvWvf4Qe3FL0cNOH6l9ptA77Fcu1j7yXOUqCpxVtLL8sRe9qTsciUGT8eSAaHLBKbUxcw4TMn4P7G1/6b4tfkm1aQZX5cJROccdYt6FY5z1hNUUUN3e1bn1Hj/iY2zUZm1F4LHC5AcfZFUu/fAGhoLbB1B/hEZ4zrE7RLKaCR1vWCbr54/Xn+uWdDWPL6dbJ3EzCIuDtGDoHAJBfRos9HsCdfizB5A/yg0OS3RscK7Umh+gEznJyQCvAIbndAaryxKeOxVPd6tIdoW0fi0yaWmq9UHKVUt7lLd8rxlGWhGh6/GPLCME7+EAR84RyvP7e6VUHxUSnB3bly3oeJZd/QZpHmrIPqE6K9iobDJyERjNd+Bvuay8bw65NcqR9ZtAYlK7tUYxyEhw23fyybrwTB7ONUu/DDUSLQrtP7YvhtGnxMS+cU4O5GZrkuIP7urUS5KAnYbzb5I3F1Bk8PDLNFEdsGUOKq9U5ywRypmVfSxldGQHvEl4x1CO3E3zJEhkDOIaI=
13 | file:
14 | - "bin/discord-image-downloader-go-windows-amd64.exe"
15 | - "bin/discord-image-downloader-go-linux-amd64"
16 | - "bin/discord-image-downloader-go-linux-arm"
17 | skip_cleanup: true
18 | on:
19 | repo: Seklfreak/discord-image-downloader-go
20 | tags: true
21 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2016 Sebastian Winkler
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/common.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "path"
6 | "strings"
7 |
8 | "github.com/bwmarrin/discordgo"
9 | )
10 |
11 | // isDiscordEmoji matches https://cdn.discordapp.com/emojis/503141595860959243.gif, and similar URLs/filenames
12 | func isDiscordEmoji(link string) bool {
13 | // always match discord emoji URLs, eg https://cdn.discordapp.com/emojis/340989430460317707.png
14 | if strings.HasPrefix(link, discordEmojiBaseUrl) {
15 | return true
16 | }
17 |
18 | return false
19 | }
20 |
21 | // deduplicateDownloadItems removes duplicates from a slice of *DownloadItem s identified by the Link
22 | func deduplicateDownloadItems(DownloadItems []*DownloadItem) []*DownloadItem {
23 | var result []*DownloadItem
24 | seen := map[string]bool{}
25 |
26 | for _, item := range DownloadItems {
27 | if seen[item.Link] {
28 | continue
29 | }
30 |
31 | seen[item.Link] = true
32 | result = append(result, item)
33 | }
34 |
35 | return result
36 | }
37 |
38 | func updateDiscordStatus() {
39 | if StatusEnabled {
40 | dg.UpdateStatusComplex(discordgo.UpdateStatusData{
41 | Game: &discordgo.Game{
42 | Name: fmt.Sprintf("%d %s", countDownloadedImages(), StatusSuffix),
43 | Type: StatusLabel,
44 | },
45 | Status: StatusType,
46 | })
47 | } else if StatusType != "online" {
48 | dg.UpdateStatusComplex(discordgo.UpdateStatusData{
49 | Status: StatusType,
50 | })
51 | }
52 | }
53 |
54 | func Pagify(text string, delimiter string) []string {
55 | result := make([]string, 0)
56 | textParts := strings.Split(text, delimiter)
57 | currentOutputPart := ""
58 | for _, textPart := range textParts {
59 | if len(currentOutputPart)+len(textPart)+len(delimiter) <= 1992 {
60 | if len(currentOutputPart) > 0 || len(result) > 0 {
61 | currentOutputPart += delimiter + textPart
62 | } else {
63 | currentOutputPart += textPart
64 | }
65 | } else {
66 | result = append(result, currentOutputPart)
67 | currentOutputPart = ""
68 | if len(textPart) <= 1992 {
69 | currentOutputPart = textPart
70 | }
71 | }
72 | }
73 | if currentOutputPart != "" {
74 | result = append(result, currentOutputPart)
75 | }
76 | return result
77 | }
78 |
79 | func filepathExtension(filepath string) string {
80 | if strings.Contains(filepath, "?") {
81 | filepath = strings.Split(filepath, "?")[0]
82 | }
83 | filepath = path.Ext(filepath)
84 | return filepath
85 | }
86 |
--------------------------------------------------------------------------------
/extract.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net/url"
6 | "time"
7 |
8 | "github.com/bwmarrin/discordgo"
9 | "mvdan.cc/xurls/v2"
10 | )
11 |
12 | func getDownloadLinks(inputURL string, channelID string, interactive bool) (map[string]string, bool) {
13 | if RegexpUrlTwitter.MatchString(inputURL) {
14 | links, err := getTwitterUrls(inputURL)
15 | if err != nil {
16 | fmt.Println("twitter URL failed,", inputURL, ",", err)
17 | } else if len(links) > 0 {
18 | return skipDuplicateLinks(links, channelID, interactive), true
19 | }
20 | }
21 | if RegexpUrlTwitterParams.MatchString(inputURL) {
22 | links, err := getTwitterParamsUrls(inputURL)
23 | if err != nil {
24 | fmt.Println("twitter params URL failed,", inputURL, ",", err)
25 | } else if len(links) > 0 {
26 | return skipDuplicateLinks(links, channelID, interactive), true
27 | }
28 | }
29 | if RegexpUrlTwitterStatus.MatchString(inputURL) {
30 | links, err := getTwitterStatusUrls(inputURL, channelID)
31 | if err != nil {
32 | fmt.Println("twitter status URL failed,", inputURL, ",", err)
33 | } else if len(links) > 0 {
34 | return skipDuplicateLinks(links, channelID, interactive), true
35 | }
36 | }
37 | if RegexpUrlTistory.MatchString(inputURL) {
38 | links, err := getTistoryUrls(inputURL)
39 | if err != nil {
40 | fmt.Println("tistory URL failed,", inputURL, ",", err)
41 | } else if len(links) > 0 {
42 | return skipDuplicateLinks(links, channelID, interactive), true
43 | }
44 | }
45 | if RegexpUrlTistoryLegacy.MatchString(inputURL) {
46 | links, err := getLegacyTistoryUrls(inputURL)
47 | if err != nil {
48 | fmt.Println("legacy tistory URL failed,", inputURL, ",", err)
49 | } else if len(links) > 0 {
50 | return skipDuplicateLinks(links, channelID, interactive), true
51 | }
52 | }
53 | if RegexpUrlGfycat.MatchString(inputURL) {
54 | links, err := getGfycatUrls(inputURL)
55 | if err != nil {
56 | fmt.Println("gfycat URL failed,", inputURL, ",", err)
57 | } else if len(links) > 0 {
58 | return skipDuplicateLinks(links, channelID, interactive), true
59 | }
60 | }
61 | if RegexpUrlInstagram.MatchString(inputURL) {
62 | links, err := getInstagramUrls(inputURL)
63 | if err != nil {
64 | fmt.Println("instagram URL failed,", inputURL, ",", err)
65 | } else if len(links) > 0 {
66 | return skipDuplicateLinks(links, channelID, interactive), true
67 | }
68 | }
69 | if RegexpUrlImgurSingle.MatchString(inputURL) {
70 | links, err := getImgurSingleUrls(inputURL)
71 | if err != nil {
72 | fmt.Println("imgur single URL failed, ", inputURL, ",", err)
73 | } else if len(links) > 0 {
74 | return skipDuplicateLinks(links, channelID, interactive), true
75 | }
76 | }
77 | if RegexpUrlImgurAlbum.MatchString(inputURL) {
78 | links, err := getImgurAlbumUrls(inputURL)
79 | if err != nil {
80 | fmt.Println("imgur album URL failed, ", inputURL, ",", err)
81 | } else if len(links) > 0 {
82 | return skipDuplicateLinks(links, channelID, interactive), true
83 | }
84 | }
85 | if RegexpUrlGoogleDrive.MatchString(inputURL) {
86 | links, err := getGoogleDriveUrls(inputURL)
87 | if err != nil {
88 | fmt.Println("google drive album URL failed, ", inputURL, ",", err)
89 | } else if len(links) > 0 {
90 | return skipDuplicateLinks(links, channelID, interactive), true
91 | }
92 | }
93 | if RegexpUrlFlickrPhoto.MatchString(inputURL) {
94 | links, err := getFlickrPhotoUrls(inputURL)
95 | if err != nil {
96 | fmt.Println("flickr photo URL failed, ", inputURL, ",", err)
97 | } else if len(links) > 0 {
98 | return skipDuplicateLinks(links, channelID, interactive), true
99 | }
100 | }
101 | if RegexpUrlFlickrAlbum.MatchString(inputURL) {
102 | links, err := getFlickrAlbumUrls(inputURL)
103 | if err != nil {
104 | fmt.Println("flickr album URL failed, ", inputURL, ",", err)
105 | } else if len(links) > 0 {
106 | return skipDuplicateLinks(links, channelID, interactive), true
107 | }
108 | }
109 | if RegexpUrlFlickrAlbumShort.MatchString(inputURL) {
110 | links, err := getFlickrAlbumShortUrls(inputURL)
111 | if err != nil {
112 | fmt.Println("flickr album short URL failed, ", inputURL, ",", err)
113 | } else if len(links) > 0 {
114 | return skipDuplicateLinks(links, channelID, interactive), true
115 | }
116 | }
117 | if RegexpUrlStreamable.MatchString(inputURL) {
118 | links, err := getStreamableUrls(inputURL)
119 | if err != nil {
120 | fmt.Println("streamable URL failed, ", inputURL, ",", err)
121 | } else if len(links) > 0 {
122 | return skipDuplicateLinks(links, channelID, interactive), true
123 | }
124 | }
125 | if DownloadTistorySites {
126 | if RegexpUrlPossibleTistorySite.MatchString(inputURL) {
127 | links, err := getPossibleTistorySiteUrls(inputURL)
128 | if err != nil {
129 | fmt.Println("checking for tistory site failed, ", inputURL, ",", err)
130 | } else if len(links) > 0 {
131 | return skipDuplicateLinks(links, channelID, interactive), true
132 | }
133 | }
134 | }
135 | if RegexpUrlGoogleDriveFolder.MatchString(inputURL) {
136 | if interactive {
137 | links, err := getGoogleDriveFolderUrls(inputURL)
138 | if err != nil {
139 | fmt.Println("google drive folder URL failed, ", inputURL, ",", err)
140 | } else if len(links) > 0 {
141 | return skipDuplicateLinks(links, channelID, interactive), true
142 | }
143 | } else {
144 | fmt.Println("google drive folder only accepted in interactive channels")
145 | }
146 | }
147 |
148 | if !interactive && isDiscordEmoji(inputURL) {
149 | fmt.Printf("skipped %s as it is a Discord emoji\n", inputURL)
150 | return nil, true
151 | }
152 |
153 | // try without queries
154 | parsedURL, err := url.Parse(inputURL)
155 | if err == nil {
156 | parsedURL.RawQuery = ""
157 | inputURLWithoutQueries := parsedURL.String()
158 | if inputURLWithoutQueries != inputURL {
159 | foundLinks, match := getDownloadLinks(inputURLWithoutQueries, channelID, interactive)
160 | if match {
161 |
162 | return skipDuplicateLinks(foundLinks, channelID, interactive), true
163 | }
164 | }
165 | }
166 |
167 | return skipDuplicateLinks(map[string]string{inputURL: ""}, channelID, interactive), false
168 | }
169 |
170 | // getDownloadItemsOfMessage will extract all unique download links out of a message
171 | func getDownloadItemsOfMessage(message *discordgo.Message) []*DownloadItem {
172 | var downloadItems []*DownloadItem
173 |
174 | linkTime, err := message.Timestamp.Parse()
175 | if err != nil {
176 | linkTime = time.Now()
177 | }
178 |
179 | rawLinks := getRawLinksOfMessage(message)
180 | for _, rawLink := range rawLinks {
181 | downloadLinks, _ := getDownloadLinks(
182 | rawLink.Link,
183 | message.ChannelID,
184 | false,
185 | )
186 | for link, filename := range downloadLinks {
187 | if rawLink.Filename != "" {
188 | filename = rawLink.Filename
189 | }
190 |
191 | downloadItems = append(downloadItems, &DownloadItem{
192 | Link: link,
193 | Filename: filename,
194 | Time: linkTime,
195 | })
196 | }
197 | }
198 |
199 | downloadItems = deduplicateDownloadItems(downloadItems)
200 |
201 | return downloadItems
202 | }
203 |
204 | // getRawLinksOfMessage will extract all raw links of a message
205 | func getRawLinksOfMessage(message *discordgo.Message) []*DownloadItem {
206 | var links []*DownloadItem
207 |
208 | if message.Author == nil {
209 | message.Author = new(discordgo.User)
210 | }
211 |
212 | for _, attachment := range message.Attachments {
213 | links = append(links, &DownloadItem{
214 | Link: attachment.URL,
215 | Filename: attachment.Filename,
216 | })
217 | }
218 |
219 | foundLinks := xurls.Strict().FindAllString(message.Content, -1)
220 | for _, foundLink := range foundLinks {
221 | links = append(links, &DownloadItem{
222 | Link: foundLink,
223 | })
224 | }
225 |
226 | for _, embed := range message.Embeds {
227 | if embed.URL != "" {
228 | links = append(links, &DownloadItem{
229 | Link: embed.URL,
230 | })
231 | }
232 |
233 | if embed.Description != "" {
234 | foundLinks = xurls.Strict().FindAllString(embed.Description, -1)
235 | for _, foundLink := range foundLinks {
236 | links = append(links, &DownloadItem{
237 | Link: foundLink,
238 | })
239 | }
240 | }
241 |
242 | if embed.Image != nil && embed.Image.URL != "" {
243 | links = append(links, &DownloadItem{
244 | Link: embed.Image.URL,
245 | })
246 | }
247 |
248 | if embed.Video != nil && embed.Video.URL != "" {
249 | links = append(links, &DownloadItem{
250 | Link: embed.Video.URL,
251 | })
252 | }
253 | }
254 |
255 | return links
256 | }
257 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/Seklfreak/discord-image-downloader-go
2 |
3 | go 1.15
4 |
5 | require (
6 | bou.ke/monkey v1.0.1 // indirect
7 | github.com/ChimeraCoder/anaconda v2.0.0+incompatible
8 | github.com/ChimeraCoder/tokenbucket v0.0.0-20131201223612-c5a927568de7 // indirect
9 | github.com/HouzuoGuo/tiedot v0.0.0-20190123063644-be8ab1f1598e
10 | github.com/Jeffail/gabs v1.2.0
11 | github.com/PuerkitoBio/goquery v1.5.0
12 | github.com/azr/backoff v0.0.0-20160115115103-53511d3c7330 // indirect
13 | github.com/bwmarrin/discordgo v0.22.0
14 | github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc // indirect
15 | github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect
16 | github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17 // indirect
17 | github.com/hashicorp/go-version v1.1.0
18 | github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff // indirect
19 | golang.org/x/net v0.0.0-20190322120337-addf6b3196f6
20 | golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914
21 | google.golang.org/api v0.2.0
22 | gopkg.in/ini.v1 v1.42.0
23 | mvdan.cc/xurls/v2 v2.2.0
24 | )
25 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg=
2 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
3 | cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
4 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
5 | git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
6 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
7 | github.com/ChimeraCoder/anaconda v2.0.0+incompatible h1:F0eD7CHXieZ+VLboCD5UAqCeAzJZxcr90zSCcuJopJs=
8 | github.com/ChimeraCoder/anaconda v2.0.0+incompatible/go.mod h1:TCt3MijIq3Qqo9SBtuW/rrM4x7rDfWqYWHj8T7hLcLg=
9 | github.com/ChimeraCoder/tokenbucket v0.0.0-20131201223612-c5a927568de7 h1:r+EmXjfPosKO4wfiMLe1XQictsIlhErTufbWUsjOTZs=
10 | github.com/ChimeraCoder/tokenbucket v0.0.0-20131201223612-c5a927568de7/go.mod h1:b2EuEMLSG9q3bZ95ql1+8oVqzzrTNSiOQqSXWFBzxeI=
11 | github.com/HouzuoGuo/tiedot v0.0.0-20190123063644-be8ab1f1598e h1:ZlkqcGyUHfuEnRRUbjFQBSiPceYOwMxcXiiEQ2dzWdg=
12 | github.com/HouzuoGuo/tiedot v0.0.0-20190123063644-be8ab1f1598e/go.mod h1:J2FcoVwTshOscfh8D4LCCVRoHJJQTeCAEkeRSVGnLQs=
13 | github.com/Jeffail/gabs v1.2.0 h1:uFhoIVTtsX7hV2RxNgWad8gMU+8OJdzFbOathJdhD3o=
14 | github.com/Jeffail/gabs v1.2.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
15 | github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
16 | github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
17 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
18 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
19 | github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
20 | github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
21 | github.com/azr/backoff v0.0.0-20160115115103-53511d3c7330 h1:ekDALXAVvY/Ub1UtNta3inKQwZ/jMB/zpOtD8rAYh78=
22 | github.com/azr/backoff v0.0.0-20160115115103-53511d3c7330/go.mod h1:nH+k0SvAt3HeiYyOlJpLLv1HG1p7KWP7qU9QPp2/pCo=
23 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
24 | github.com/bwmarrin/discordgo v0.22.0 h1:uBxY1HmlVCsW1IuaPjpCGT6A2DBwRn0nvOguQIxDdFM=
25 | github.com/bwmarrin/discordgo v0.22.0/go.mod h1:c1WtWUGN6nREDmzIpyTp/iD3VYt4Fpx+bVyfBG7JE+M=
26 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
27 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
28 | github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc h1:tP7tkU+vIsEOKiK+l/NSLN4uUtkyuxc6hgYpQeCWAeI=
29 | github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc/go.mod h1:ORH5Qp2bskd9NzSfKqAF7tKfONsEkCarTE5ESr/RVBw=
30 | github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad h1:Qk76DOWdOp+GlyDKBAG3Klr9cn7N+LcYc82AZ2S7+cA=
31 | github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad/go.mod h1:mPKfmRa823oBIgl2r20LeMSpTAteW5j7FLkc0vjmzyQ=
32 | github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17 h1:GOfMz6cRgTJ9jWV0qAezv642OhPnKEG7gtUjJSdStHE=
33 | github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17/go.mod h1:HfkOCN6fkKKaPSAeNq/er3xObxTW4VLeY6UUK895gLQ=
34 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
35 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
36 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
37 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
38 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
39 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
40 | github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
41 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
42 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
43 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
44 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
45 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
46 | github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
47 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
48 | github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
49 | github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
50 | github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
51 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
52 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
53 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
54 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
55 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
56 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
57 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
58 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
59 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
60 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
61 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
62 | github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
63 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
64 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
65 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
66 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
67 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
68 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
69 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
70 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
71 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
72 | github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
73 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
74 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
75 | github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
76 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
77 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
78 | go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A=
79 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
80 | golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
81 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
82 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
83 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
84 | golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
85 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
86 | golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
87 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
88 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
89 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
90 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
91 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
92 | golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
93 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
94 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
95 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
96 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
97 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
98 | golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 h1:78jEq2G3J16aXneH23HSnTQQTCwMHoyO8VEiUH+bpPM=
99 | golang.org/x/net v0.0.0-20190322120337-addf6b3196f6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
100 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
101 | golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
102 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
103 | golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914 h1:jIOcLT9BZzyJ9ce+IwwZ+aF9yeCqzrR+NrD68a/SHKw=
104 | golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
105 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
106 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
107 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
108 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
109 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
110 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
111 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
112 | golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
113 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
114 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
115 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
116 | golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
117 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
118 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
119 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
120 | google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
121 | google.golang.org/api v0.2.0 h1:B5VXkdjt7K2Gm6fGBC9C9a1OAKJDT95cTqwet+2zib0=
122 | google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU=
123 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
124 | google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
125 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
126 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
127 | google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
128 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
129 | google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
130 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
131 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
132 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
133 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
134 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
135 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
136 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
137 | gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
138 | gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
139 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
140 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
141 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
142 | honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
143 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
144 | mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A=
145 | mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
146 |
--------------------------------------------------------------------------------
/handler.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "strings"
6 | "time"
7 |
8 | "github.com/bwmarrin/discordgo"
9 | "mvdan.cc/xurls/v2"
10 | )
11 |
12 | // helpHandler displays the help message
13 | func helpHandler(message *discordgo.Message) {
14 | dg.ChannelMessageSend(
15 | message.ChannelID,
16 | "**** to download a link\n"+
17 | "**version** to find out the version\n"+
18 | "**stats** to view stats\n"+
19 | "**channels** to list active channels\n"+
20 | "**history** to download a full channel history\n"+
21 | "**help** to open this help\n",
22 | )
23 | }
24 |
25 | // versionHandler displays the current version
26 | func versionHandler(message *discordgo.Message) {
27 | result := fmt.Sprintf("discord-image-downloder-go **v%s**\n", VERSION)
28 |
29 | if !isLatestRelease() {
30 | result += fmt.Sprintf("**update available on <%s>**", RELEASE_URL)
31 | } else {
32 | result += "version is up to date"
33 | }
34 |
35 | dg.ChannelMessageSend(message.ChannelID, result)
36 | }
37 |
38 | // channelsHandler displays enabled channels
39 | func channelsHandler(message *discordgo.Message) {
40 | var err error
41 | var channel *discordgo.Channel
42 | var channelRecipientUsername string
43 |
44 | result := "**Channels**\n"
45 | for channelID, channelFolder := range ChannelWhitelist {
46 | channel, err = dg.State.Channel(channelID)
47 | if err != nil {
48 | continue
49 | }
50 |
51 | switch channel.Type {
52 | case discordgo.ChannelTypeDM:
53 | channelRecipientUsername = ""
54 | for _, recipient := range channel.Recipients {
55 | channelRecipientUsername = recipient.Username + ", "
56 | }
57 | channelRecipientUsername = strings.TrimRight(channelRecipientUsername, ", ")
58 | if channelRecipientUsername == "" {
59 | channelRecipientUsername = "N/A"
60 | }
61 | result += fmt.Sprintf("@%s (`#%s`): `%s`\n", channelRecipientUsername, channel.ID, channelFolder)
62 | default:
63 | result += fmt.Sprintf("<#%s> (`#%s`): `%s`\n", channel.ID, channel.ID, channelFolder)
64 | }
65 |
66 | }
67 | result += "**Interactive Channels**\n"
68 | for channelID, channelFolder := range InteractiveChannelWhitelist {
69 | channel, err = dg.State.Channel(channelID)
70 | if err != nil {
71 | continue
72 | }
73 |
74 | switch channel.Type {
75 | case discordgo.ChannelTypeDM:
76 | channelRecipientUsername = ""
77 | for _, recipient := range channel.Recipients {
78 | channelRecipientUsername = recipient.Username + ", "
79 | }
80 | channelRecipientUsername = strings.TrimRight(channelRecipientUsername, ", ")
81 | if channelRecipientUsername == "" {
82 | channelRecipientUsername = "N/A"
83 | }
84 | result += fmt.Sprintf("@%s (`#%s`): `%s`\n", channelRecipientUsername, channel.ID, channelFolder)
85 | default:
86 | result += fmt.Sprintf("<#%s> (`#%s`): `%s`\n", channel.ID, channel.ID, channelFolder)
87 | }
88 | }
89 |
90 | for _, page := range Pagify(result, "\n") {
91 | dg.ChannelMessageSend(message.ChannelID, page)
92 | }
93 | }
94 |
95 | func statsHandler(message *discordgo.Message) {
96 | channelStats := make(map[string]int)
97 | userStats := make(map[string]int)
98 | userGuilds := make(map[string]string)
99 |
100 | var i int
101 | myDB.Use("Downloads").ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) {
102 | downloadedImage := findDownloadedImageById(id)
103 | channelStats[downloadedImage.ChannelId] += 1
104 | userStats[downloadedImage.UserId] += 1
105 | if _, ok := userGuilds[downloadedImage.UserId]; !ok {
106 | channel, err := dg.State.Channel(downloadedImage.ChannelId)
107 | if err == nil && channel.GuildID != "" {
108 | userGuilds[downloadedImage.UserId] = channel.GuildID
109 | }
110 | }
111 | i++
112 | return true
113 | })
114 | channelStatsSorted := sortStringIntMapByValue(channelStats)
115 | userStatsSorted := sortStringIntMapByValue(userStats)
116 | replyMessage := fmt.Sprintf("I downloaded **%d** pictures in **%d** channels by **%d** users\n", i, len(channelStats), len(userStats))
117 |
118 | replyMessage += "**channel breakdown**\n"
119 | for _, downloads := range channelStatsSorted {
120 | channel, err := dg.State.Channel(downloads.Key)
121 | if err == nil {
122 | if channel.Type == discordgo.ChannelTypeDM {
123 | channelRecipientUsername := "N/A"
124 | for _, recipient := range channel.Recipients {
125 | channelRecipientUsername = recipient.Username
126 | }
127 | replyMessage += fmt.Sprintf("@%s (`#%s`): **%d** downloads\n", channelRecipientUsername, downloads.Key, downloads.Value)
128 | } else {
129 | guild, err := dg.State.Guild(channel.GuildID)
130 | if err == nil {
131 | replyMessage += fmt.Sprintf("#%s/%s (`#%s`): **%d** downloads\n", guild.Name, channel.Name, downloads.Key, downloads.Value)
132 | } else {
133 | fmt.Println(err)
134 | }
135 | }
136 | } else {
137 | fmt.Println(err)
138 | }
139 | }
140 | replyMessage += "**user breakdown**\n"
141 | var userI int
142 | for _, downloads := range userStatsSorted {
143 | userI++
144 | if userI > 10 {
145 | replyMessage += "_only the top 10 users get shown_\n"
146 | break
147 | }
148 | if guildId, ok := userGuilds[downloads.Key]; ok {
149 | user, err := dg.State.Member(guildId, downloads.Key)
150 | if err == nil {
151 | replyMessage += fmt.Sprintf("@%s: **%d** downloads\n", user.User.Username, downloads.Value)
152 | } else {
153 | replyMessage += fmt.Sprintf("@`%s`: **%d** downloads\n", downloads.Key, downloads.Value)
154 | }
155 | } else {
156 | replyMessage += fmt.Sprintf("@`%s`: **%d** downloads\n", downloads.Key, downloads.Value)
157 | }
158 | }
159 |
160 | for _, page := range Pagify(replyMessage, "\n") {
161 | dg.ChannelMessageSend(message.ChannelID, page)
162 | }
163 | }
164 |
165 | func historyHandler(message *discordgo.Message) {
166 | i := 0
167 | _, historyCommandIsSet := historyCommandActive[message.ChannelID]
168 | if !historyCommandIsSet || historyCommandActive[message.ChannelID] == "" {
169 | historyCommandActive[message.ChannelID] = ""
170 |
171 | idArray := strings.Split(message.Content, ",")
172 | for _, channelValue := range idArray {
173 | channelValue = strings.TrimSpace(channelValue)
174 | if folder, ok := ChannelWhitelist[channelValue]; ok {
175 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("downloading to `%s`", folder))
176 | historyCommandActive[message.ChannelID] = "downloading"
177 | lastBefore := ""
178 | var lastBeforeTime time.Time
179 | MessageRequestingLoop:
180 | for true {
181 | if lastBeforeTime != (time.Time{}) {
182 | fmt.Printf("[%s] Requesting 100 more messages, (before %s)\n", time.Now().Format(time.Stamp), lastBeforeTime)
183 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Requesting 100 more messages, (before %s)\n", lastBeforeTime))
184 | }
185 | messages, err := dg.ChannelMessages(channelValue, 100, lastBefore, "", "")
186 | if err == nil {
187 | if len(messages) <= 0 {
188 | delete(historyCommandActive, message.ChannelID)
189 | break MessageRequestingLoop
190 | }
191 | lastBefore = messages[len(messages)-1].ID
192 | lastBeforeTime, err = messages[len(messages)-1].Timestamp.Parse()
193 | if err != nil {
194 | fmt.Println(err)
195 | }
196 | for _, msg := range messages {
197 | fileTime := time.Now()
198 | if msg.Timestamp != "" {
199 | fileTime, err = msg.Timestamp.Parse()
200 | if err != nil {
201 | fmt.Println(err)
202 | }
203 | }
204 | if historyCommandActive[message.ChannelID] == "cancel" {
205 | delete(historyCommandActive, message.ChannelID)
206 | dg.ChannelMessageSend(message.ChannelID, "cancelled history downloading")
207 | break MessageRequestingLoop
208 | }
209 | for _, iAttachment := range msg.Attachments {
210 | if len(findDownloadedImageByUrl(iAttachment.URL)) == 0 {
211 | i++
212 | startDownload(iAttachment.URL, iAttachment.Filename, folder, msg.ChannelID, msg.Author.ID, fileTime)
213 | }
214 | }
215 | foundUrls := xurls.Strict().FindAllString(msg.Content, -1)
216 | for _, iFoundUrl := range foundUrls {
217 | links, _ := getDownloadLinks(iFoundUrl, msg.ChannelID, false)
218 | for link, filename := range links {
219 | if len(findDownloadedImageByUrl(link)) == 0 {
220 | i++
221 | startDownload(link, filename, folder, msg.ChannelID, msg.Author.ID, fileTime)
222 | }
223 | }
224 | }
225 | }
226 | } else {
227 | dg.ChannelMessageSend(message.ChannelID, err.Error())
228 | fmt.Println(err)
229 | delete(historyCommandActive, message.ChannelID)
230 | break MessageRequestingLoop
231 | }
232 | }
233 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("done, %d download links started!", i))
234 | } else {
235 | dg.ChannelMessageSend(message.ChannelID, "Please tell me one or multiple Channel IDs (separated by commas)\nPlease make sure the channels have been whitelisted before submitting.")
236 | }
237 | }
238 | } else if historyCommandActive[message.ChannelID] == "downloading" && strings.ToLower(message.Content) == "cancel" {
239 | historyCommandActive[message.ChannelID] = "cancel"
240 | }
241 | }
242 |
243 | func defaultHandler(message *discordgo.Message) {
244 | folderName := InteractiveChannelWhitelist[message.ChannelID]
245 |
246 | if link, ok := interactiveChannelLinkTemp[message.ChannelID]; ok {
247 | fileTime := time.Now()
248 | var err error
249 | if message.Timestamp != "" {
250 | fileTime, err = message.Timestamp.Parse()
251 | if err != nil {
252 | fmt.Println(err)
253 | }
254 | }
255 | if message.Content == "." {
256 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> started", link))
257 | dg.ChannelTyping(message.ChannelID)
258 | delete(interactiveChannelLinkTemp, message.ChannelID)
259 | links, _ := getDownloadLinks(link, message.ChannelID, true)
260 | for linkR, filename := range links {
261 | startDownload(linkR, filename, folderName, message.ChannelID, message.Author.ID, fileTime)
262 | }
263 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> finished", link))
264 | } else if strings.ToLower(message.Content) == "cancel" {
265 | delete(interactiveChannelLinkTemp, message.ChannelID)
266 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> cancelled", link))
267 | } else if IsValid(message.Content) {
268 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> started", link))
269 | dg.ChannelTyping(message.ChannelID)
270 | delete(interactiveChannelLinkTemp, message.ChannelID)
271 | links, _ := getDownloadLinks(link, message.ChannelID, true)
272 | for linkR, filename := range links {
273 | startDownload(linkR, filename, message.Content, message.ChannelID, message.Author.ID, fileTime)
274 | }
275 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> finished", link))
276 | } else {
277 | dg.ChannelMessageSend(message.ChannelID, "invalid path")
278 | }
279 | } else {
280 | _ = folderName
281 | foundLinks := false
282 | for _, iAttachment := range message.Attachments {
283 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Where do you want to save <%s>?\nType **.** for default path or **cancel** to cancel the download %s", iAttachment.URL, folderName))
284 | interactiveChannelLinkTemp[message.ChannelID] = iAttachment.URL
285 | foundLinks = true
286 | }
287 | foundUrls := xurls.Strict().FindAllString(message.Content, -1)
288 | for _, iFoundUrl := range foundUrls {
289 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Where do you want to save <%s>?\nType **.** for default path or **cancel** to cancel the download %s", iFoundUrl, folderName))
290 | interactiveChannelLinkTemp[message.ChannelID] = iFoundUrl
291 | foundLinks = true
292 | }
293 | if foundLinks == false {
294 | dg.ChannelMessageSend(message.ChannelID, "unable to find valid link")
295 | }
296 | }
297 | }
298 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "flag"
7 | "fmt"
8 | "io/ioutil"
9 | "mime"
10 | "net/http"
11 | "net/url"
12 | "os"
13 | "os/signal"
14 | "path"
15 | "path/filepath"
16 | "regexp"
17 | "sort"
18 | "strconv"
19 | "strings"
20 | "syscall"
21 | "time"
22 |
23 | "github.com/ChimeraCoder/anaconda"
24 | "github.com/HouzuoGuo/tiedot/db"
25 | "github.com/Jeffail/gabs"
26 | "github.com/PuerkitoBio/goquery"
27 | "github.com/bwmarrin/discordgo"
28 | "github.com/hashicorp/go-version"
29 | "golang.org/x/net/context"
30 | "golang.org/x/net/html"
31 | "golang.org/x/oauth2/google"
32 | "google.golang.org/api/drive/v3"
33 | "google.golang.org/api/googleapi"
34 | "gopkg.in/ini.v1"
35 | )
36 |
37 | var (
38 | ChannelWhitelist map[string]string
39 | InteractiveChannelWhitelist map[string]string
40 | dg *discordgo.Session
41 | DownloadTistorySites bool
42 | interactiveChannelLinkTemp map[string]string
43 | DiscordUserId string
44 | myDB *db.DB
45 | historyCommandActive map[string]string
46 | MaxDownloadRetries int
47 | flickrApiKey string
48 | twitterClient *anaconda.TwitterApi
49 | DownloadTimeout int
50 | SendNoticesToInteractiveChannels bool
51 | StatusEnabled bool
52 | StatusType string
53 | StatusLabel discordgo.GameType
54 | StatusSuffix string
55 | clientCredentialsJson string
56 | DriveService *drive.Service
57 | RegexpFilename *regexp.Regexp
58 | )
59 |
60 | type GfycatObject struct {
61 | GfyItem struct {
62 | Mp4URL string `json:"mp4Url"`
63 | } `json:"gfyItem"`
64 | }
65 |
66 | type ImgurAlbumObject struct {
67 | Data []struct {
68 | Link string
69 | }
70 | }
71 |
72 | func main() {
73 | fmt.Printf("> discord-image-downloader-go v%s -- discordgo v%s\n", VERSION, discordgo.VERSION)
74 | if !isLatestRelease() {
75 | fmt.Printf("update available on %s !\n", RELEASE_URL)
76 | }
77 |
78 | var err error
79 |
80 | var configFile string
81 | flag.StringVar(&configFile, "config", DEFAULT_CONFIG_FILE, "config file to read from")
82 |
83 | flag.Parse()
84 |
85 | if configFile == "" {
86 | configFile = DEFAULT_CONFIG_FILE
87 | }
88 |
89 | fmt.Printf("reading config from %s\n", configFile)
90 | cfg, err := ini.Load(configFile)
91 | if err != nil {
92 | fmt.Println("unable to read config file", err)
93 | cfg = ini.Empty()
94 | }
95 |
96 | if (!cfg.Section("auth").HasKey("email") ||
97 | !cfg.Section("auth").HasKey("password")) &&
98 | !cfg.Section("auth").HasKey("token") {
99 | cfg.Section("auth").NewKey("email", "your@email.com")
100 | cfg.Section("auth").NewKey("password", "your password")
101 | cfg.Section("general").NewKey("skip edits", "true")
102 | cfg.Section("general").NewKey("download tistory sites", "false")
103 | cfg.Section("general").NewKey("max download retries", "5")
104 | cfg.Section("general").NewKey("download timeout", "60")
105 | cfg.Section("general").NewKey("send notices to interactive channels", "false")
106 | cfg.Section("status").NewKey("status enabled", "true")
107 | cfg.Section("status").NewKey("status type", "online")
108 | cfg.Section("status").NewKey("status label", fmt.Sprint(discordgo.GameTypeWatching))
109 | cfg.Section("status").NewKey("status suffix", "downloaded pictures")
110 | cfg.Section("channels").NewKey("channelid1", "C:\\full\\path\\1")
111 | cfg.Section("channels").NewKey("channelid2", "C:\\full\\path\\2")
112 | cfg.Section("channels").NewKey("channelid3", "C:\\full\\path\\3")
113 | cfg.Section("flickr").NewKey("api key", "your flickr api key")
114 | cfg.Section("twitter").NewKey("consumer key", "your consumer key")
115 | cfg.Section("twitter").NewKey("consumer secret", "your consumer secret")
116 | cfg.Section("twitter").NewKey("access token", "your access token")
117 | cfg.Section("twitter").NewKey("access token secret", "your access token secret")
118 | err = cfg.SaveTo("config.ini")
119 |
120 | if err != nil {
121 | fmt.Println("unable to write config file", err)
122 | return
123 | }
124 | fmt.Println("Wrote config file, please fill out and restart the program")
125 | return
126 | }
127 |
128 | myDB, err = db.OpenDB(DATABASE_DIR)
129 | if err != nil {
130 | fmt.Println("unable to create db", err)
131 | return
132 | }
133 | if myDB.Use("Downloads") == nil {
134 | fmt.Println("Creating new database...")
135 | if err := myDB.Create("Downloads"); err != nil {
136 | fmt.Println("unable to create db", err)
137 | return
138 | }
139 | if err := myDB.Use("Downloads").Index([]string{"Url"}); err != nil {
140 | fmt.Println("unable to create index", err)
141 | return
142 | }
143 | }
144 |
145 | ChannelWhitelist = cfg.Section("channels").KeysHash()
146 | InteractiveChannelWhitelist = cfg.Section("interactive channels").KeysHash()
147 | interactiveChannelLinkTemp = make(map[string]string)
148 | historyCommandActive = make(map[string]string)
149 | flickrApiKey = cfg.Section("flickr").Key("api key").MustString("yourflickrapikey")
150 | twitterConsumerKey := cfg.Section("twitter").Key("consumer key").MustString("your consumer key")
151 | twitterConsumerSecret := cfg.Section("twitter").Key("consumer secret").MustString("your consumer secret")
152 | twitterAccessToken := cfg.Section("twitter").Key("access token").MustString("your access token")
153 | twitterAccessTokenSecret := cfg.Section("twitter").Key("access token secret").MustString("your access token secret")
154 | if twitterAccessToken != "" &&
155 | twitterAccessTokenSecret != "" &&
156 | twitterConsumerKey != "" &&
157 | twitterConsumerSecret != "" {
158 | twitterClient = anaconda.NewTwitterApiWithCredentials(
159 | twitterAccessToken,
160 | twitterAccessTokenSecret,
161 | twitterConsumerKey,
162 | twitterConsumerSecret,
163 | )
164 | }
165 |
166 | err = initRegex()
167 | if err != nil {
168 | fmt.Println("error initialising regex,", err.Error())
169 | return
170 | }
171 |
172 | if cfg.Section("auth").HasKey("token") {
173 | dg, err = discordgo.New(cfg.Section("auth").Key("token").String())
174 | } else {
175 | dg, err = discordgo.New(
176 | cfg.Section("auth").Key("email").String(),
177 | cfg.Section("auth").Key("password").String())
178 | }
179 | if err != nil {
180 | // Newer discordgo throws this error for some reason with Email/Password login
181 | if err.Error() != "Unable to fetch discord authentication token. " {
182 | fmt.Println("error creating Discord session,", err)
183 | return
184 | }
185 | }
186 |
187 | dg.AddHandler(messageCreate)
188 |
189 | if cfg.Section("general").HasKey("skip edits") {
190 | if cfg.Section("general").Key("skip edits").MustBool() == false {
191 | dg.AddHandler(messageUpdate)
192 | }
193 | }
194 |
195 | DownloadTistorySites = cfg.Section("general").Key("download tistory sites").MustBool()
196 | MaxDownloadRetries = cfg.Section("general").Key("max download retries").MustInt(3)
197 | DownloadTimeout = cfg.Section("general").Key("download timeout").MustInt(60)
198 | SendNoticesToInteractiveChannels = cfg.Section("general").Key("send notices to interactive channels").MustBool(false)
199 |
200 | StatusEnabled = cfg.Section("status").Key("status enabled").MustBool(true)
201 | StatusType = cfg.Section("status").Key("status type").MustString("online")
202 | StatusLabel = discordgo.GameType(cfg.Section("status").Key("status label").MustInt(int(discordgo.GameTypeWatching)))
203 | StatusSuffix = cfg.Section("status").Key("status suffix").MustString("downloaded pictures")
204 |
205 | // setup google drive client
206 | clientCredentialsJson = cfg.Section("google").Key("client credentials json").MustString("")
207 | if clientCredentialsJson != "" {
208 | ctx := context.Background()
209 | authJson, err := ioutil.ReadFile(clientCredentialsJson)
210 | if err != nil {
211 | fmt.Println("error opening google credentials json,", err)
212 | } else {
213 | config, err := google.JWTConfigFromJSON(authJson, drive.DriveReadonlyScope)
214 | if err != nil {
215 | fmt.Println("error parsing google credentials json,", err)
216 | } else {
217 | client := config.Client(ctx)
218 | DriveService, err = drive.New(client)
219 | if err != nil {
220 | fmt.Println("error setting up google drive client,", err)
221 | }
222 | }
223 | }
224 | }
225 | dg.LogLevel = -1 // to ignore dumb wsapi error
226 | err = dg.Open()
227 | if err != nil {
228 | fmt.Println("error opening connection,", err)
229 | return
230 | }
231 | dg.LogLevel = 0 // reset
232 |
233 | u, err := dg.User("@me")
234 | if err != nil {
235 | fmt.Println("error obtaining account details,", err)
236 | }
237 |
238 | fmt.Printf("Client is now connected as %s. Press CTRL-C to exit.\n",
239 | u.Username)
240 | DiscordUserId = u.ID
241 |
242 | updateDiscordStatus()
243 |
244 | // keep program running until CTRL-C is pressed.
245 | sc := make(chan os.Signal, 1)
246 | signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt)
247 | <-sc
248 | fmt.Println("Closing database...")
249 | myDB.Close()
250 | fmt.Println("Logging out of Discord...")
251 | dg.Close()
252 | fmt.Println("Exiting...")
253 | return
254 | }
255 |
256 | func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
257 | handleDiscordMessage(m.Message)
258 | }
259 |
260 | func messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) {
261 | if m.EditedTimestamp != discordgo.Timestamp("") {
262 | handleDiscordMessage(m.Message)
263 | }
264 | }
265 |
266 | func skipDuplicateLinks(linkList map[string]string, channelID string, interactive bool) map[string]string {
267 | if interactive == false {
268 | newList := make(map[string]string, 0)
269 | for link, filename := range linkList {
270 | downloadedImages := findDownloadedImageByUrl(link)
271 | isMatched := false
272 | for _, downloadedImage := range downloadedImages {
273 | if downloadedImage.ChannelId == channelID {
274 | isMatched = true
275 | }
276 | }
277 | if isMatched == false {
278 | newList[link] = filename
279 | } else {
280 | fmt.Println("url already downloaded in this channel:", link)
281 | }
282 | }
283 | return newList
284 | }
285 | return linkList
286 | }
287 |
288 | func handleDiscordMessage(m *discordgo.Message) {
289 | if folderName, ok := ChannelWhitelist[m.ChannelID]; ok {
290 | // download from whitelisted channels
291 | downloadItems := getDownloadItemsOfMessage(m)
292 |
293 | for _, downloadItem := range downloadItems {
294 | startDownload(
295 | downloadItem.Link,
296 | downloadItem.Filename,
297 | folderName,
298 | m.ChannelID,
299 | m.Author.ID,
300 | downloadItem.Time,
301 | )
302 | }
303 |
304 | } else if _, ok := InteractiveChannelWhitelist[m.ChannelID]; ok {
305 | // handle interactive channel
306 |
307 | // skip messages from the Bot
308 | if m.Author == nil || m.Author.ID == DiscordUserId {
309 | return
310 | }
311 |
312 | dg.ChannelTyping(m.ChannelID)
313 |
314 | args := strings.Fields(m.Content)
315 | if len(args) <= 0 {
316 | return
317 | }
318 |
319 | _, historyCommandIsActive := historyCommandActive[m.ChannelID]
320 |
321 | switch strings.ToLower(args[0]) {
322 | case "help":
323 | helpHandler(m)
324 | case "version":
325 | versionHandler(m)
326 | case "channels":
327 | channelsHandler(m)
328 | case "stats":
329 | statsHandler(m)
330 | case "history":
331 | historyHandler(m)
332 | default:
333 | if historyCommandIsActive {
334 | historyHandler(m)
335 | return
336 | }
337 |
338 | defaultHandler(m)
339 | }
340 | }
341 | }
342 |
343 | type GithubReleaseApiObject struct {
344 | TagName string `json:"tag_name"`
345 | }
346 |
347 | func isLatestRelease() bool {
348 | githubReleaseApiObject := new(GithubReleaseApiObject)
349 | getJson(RELEASE_API_URL, githubReleaseApiObject)
350 | currentVer, err := version.NewVersion(VERSION)
351 | if err != nil {
352 | fmt.Println(err)
353 | return true
354 | }
355 | lastVer, err := version.NewVersion(githubReleaseApiObject.TagName)
356 | if err != nil {
357 | fmt.Println(err)
358 | return true
359 | }
360 | if lastVer.GreaterThan(currentVer) {
361 | return false
362 | }
363 | return true
364 | }
365 |
366 | // http://stackoverflow.com/a/35240286/1443726
367 | func IsValid(fp string) bool {
368 | // Check if file already exists
369 | if _, err := os.Stat(fp); err == nil {
370 | return true
371 | }
372 |
373 | // Attempt to create it
374 | var d []byte
375 | if err := ioutil.WriteFile(fp, d, 0644); err == nil {
376 | os.Remove(fp) // And delete it
377 | return true
378 | }
379 |
380 | return false
381 | }
382 |
383 | func getTwitterUrls(url string) (map[string]string, error) {
384 | parts := strings.Split(url, ":")
385 | if len(parts) < 2 {
386 | return nil, errors.New("unable to parse twitter url")
387 | }
388 | return map[string]string{"https:" + parts[1] + ":orig": filenameFromUrl(parts[1])}, nil
389 | }
390 |
391 | func getTwitterParamsUrls(url string) (map[string]string, error) {
392 | matches := RegexpUrlTwitterParams.FindStringSubmatch(url)
393 |
394 | return map[string]string{
395 | "https://pbs.twimg.com/media/" + matches[3] + "." + matches[4] + ":orig": matches[3] + "." + matches[4],
396 | }, nil
397 | }
398 |
399 | func getTwitterStatusUrls(url string, channelID string) (map[string]string, error) {
400 | if twitterClient == nil {
401 | return nil, errors.New("invalid twitter api keys set")
402 | }
403 |
404 | matches := RegexpUrlTwitterStatus.FindStringSubmatch(url)
405 | statusId, err := strconv.ParseInt(matches[4], 10, 64)
406 | if err != nil {
407 | return nil, err
408 | }
409 |
410 | tweet, err := twitterClient.GetTweet(statusId, nil)
411 | if err != nil {
412 | return nil, err
413 | }
414 |
415 | links := make(map[string]string)
416 | for _, tweetMedia := range tweet.ExtendedEntities.Media {
417 | if len(tweetMedia.VideoInfo.Variants) > 0 {
418 | var lastVideoVariant anaconda.Variant
419 | for _, videoVariant := range tweetMedia.VideoInfo.Variants {
420 | if videoVariant.Bitrate >= lastVideoVariant.Bitrate {
421 | lastVideoVariant = videoVariant
422 | }
423 | }
424 | if lastVideoVariant.Url != "" {
425 | links[lastVideoVariant.Url] = ""
426 | }
427 | } else {
428 | foundUrls, _ := getDownloadLinks(tweetMedia.Media_url_https, channelID, false)
429 | for foundUrlKey, foundUrlValue := range foundUrls {
430 | links[foundUrlKey] = foundUrlValue
431 | }
432 | }
433 | }
434 | for _, tweetUrl := range tweet.Entities.Urls {
435 | foundUrls, _ := getDownloadLinks(tweetUrl.Expanded_url, channelID, false)
436 | for foundUrlKey, foundUrlValue := range foundUrls {
437 | links[foundUrlKey] = foundUrlValue
438 | }
439 | }
440 |
441 | return links, nil
442 | }
443 |
444 | func getGfycatUrls(url string) (map[string]string, error) {
445 | parts := strings.Split(url, "/")
446 | if len(parts) < 3 {
447 | return nil, errors.New("unable to parse gfycat url")
448 | } else {
449 | gfycatId := parts[len(parts)-1]
450 | gfycatObject := new(GfycatObject)
451 | getJson("https://api.gfycat.com/v1/gfycats/"+gfycatId, gfycatObject)
452 | gfycatUrl := gfycatObject.GfyItem.Mp4URL
453 | if url == "" {
454 | return nil, errors.New("failed to read response from gfycat")
455 | }
456 | return map[string]string{gfycatUrl: ""}, nil
457 | }
458 | }
459 |
460 | func getInstagramUrls(url string) (map[string]string, error) {
461 | username, shortcode := getInstagramInfo(url)
462 | filename := fmt.Sprintf("instagram %s - %s", username, shortcode)
463 | // if instagram video
464 | videoUrl := getInstagramVideoUrl(url)
465 | if videoUrl != "" {
466 | return map[string]string{videoUrl: filename + filepathExtension(videoUrl)}, nil
467 | }
468 | // if instagram album
469 | albumUrls := getInstagramAlbumUrls(url)
470 | if len(albumUrls) > 0 {
471 | //fmt.Println("is instagram album")
472 | links := make(map[string]string)
473 | for i, albumUrl := range albumUrls {
474 | links[albumUrl] = filename + " " + strconv.Itoa(i+1) + filepathExtension(albumUrl)
475 | }
476 | return links, nil
477 | }
478 | // if instagram picture
479 | afterLastSlash := strings.LastIndex(url, "/")
480 | mediaUrl := url[:afterLastSlash]
481 | mediaUrl += strings.Replace(strings.Replace(url[afterLastSlash:], "?", "&", -1), "/", "/media/?size=l", -1)
482 | return map[string]string{mediaUrl: filename + ".jpg"}, nil
483 | }
484 |
485 | func getInstagramInfo(url string) (string, string) {
486 | resp, err := http.Get(url)
487 |
488 | if err != nil {
489 | return "N/A", "N/A"
490 | }
491 |
492 | defer resp.Body.Close()
493 | z := html.NewTokenizer(resp.Body)
494 |
495 | ParseLoop:
496 | for {
497 | tt := z.Next()
498 | switch {
499 | case tt == html.ErrorToken:
500 | break ParseLoop
501 | }
502 | if tt == html.StartTagToken || tt == html.SelfClosingTagToken {
503 | t := z.Token()
504 | for _, a := range t.Attr {
505 | if a.Key == "type" {
506 | if a.Val == "text/javascript" {
507 | z.Next()
508 | content := string(z.Text())
509 | if strings.Contains(content, "window._sharedData = ") {
510 | content = strings.Replace(content, "window._sharedData = ", "", 1)
511 | content = content[:len(content)-1]
512 | jsonParsed, err := gabs.ParseJSON([]byte(content))
513 | if err != nil {
514 | fmt.Println("error parsing instagram json: ", err)
515 | continue ParseLoop
516 | }
517 | entryChildren, err := jsonParsed.Path("entry_data.PostPage").Children()
518 | if err != nil {
519 | fmt.Println("unable to find entries children: ", err)
520 | continue ParseLoop
521 | }
522 | for _, entryChild := range entryChildren {
523 | shortcode := entryChild.Path("graphql.shortcode_media.shortcode").Data().(string)
524 | username := entryChild.Path("graphql.shortcode_media.owner.username").Data().(string)
525 | return username, shortcode
526 | }
527 | }
528 | }
529 | }
530 | }
531 | }
532 | }
533 | return "N/A", "N/A"
534 | }
535 |
536 | func getImgurSingleUrls(url string) (map[string]string, error) {
537 | url = regexp.MustCompile(`(r\/[^\/]+\/)`).ReplaceAllString(url, "") // remove subreddit url
538 | fmt.Println(url)
539 | url = strings.Replace(url, "imgur.com/", "imgur.com/download/", -1)
540 | url = strings.Replace(url, ".gifv", "", -1)
541 | return map[string]string{url: ""}, nil
542 | }
543 |
544 | func getImgurAlbumUrls(url string) (map[string]string, error) {
545 | url = regexp.MustCompile(`(#[A-Za-z0-9]+)?$`).ReplaceAllString(url, "") // remove anchor
546 | afterLastSlash := strings.LastIndex(url, "/")
547 | albumId := url[afterLastSlash+1:]
548 | headers := make(map[string]string)
549 | headers["Authorization"] = "Client-ID " + IMGUR_CLIENT_ID
550 | imgurAlbumObject := new(ImgurAlbumObject)
551 | getJsonWithHeaders("https://api.imgur.com/3/album/"+albumId+"/images", imgurAlbumObject, headers)
552 | links := make(map[string]string)
553 | for _, v := range imgurAlbumObject.Data {
554 | links[v.Link] = ""
555 | }
556 | if len(links) <= 0 {
557 | return getImgurSingleUrls(url)
558 | }
559 | fmt.Printf("[%s] Found imgur album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url)
560 | return links, nil
561 | }
562 |
563 | func getGoogleDriveUrls(url string) (map[string]string, error) {
564 | parts := strings.Split(url, "/")
565 | if len(parts) != 7 {
566 | return nil, errors.New("unable to parse google drive url")
567 | }
568 | fileId := parts[len(parts)-2]
569 | return map[string]string{"https://drive.google.com/uc?export=download&id=" + fileId: ""}, nil
570 | }
571 |
572 | func getGoogleDriveFolderUrls(url string) (map[string]string, error) {
573 | matches := RegexpUrlGoogleDriveFolder.FindStringSubmatch(url)
574 | if len(matches) < 4 || matches[3] == "" {
575 | return nil, errors.New("unable to find google drive folder ID in link")
576 | }
577 | if DriveService.BasePath == "" {
578 | return nil, errors.New("please set up google credentials")
579 | }
580 | googleDriveFolderID := matches[3]
581 |
582 | links := make(map[string]string)
583 |
584 | driveQuery := fmt.Sprintf("\"%s\" in parents", googleDriveFolderID)
585 | driveFields := "nextPageToken, files(id)"
586 | result, err := DriveService.Files.List().Q(driveQuery).Fields(googleapi.Field(driveFields)).PageSize(1000).Do()
587 | if err != nil {
588 | fmt.Println("driveQuery:", driveQuery)
589 | fmt.Println("driveFields:", driveFields)
590 | fmt.Println("err:", err)
591 | return nil, err
592 | }
593 | for _, file := range result.Files {
594 | fileUrl := "https://drive.google.com/uc?export=download&id=" + file.Id
595 | links[fileUrl] = ""
596 | }
597 |
598 | for {
599 | if result.NextPageToken == "" {
600 | break
601 | }
602 | result, err = DriveService.Files.List().Q(driveQuery).Fields(googleapi.Field(driveFields)).PageSize(1000).PageToken(result.NextPageToken).Do()
603 | if err != nil {
604 | return nil, err
605 | }
606 | for _, file := range result.Files {
607 | links[file.Id] = ""
608 | }
609 | }
610 | return links, nil
611 | }
612 |
613 | type FlickrPhotoSizeObject struct {
614 | Label string `json:"label"`
615 | Width int `json:"width,int,string"`
616 | Height int `json:"height,int,string"`
617 | Source string `json:"source"`
618 | URL string `json:"url"`
619 | Media string `json:"media"`
620 | }
621 |
622 | type FlickrPhotoObject struct {
623 | Sizes struct {
624 | Canblog int `json:"canblog"`
625 | Canprint int `json:"canprint"`
626 | Candownload int `json:"candownload"`
627 | Size []FlickrPhotoSizeObject `json:"size"`
628 | } `json:"sizes"`
629 | Stat string `json:"stat"`
630 | }
631 |
632 | func getFlickrUrlFromPhotoId(photoId string) string {
633 | reqUrl := fmt.Sprintf("https://www.flickr.com/services/rest/?format=json&nojsoncallback=1&method=%s&api_key=%s&photo_id=%s",
634 | "flickr.photos.getSizes", flickrApiKey, photoId)
635 | flickrPhoto := new(FlickrPhotoObject)
636 | getJson(reqUrl, flickrPhoto)
637 | var bestSize FlickrPhotoSizeObject
638 | for _, size := range flickrPhoto.Sizes.Size {
639 | if bestSize.Label == "" {
640 | bestSize = size
641 | } else {
642 | if size.Width > bestSize.Width || size.Height > bestSize.Height {
643 | bestSize = size
644 | }
645 | }
646 | }
647 | return bestSize.Source
648 | }
649 |
650 | func getFlickrPhotoUrls(url string) (map[string]string, error) {
651 | if flickrApiKey == "" || flickrApiKey == "yourflickrapikey" || flickrApiKey == "your flickr api key" {
652 | return nil, errors.New("invalid flickr api key set")
653 | }
654 | matches := RegexpUrlFlickrPhoto.FindStringSubmatch(url)
655 | photoId := matches[5]
656 | if photoId == "" {
657 | return nil, errors.New("unable to get photo id from url")
658 | }
659 | return map[string]string{getFlickrUrlFromPhotoId(photoId): ""}, nil
660 | }
661 |
662 | type FlickrAlbumObject struct {
663 | Photoset struct {
664 | ID string `json:"id"`
665 | Primary string `json:"primary"`
666 | Owner string `json:"owner"`
667 | Ownername string `json:"ownername"`
668 | Photo []struct {
669 | ID string `json:"id"`
670 | Secret string `json:"secret"`
671 | Server string `json:"server"`
672 | Farm int `json:"farm"`
673 | Title string `json:"title"`
674 | Isprimary string `json:"isprimary"`
675 | Ispublic int `json:"ispublic"`
676 | Isfriend int `json:"isfriend"`
677 | Isfamily int `json:"isfamily"`
678 | } `json:"photo"`
679 | Page int `json:"page"`
680 | PerPage int `json:"per_page"`
681 | Perpage int `json:"perpage"`
682 | Pages int `json:"pages"`
683 | Total string `json:"total"`
684 | Title string `json:"title"`
685 | } `json:"photoset"`
686 | Stat string `json:"stat"`
687 | }
688 |
689 | func getFlickrAlbumUrls(url string) (map[string]string, error) {
690 | if flickrApiKey == "" || flickrApiKey == "yourflickrapikey" {
691 | return nil, errors.New("invalid flickr api key set")
692 | }
693 | matches := RegexpUrlFlickrAlbum.FindStringSubmatch(url)
694 | if len(matches) < 10 || matches[9] == "" {
695 | return nil, errors.New("unable to find flickr album ID in link")
696 | }
697 | albumId := matches[9]
698 | if albumId == "" {
699 | return nil, errors.New("unable to get album id from url")
700 | }
701 | reqUrl := fmt.Sprintf("https://www.flickr.com/services/rest/?format=json&nojsoncallback=1&method=%s&api_key=%s&photoset_id=%s&per_page=500",
702 | "flickr.photosets.getPhotos", flickrApiKey, albumId)
703 | flickrAlbum := new(FlickrAlbumObject)
704 | getJson(reqUrl, flickrAlbum)
705 | links := make(map[string]string)
706 | for _, photo := range flickrAlbum.Photoset.Photo {
707 | links[getFlickrUrlFromPhotoId(photo.ID)] = ""
708 | }
709 | fmt.Printf("[%s] Found flickr album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url)
710 | return links, nil
711 | }
712 |
713 | func getFlickrAlbumShortUrls(url string) (map[string]string, error) {
714 | result, err := http.Get(url)
715 | if err != nil {
716 | return nil, errors.New("error getting long url from shortened flickr album url: " + err.Error())
717 | }
718 | if RegexpUrlFlickrAlbum.MatchString(result.Request.URL.String()) {
719 | return getFlickrAlbumUrls(result.Request.URL.String())
720 | }
721 | return nil, errors.New("got invalid url while trying to get long url from short flickr album url")
722 | }
723 |
724 | type StreamableObject struct {
725 | Status int `json:"status"`
726 | Title string `json:"title"`
727 | Files struct {
728 | Mp4 struct {
729 | URL string `json:"url"`
730 | Width int `json:"width"`
731 | Height int `json:"height"`
732 | } `json:"mp4"`
733 | Mp4Mobile struct {
734 | URL string `json:"url"`
735 | Width int `json:"width"`
736 | Height int `json:"height"`
737 | } `json:"mp4-mobile"`
738 | } `json:"files"`
739 | URL string `json:"url"`
740 | ThumbnailURL string `json:"thumbnail_url"`
741 | Message interface{} `json:"message"`
742 | }
743 |
744 | func getStreamableUrls(url string) (map[string]string, error) {
745 | matches := RegexpUrlStreamable.FindStringSubmatch(url)
746 | shortcode := matches[3]
747 | if shortcode == "" {
748 | return nil, errors.New("unable to get shortcode from url")
749 | }
750 | reqUrl := fmt.Sprintf("https://api.streamable.com/videos/%s", shortcode)
751 | streamable := new(StreamableObject)
752 | getJson(reqUrl, streamable)
753 | if streamable.Status != 2 || streamable.Files.Mp4.URL == "" {
754 | return nil, errors.New("streamable object has no download candidate")
755 | }
756 | link := streamable.Files.Mp4.URL
757 | if !strings.HasPrefix(link, "http") {
758 | link = "https:" + link
759 | }
760 | links := make(map[string]string)
761 | links[link] = ""
762 | return links, nil
763 | }
764 |
765 | func getPossibleTistorySiteUrls(url string) (map[string]string, error) {
766 | client := new(http.Client)
767 | request, err := http.NewRequest("HEAD", url, nil)
768 | if err != nil {
769 | return nil, err
770 | }
771 | request.Header.Add("Accept-Encoding", "identity")
772 | request.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36")
773 | respHead, err := client.Do(request)
774 | if err != nil {
775 | return nil, err
776 | }
777 |
778 | contentType := ""
779 | for headerKey, headerValue := range respHead.Header {
780 | if headerKey == "Content-Type" {
781 | contentType = headerValue[0]
782 | }
783 | }
784 | if !strings.Contains(contentType, "text/html") {
785 | return nil, nil
786 | }
787 |
788 | request, err = http.NewRequest("GET", url, nil)
789 | if err != nil {
790 | return nil, err
791 | }
792 | request.Header.Add("Accept-Encoding", "identity")
793 | request.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36")
794 | resp, err := client.Do(request)
795 | if err != nil {
796 | return nil, err
797 | }
798 |
799 | doc, err := goquery.NewDocumentFromResponse(resp)
800 | if err != nil {
801 | return nil, err
802 | }
803 |
804 | var links = make(map[string]string)
805 |
806 | doc.Find(".article img, #content img, div[role=main] img, .section_blogview img").Each(func(i int, s *goquery.Selection) {
807 | foundUrl, exists := s.Attr("src")
808 | if exists {
809 | isTistoryCdnUrl := RegexpUrlTistoryLegacyWithCDN.MatchString(foundUrl)
810 | isTistoryUrl := RegexpUrlTistoryLegacy.MatchString(foundUrl)
811 | if isTistoryCdnUrl == true {
812 | finalTistoryUrls, _ := getTistoryWithCDNUrls(foundUrl)
813 | if len(finalTistoryUrls) > 0 {
814 | for finalTistoryUrl := range finalTistoryUrls {
815 | foundFilename := s.AttrOr("filename", "")
816 | links[finalTistoryUrl] = foundFilename
817 | }
818 | }
819 | } else if isTistoryUrl == true {
820 | finalTistoryUrls, _ := getLegacyTistoryUrls(foundUrl)
821 | if len(finalTistoryUrls) > 0 {
822 | for finalTistoryUrl := range finalTistoryUrls {
823 | foundFilename := s.AttrOr("filename", "")
824 | links[finalTistoryUrl] = foundFilename
825 | }
826 | }
827 | }
828 | }
829 | })
830 |
831 | if len(links) > 0 {
832 | fmt.Printf("[%s] Found tistory album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url)
833 | }
834 | return links, nil
835 | }
836 |
837 | func getJson(url string, target interface{}) error {
838 | r, err := http.Get(url)
839 | if err != nil {
840 | return err
841 | }
842 | defer r.Body.Close()
843 |
844 | return json.NewDecoder(r.Body).Decode(target)
845 | }
846 |
847 | func getJsonWithHeaders(url string, target interface{}, headers map[string]string) error {
848 | client := &http.Client{}
849 | req, _ := http.NewRequest("GET", url, nil)
850 |
851 | for k, v := range headers {
852 | req.Header.Set(k, v)
853 | }
854 |
855 | r, err := client.Do(req)
856 | if err != nil {
857 | return err
858 | }
859 | defer r.Body.Close()
860 |
861 | return json.NewDecoder(r.Body).Decode(target)
862 | }
863 |
864 | func getInstagramVideoUrl(url string) string {
865 | resp, err := http.Get(url)
866 |
867 | if err != nil {
868 | return ""
869 | }
870 |
871 | defer resp.Body.Close()
872 | z := html.NewTokenizer(resp.Body)
873 |
874 | for {
875 | tt := z.Next()
876 | switch {
877 | case tt == html.ErrorToken:
878 | return ""
879 | }
880 | if tt == html.StartTagToken || tt == html.SelfClosingTagToken {
881 | t := z.Token()
882 | if t.Data == "meta" {
883 | for _, a := range t.Attr {
884 | if a.Key == "property" {
885 | if a.Val == "og:video" || a.Val == "og:video:secure_url" {
886 | for _, at := range t.Attr {
887 | if at.Key == "content" {
888 | return at.Val
889 | }
890 | }
891 | }
892 | }
893 | }
894 | }
895 | }
896 | }
897 | }
898 |
899 | func getInstagramAlbumUrls(url string) []string {
900 | var links []string
901 | resp, err := http.Get(url)
902 |
903 | if err != nil {
904 | return links
905 | }
906 |
907 | defer resp.Body.Close()
908 | z := html.NewTokenizer(resp.Body)
909 |
910 | ParseLoop:
911 | for {
912 | tt := z.Next()
913 | switch {
914 | case tt == html.ErrorToken:
915 | break ParseLoop
916 | }
917 | if tt == html.StartTagToken || tt == html.SelfClosingTagToken {
918 | t := z.Token()
919 | for _, a := range t.Attr {
920 | if a.Key == "type" {
921 | if a.Val == "text/javascript" {
922 | z.Next()
923 | content := string(z.Text())
924 | if strings.Contains(content, "window._sharedData = ") {
925 | content = strings.Replace(content, "window._sharedData = ", "", 1)
926 | content = content[:len(content)-1]
927 | jsonParsed, err := gabs.ParseJSON([]byte(content))
928 | if err != nil {
929 | fmt.Println("error parsing instagram json: ", err)
930 | continue ParseLoop
931 | }
932 | entryChildren, err := jsonParsed.Path("entry_data.PostPage").Children()
933 | if err != nil {
934 | fmt.Println("unable to find entries children: ", err)
935 | continue ParseLoop
936 | }
937 | for _, entryChild := range entryChildren {
938 | albumChildren, err := entryChild.Path("graphql.shortcode_media.edge_sidecar_to_children.edges").Children()
939 | if err != nil {
940 | continue ParseLoop
941 | }
942 | for _, albumChild := range albumChildren {
943 | link, ok := albumChild.Path("node.display_url").Data().(string)
944 | if ok {
945 | links = append(links, link)
946 | }
947 | }
948 | }
949 | }
950 | }
951 | }
952 | }
953 | }
954 | }
955 | if len(links) > 0 {
956 | fmt.Printf("[%s] Found instagram album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url)
957 | }
958 |
959 | return links
960 | }
961 |
962 | func filenameFromUrl(dUrl string) string {
963 | base := path.Base(dUrl)
964 | parts := strings.Split(base, "?")
965 | return parts[0]
966 | }
967 |
968 | func startDownload(dUrl string, filename string, path string, channelId string, userId string, fileTime time.Time) {
969 | success := false
970 | for i := 0; i < MaxDownloadRetries; i++ {
971 | success = downloadFromUrl(dUrl, filename, path, channelId, userId, fileTime)
972 | if success == true {
973 | break
974 | } else {
975 | time.Sleep(5 * time.Second)
976 | }
977 | }
978 | if success == false {
979 | fmt.Println("Gave up on downloading", dUrl)
980 | if SendNoticesToInteractiveChannels == true {
981 | for channelId := range InteractiveChannelWhitelist {
982 | content := fmt.Sprintf("Gave up on downloading %s, no success after %d retries", dUrl, MaxDownloadRetries)
983 | _, err := dg.ChannelMessageSend(channelId, content)
984 | if err != nil {
985 | fmt.Println("Failed to send notice to", channelId, "-", err)
986 | }
987 | }
988 | }
989 | }
990 | }
991 |
992 | func downloadFromUrl(dUrl string, filename string, path string, channelId string, userId string, fileTime time.Time) bool {
993 | err := os.MkdirAll(path, 0755)
994 | if err != nil {
995 | fmt.Println("Error while creating folder", path, "-", err)
996 | return false
997 | }
998 |
999 | timeout := time.Duration(time.Duration(DownloadTimeout) * time.Second)
1000 | client := &http.Client{
1001 | Timeout: timeout,
1002 | }
1003 | request, err := http.NewRequest("GET", dUrl, nil)
1004 | if err != nil {
1005 | fmt.Println("Error while downloading", dUrl, "-", err)
1006 | return false
1007 | }
1008 | request.Header.Add("Accept-Encoding", "identity")
1009 | response, err := client.Do(request)
1010 | if err != nil {
1011 | fmt.Println("Error while downloading", dUrl, "-", err)
1012 | return false
1013 | }
1014 | defer response.Body.Close()
1015 |
1016 | if filename == "" {
1017 | filename = filenameFromUrl(response.Request.URL.String())
1018 | for key, iHeader := range response.Header {
1019 | if key == "Content-Disposition" {
1020 | _, params, err := mime.ParseMediaType(iHeader[0])
1021 | if err == nil {
1022 | newFilename, err := url.QueryUnescape(params["filename"])
1023 | if err != nil {
1024 | newFilename = params["filename"]
1025 | }
1026 | if newFilename != "" {
1027 | filename = newFilename
1028 | }
1029 | }
1030 | }
1031 | }
1032 | }
1033 |
1034 | bodyOfResp, err := ioutil.ReadAll(response.Body)
1035 | if err != nil {
1036 | fmt.Println("Could not read response", dUrl, "-", err)
1037 | return false
1038 | }
1039 |
1040 | contentType := http.DetectContentType(bodyOfResp)
1041 |
1042 | // check for valid filename, if not, replace with generic filename
1043 | if !RegexpFilename.MatchString(filename) {
1044 | filename = time.Now().Format("2006-01-02 15-04-05")
1045 | possibleExtension, _ := mime.ExtensionsByType(contentType)
1046 | if len(possibleExtension) > 0 {
1047 | filename += possibleExtension[0]
1048 | }
1049 | }
1050 |
1051 | completePath := path + string(os.PathSeparator) + filename
1052 | if _, err := os.Stat(completePath); err == nil {
1053 | tmpPath := completePath
1054 | i := 1
1055 | for {
1056 | completePath = tmpPath[0:len(tmpPath)-len(filepathExtension(tmpPath))] +
1057 | "-" + strconv.Itoa(i) + filepathExtension(tmpPath)
1058 | if _, err := os.Stat(completePath); os.IsNotExist(err) {
1059 | break
1060 | }
1061 | i = i + 1
1062 | }
1063 | fmt.Printf("[%s] Saving possible duplicate (filenames match): %s to %s\n", time.Now().Format(time.Stamp), tmpPath, completePath)
1064 | }
1065 |
1066 | extension := filepath.Ext(filename)
1067 | contentTypeParts := strings.Split(contentType, "/")
1068 | if t := contentTypeParts[0]; t != "image" && t != "video" && t != "audio" &&
1069 | !(t == "application" && isAudioFile(filename)) &&
1070 | strings.ToLower(extension) != ".mov" &&
1071 | strings.ToLower(extension) != ".mp4" &&
1072 | strings.ToLower(extension) != ".webm" {
1073 | fmt.Println("No image, video, or audio found at", dUrl)
1074 | return true
1075 | }
1076 |
1077 | err = ioutil.WriteFile(completePath, bodyOfResp, 0644)
1078 | if err != nil {
1079 | fmt.Println("Error while writing to disk", dUrl, "-", err)
1080 | return false
1081 | }
1082 |
1083 | err = os.Chtimes(completePath, fileTime, fileTime)
1084 | if err != nil {
1085 | fmt.Println("Error while changing date", dUrl, "-", err)
1086 | }
1087 |
1088 | sourceChannelName := channelId
1089 | sourceGuildName := "N/A"
1090 | sourceChannel, _ := dg.State.Channel(channelId)
1091 | if sourceChannel != nil && sourceChannel.Name != "" {
1092 | sourceChannelName = sourceChannel.Name
1093 | sourceGuild, _ := dg.State.Guild(sourceChannel.GuildID)
1094 | if sourceGuild != nil && sourceGuild.Name != "" {
1095 | sourceGuildName = sourceGuild.Name
1096 | }
1097 | }
1098 |
1099 | fmt.Printf("[%s] Saved URL %s to %s from #%s/%s\n",
1100 | time.Now().Format(time.Stamp), dUrl, completePath, sourceChannelName, sourceGuildName)
1101 | err = insertDownloadedImage(&DownloadedImage{Url: dUrl, Time: time.Now(), Destination: completePath, ChannelId: channelId, UserId: userId})
1102 | if err != nil {
1103 | fmt.Println("Error while writing to database", err)
1104 | }
1105 |
1106 | updateDiscordStatus()
1107 | return true
1108 | }
1109 |
1110 | func isAudioFile(f string) bool {
1111 | switch strings.ToLower(path.Ext(f)) {
1112 | case ".mp3", ".wav", ".aif":
1113 | return true
1114 | default:
1115 | return false
1116 | }
1117 | }
1118 |
1119 | type DownloadedImage struct {
1120 | Url string
1121 | Time time.Time
1122 | Destination string
1123 | ChannelId string
1124 | UserId string
1125 | }
1126 |
1127 | func insertDownloadedImage(downloadedImage *DownloadedImage) error {
1128 | _, err := myDB.Use("Downloads").Insert(map[string]interface{}{
1129 | "Url": downloadedImage.Url,
1130 | "Time": downloadedImage.Time.String(),
1131 | "Destination": downloadedImage.Destination,
1132 | "ChannelId": downloadedImage.ChannelId,
1133 | "UserId": downloadedImage.UserId,
1134 | })
1135 | return err
1136 | }
1137 |
1138 | func findDownloadedImageById(id int) *DownloadedImage {
1139 | downloads := myDB.Use("Downloads")
1140 | //var query interface{}
1141 | //json.Unmarshal([]byte(fmt.Sprintf(`[{"eq": "%d", "in": ["Id"]}]`, id)), &query)
1142 | //queryResult := make(map[int]struct{})
1143 | //db.EvalQuery(query, myDB.Use("Downloads"), &queryResult)
1144 |
1145 | readBack, err := downloads.Read(id)
1146 | if err != nil {
1147 | fmt.Println(err)
1148 | }
1149 | timeT, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", readBack["Time"].(string))
1150 | return &DownloadedImage{
1151 | Url: readBack["Url"].(string),
1152 | Time: timeT,
1153 | Destination: readBack["Destination"].(string),
1154 | ChannelId: readBack["ChannelId"].(string),
1155 | UserId: readBack["UserId"].(string),
1156 | }
1157 | }
1158 |
1159 | func findDownloadedImageByUrl(url string) []*DownloadedImage {
1160 | var query interface{}
1161 | json.Unmarshal([]byte(fmt.Sprintf(`[{"eq": "%s", "in": ["Url"]}]`, url)), &query)
1162 | queryResult := make(map[int]struct{})
1163 | db.EvalQuery(query, myDB.Use("Downloads"), &queryResult)
1164 |
1165 | downloadedImages := make([]*DownloadedImage, 0)
1166 | for id := range queryResult {
1167 | downloadedImages = append(downloadedImages, findDownloadedImageById(id))
1168 | }
1169 | return downloadedImages
1170 | }
1171 |
1172 | func countDownloadedImages() int {
1173 | i := 0
1174 | myDB.Use("Downloads").ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) {
1175 | // fmt.Printf("%v\n", findDownloadedImageById(id))
1176 | i++
1177 | return true
1178 | })
1179 | return i
1180 | // fmt.Println(myDB.Use("Downloads").ApproxDocCount()) TODO?
1181 | }
1182 |
1183 | // http://stackoverflow.com/a/18695740/1443726
1184 | func sortStringIntMapByValue(m map[string]int) PairList {
1185 | pl := make(PairList, len(m))
1186 | i := 0
1187 | for k, v := range m {
1188 | pl[i] = Pair{k, v}
1189 | i++
1190 | }
1191 | sort.Sort(sort.Reverse(pl))
1192 | return pl
1193 | }
1194 |
1195 | type Pair struct {
1196 | Key string
1197 | Value int
1198 | }
1199 |
1200 | type PairList []Pair
1201 |
1202 | func (p PairList) Len() int { return len(p) }
1203 | func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }
1204 | func (p PairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
1205 |
--------------------------------------------------------------------------------
/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "reflect"
6 | "regexp"
7 | "strings"
8 | "testing"
9 |
10 | "gopkg.in/ini.v1"
11 | )
12 |
13 | func init() {
14 | RegexpUrlTwitter, _ = regexp.Compile(REGEXP_URL_TWITTER)
15 | RegexpUrlTistoryLegacy, _ = regexp.Compile(REGEXP_URL_TISTORY_LEGACY)
16 | RegexpUrlTistoryLegacyWithCDN, _ = regexp.Compile(REGEXP_URL_TISTORY_LEGACY_WITH_CDN)
17 | RegexpUrlGfycat, _ = regexp.Compile(REGEXP_URL_GFYCAT)
18 | RegexpUrlInstagram, _ = regexp.Compile(REGEXP_URL_INSTAGRAM)
19 | RegexpUrlImgurSingle, _ = regexp.Compile(REGEXP_URL_IMGUR_SINGLE)
20 | RegexpUrlImgurAlbum, _ = regexp.Compile(REGEXP_URL_IMGUR_ALBUM)
21 | RegexpUrlGoogleDrive, _ = regexp.Compile(REGEXP_URL_GOOGLEDRIVE)
22 | RegexpUrlPossibleTistorySite, _ = regexp.Compile(REGEXP_URL_POSSIBLE_TISTORY_SITE)
23 | RegexpUrlFlickrPhoto, _ = regexp.Compile(REGEXP_URL_FLICKR_PHOTO)
24 | RegexpUrlFlickrAlbum, _ = regexp.Compile(REGEXP_URL_FLICKR_ALBUM)
25 | RegexpUrlStreamable, _ = regexp.Compile(REGEXP_URL_STREAMABLE)
26 | flickrApiKey = os.Getenv("FLICKR_API_KEY")
27 |
28 | var err error
29 | cfg, err := ini.Load("config.ini")
30 | if err == nil {
31 | flickrApiKey = cfg.Section("flickr").Key("api key").MustString("yourflickrapikey")
32 | }
33 | }
34 |
35 | type urlsTestpair struct {
36 | value string
37 | result map[string]string
38 | }
39 |
40 | var getTwitterUrlsTests = []urlsTestpair{
41 | {
42 | "https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig",
43 | map[string]string{"https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig": "CulDBM6VYAA-YhY.jpg"},
44 | },
45 | {
46 | "https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg",
47 | map[string]string{"https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig": "CulDBM6VYAA-YhY.jpg"},
48 | },
49 | {
50 | "http://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg",
51 | map[string]string{"https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig": "CulDBM6VYAA-YhY.jpg"},
52 | },
53 | }
54 |
55 | func TestGetTwitterUrls(t *testing.T) {
56 | for _, pair := range getTwitterUrlsTests {
57 | v, err := getTwitterUrls(pair.value)
58 | if err != nil {
59 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
60 | }
61 | if !reflect.DeepEqual(v, pair.result) {
62 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
63 | }
64 | }
65 | }
66 |
67 | var getTistoryUrlsTests = []urlsTestpair{
68 | {
69 | "http://cfile25.uf.tistory.com/original/235CA739582E86992EFC4E",
70 | map[string]string{"http://cfile25.uf.tistory.com/original/235CA739582E86992EFC4E": ""},
71 | },
72 | {
73 | "http://cfile25.uf.tistory.com/image/235CA739582E86992EFC4E",
74 | map[string]string{"http://cfile25.uf.tistory.com/original/235CA739582E86992EFC4E": ""},
75 | },
76 | }
77 |
78 | func TestGetTistoryUrls(t *testing.T) {
79 | for _, pair := range getTistoryUrlsTests {
80 | v, err := getLegacyTistoryUrls(pair.value)
81 | if err != nil {
82 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
83 | }
84 | if !reflect.DeepEqual(v, pair.result) {
85 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
86 | }
87 | }
88 | }
89 |
90 | var getGfycatUrlsTests = []urlsTestpair{
91 | {
92 | "https://gfycat.com/SandyChiefBoubou",
93 | map[string]string{"https://fat.gfycat.com/SandyChiefBoubou.mp4": ""},
94 | },
95 | }
96 |
97 | func TestGetGfycatUrls(t *testing.T) {
98 | for _, pair := range getGfycatUrlsTests {
99 | v, err := getGfycatUrls(pair.value)
100 | if err != nil {
101 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
102 | }
103 | if !reflect.DeepEqual(v, pair.result) {
104 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
105 | }
106 | }
107 | }
108 |
109 | var getInstagramUrlsPictureTests = []urlsTestpair{
110 | {
111 | "https://www.instagram.com/p/BHhDAmhAz33/?taken-by=s_sohye",
112 | map[string]string{"https://www.instagram.com/p/BHhDAmhAz33/media/?size=l&taken-by=s_sohye": "instagram s_sohye - BHhDAmhAz33.jpg"},
113 | },
114 | {
115 | "https://www.instagram.com/p/BHhDAmhAz33/",
116 | map[string]string{"https://www.instagram.com/p/BHhDAmhAz33/media/?size=l": "instagram s_sohye - BHhDAmhAz33.jpg"},
117 | },
118 | }
119 |
120 | var getInstagramUrlsVideoTests = []urlsTestpair{
121 | {
122 | "https://www.instagram.com/p/BL2_ZIHgYTp/?taken-by=s_sohye",
123 | map[string]string{"14811404_233311497085396_338650092456116224_n.mp4": "instagram s_sohye - BL2_ZIHgYTp.mp4"},
124 | },
125 | {
126 | "https://www.instagram.com/p/BL2_ZIHgYTp/",
127 | map[string]string{"14811404_233311497085396_338650092456116224_n.mp4": "instagram s_sohye - BL2_ZIHgYTp.mp4"},
128 | },
129 | }
130 |
131 | var getInstagramUrlsAlbumTests = []urlsTestpair{
132 | {
133 | "https://www.instagram.com/p/BRiCc0VjULk/?taken-by=gfriendofficial",
134 | map[string]string{
135 | "17265460_395888184109957_3500310922180689920_n.jpg": "instagram gfriendofficial - BRiCc0VjULk",
136 | "17265456_267171360360765_8110946520456495104_n.jpg": "instagram gfriendofficial - BRiCc0VjULk",
137 | "17265327_1394797493912862_2677004307588448256_n.jpg": "instagram gfriendofficial - BRiCc0VjULk"},
138 | },
139 | {
140 | "https://www.instagram.com/p/BRhheSPjaQ3/",
141 | map[string]string{
142 | "17125875_306909746390523_8184965703367917568_n.jpg": "instagram gfriendofficial - BRhheSPjaQ3",
143 | "17266053_188727064951899_2485556569865977856_n.jpg": "instagram gfriendofficial - BRhheSPjaQ3"},
144 | },
145 | }
146 |
147 | func TestGetInstagramUrls(t *testing.T) {
148 | for _, pair := range getInstagramUrlsPictureTests {
149 | v, err := getInstagramUrls(pair.value)
150 | if err != nil {
151 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
152 | }
153 | if !reflect.DeepEqual(v, pair.result) {
154 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
155 | }
156 | }
157 | for _, pair := range getInstagramUrlsVideoTests {
158 | v, err := getInstagramUrls(pair.value)
159 | if err != nil {
160 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
161 | }
162 | for keyResult, valueResult := range pair.result {
163 | for keyExpected, valueExpected := range v {
164 | if strings.Contains(keyResult, keyExpected) || valueResult != valueExpected { // CDN location can vary
165 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
166 | }
167 | }
168 | }
169 | }
170 | for _, pair := range getInstagramUrlsAlbumTests {
171 | v, err := getInstagramUrls(pair.value)
172 | if err != nil {
173 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
174 | }
175 | for keyResult, valueResult := range pair.result {
176 | for keyExpected, valueExpected := range v {
177 | if strings.Contains(keyResult, keyExpected) || strings.Contains(valueResult, valueExpected) { // CDN location can vary
178 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
179 | }
180 | }
181 | }
182 | }
183 | }
184 |
185 | var getImgurSingleUrlsTests = []urlsTestpair{
186 | {
187 | "http://imgur.com/viZictl",
188 | map[string]string{"http://imgur.com/download/viZictl": ""},
189 | },
190 | {
191 | "https://imgur.com/viZictl",
192 | map[string]string{"https://imgur.com/download/viZictl": ""},
193 | },
194 | {
195 | "https://i.imgur.com/viZictl.jpg",
196 | map[string]string{"https://i.imgur.com/download/viZictl.jpg": ""},
197 | },
198 | {
199 | "http://imgur.com/uYwt2VV",
200 | map[string]string{"http://imgur.com/download/uYwt2VV": ""},
201 | },
202 | {
203 | "http://i.imgur.com/uYwt2VV.gifv",
204 | map[string]string{"http://i.imgur.com/download/uYwt2VV": ""},
205 | },
206 | }
207 |
208 | func TestGetImgurSingleUrls(t *testing.T) {
209 | for _, pair := range getImgurSingleUrlsTests {
210 | v, err := getImgurSingleUrls(pair.value)
211 | if err != nil {
212 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
213 | }
214 | if !reflect.DeepEqual(v, pair.result) {
215 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
216 | }
217 | }
218 | }
219 |
220 | var getImgurAlbumUrlsTests = []urlsTestpair{
221 | {
222 | "http://imgur.com/a/ALTpi",
223 | map[string]string{
224 | "https://i.imgur.com/FKoguPh.jpg": "",
225 | "https://i.imgur.com/5FNL6Pe.jpg": "",
226 | "https://i.imgur.com/YA0V0g9.jpg": "",
227 | "https://i.imgur.com/Uc2iDhD.jpg": "",
228 | "https://i.imgur.com/J9JRSSJ.jpg": "",
229 | "https://i.imgur.com/Xrx0uyE.jpg": "",
230 | "https://i.imgur.com/3xDSq1O.jpg": "",
231 | },
232 | },
233 | }
234 |
235 | func TestGetImgurAlbumUrls(t *testing.T) {
236 | for _, pair := range getImgurAlbumUrlsTests {
237 | v, err := getImgurAlbumUrls(pair.value)
238 | if err != nil {
239 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
240 | }
241 | for expectedLink, expectedName := range pair.result {
242 | linkFound := false
243 | for gotLink, gotName := range v {
244 | if expectedLink == gotLink && expectedName == gotName {
245 | linkFound = true
246 | }
247 | }
248 | if !linkFound {
249 | t.Errorf("For expected %s %s, got %s", expectedLink, expectedName, v)
250 | }
251 | }
252 | }
253 | }
254 |
255 | var getGoogleDriveUrlsTests = []urlsTestpair{
256 | {
257 | "https://drive.google.com/file/d/0B8TnwsJqlFllSUtvUEhoSU40WkE/view",
258 | map[string]string{"https://drive.google.com/uc?export=download&id=0B8TnwsJqlFllSUtvUEhoSU40WkE": ""},
259 | },
260 | }
261 |
262 | func TestGetGoogleDriveUrls(t *testing.T) {
263 | for _, pair := range getGoogleDriveUrlsTests {
264 | v, err := getGoogleDriveUrls(pair.value)
265 | if err != nil {
266 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
267 | }
268 | if !reflect.DeepEqual(v, pair.result) {
269 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
270 | }
271 | }
272 | }
273 |
274 | var getTistoryWithCDNUrlsTests = []urlsTestpair{
275 | {
276 | "http://img1.daumcdn.net/thumb/R720x0.q80/?scode=mtistory&fname=http%3A%2F%2Fcfile24.uf.tistory.com%2Fimage%2F2658554B580BDC4C0924CA",
277 | map[string]string{"http://cfile24.uf.tistory.com/original/2658554B580BDC4C0924CA": ""},
278 | },
279 | }
280 |
281 | func TestGetTistoryWithCDNUrls(t *testing.T) {
282 | for _, pair := range getTistoryWithCDNUrlsTests {
283 | v, err := getTistoryWithCDNUrls(pair.value)
284 | if err != nil {
285 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
286 | }
287 | if !reflect.DeepEqual(v, pair.result) {
288 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
289 | }
290 | }
291 | }
292 |
293 | var getPossibleTistorySiteUrlsTests = []urlsTestpair{
294 | {
295 | "http://soonduck.tistory.com/482",
296 | map[string]string{
297 | "a": "",
298 | "b": "",
299 | "c": "",
300 | "d": "",
301 | "e": "",
302 | },
303 | },
304 | {
305 | "http://soonduck.tistory.com/m/482",
306 | map[string]string{
307 | "a": "",
308 | "b": "",
309 | "c": "",
310 | "d": "",
311 | "e": "",
312 | },
313 | },
314 | {
315 | "http://slmn.de/123",
316 | map[string]string{},
317 | },
318 | }
319 |
320 | func TestGetPossibleTistorySiteUrls(t *testing.T) {
321 | for _, pair := range getPossibleTistorySiteUrlsTests {
322 | v, err := getPossibleTistorySiteUrls(pair.value)
323 | if err != nil {
324 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
325 | }
326 | if len(pair.result) > len(v) { // only check that received amount of posts >= expected amount of posts
327 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
328 | }
329 | }
330 | }
331 |
332 | var getFlickrUrlFromPhotoIdTests = []map[string]string{
333 | {
334 | "value": "31065043320",
335 | "result": "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg",
336 | },
337 | }
338 |
339 | func TestGetFlickrUrlFromPhotoId(t *testing.T) {
340 | for _, pair := range getFlickrUrlFromPhotoIdTests {
341 | v := getFlickrUrlFromPhotoId(pair["value"])
342 | if v != pair["result"] {
343 | t.Errorf("For %s, expected %s, got %s", pair["value"], pair["result"], v)
344 | }
345 | }
346 | }
347 |
348 | var getFlickrPhotoUrlsTests = []urlsTestpair{
349 | {
350 | "https://www.flickr.com/photos/137385017@N08/31065043320/in/album-72157677350305446/",
351 | map[string]string{
352 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
353 | },
354 | },
355 | {
356 | "https://www.flickr.com/photos/137385017@N08/31065043320/in/album-72157677350305446",
357 | map[string]string{
358 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
359 | },
360 | },
361 | {
362 | "https://www.flickr.com/photos/137385017@N08/31065043320/",
363 | map[string]string{
364 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
365 | },
366 | },
367 | {
368 | "https://www.flickr.com/photos/137385017@N08/31065043320",
369 | map[string]string{
370 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
371 | },
372 | },
373 | }
374 |
375 | func TestGetFlickrPhotoUrls(t *testing.T) {
376 | for _, pair := range getFlickrPhotoUrlsTests {
377 | v, err := getFlickrPhotoUrls(pair.value)
378 | if err != nil {
379 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
380 | }
381 | if !reflect.DeepEqual(v, pair.result) {
382 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
383 | }
384 | }
385 | }
386 |
387 | var getFlickrAlbumUrlsTests = []urlsTestpair{
388 | {
389 | "https://www.flickr.com/photos/137385017@N08/albums/72157677350305446/",
390 | map[string]string{
391 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
392 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "",
393 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "",
394 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "",
395 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "",
396 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "",
397 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "",
398 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "",
399 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "",
400 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "",
401 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "",
402 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "",
403 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "",
404 | },
405 | },
406 | {
407 | "https://www.flickr.com/photos/137385017@N08/albums/72157677350305446",
408 | map[string]string{
409 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
410 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "",
411 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "",
412 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "",
413 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "",
414 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "",
415 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "",
416 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "",
417 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "",
418 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "",
419 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "",
420 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "",
421 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "",
422 | },
423 | },
424 | {
425 | "https://www.flickr.com/photos/137385017@N08/albums/with/72157677350305446/",
426 | map[string]string{
427 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
428 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "",
429 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "",
430 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "",
431 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "",
432 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "",
433 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "",
434 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "",
435 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "",
436 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "",
437 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "",
438 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "",
439 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "",
440 | },
441 | },
442 | {
443 | "https://www.flickr.com/photos/137385017@N08/albums/with/72157677350305446",
444 | map[string]string{
445 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "",
446 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "",
447 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "",
448 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "",
449 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "",
450 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "",
451 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "",
452 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "",
453 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "",
454 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "",
455 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "",
456 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "",
457 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "",
458 | },
459 | },
460 | }
461 |
462 | func TestGetFlickrAlbumUrls(t *testing.T) {
463 | for _, pair := range getFlickrAlbumUrlsTests {
464 | v, err := getFlickrAlbumUrls(pair.value)
465 | if err != nil {
466 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
467 | }
468 | if !reflect.DeepEqual(v, pair.result) {
469 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v)
470 | }
471 | }
472 | }
473 |
474 | var getStreamableUrlsTests = []urlsTestpair{
475 | {
476 | "http://streamable.com/41ajc",
477 | map[string]string{
478 | "streamable.com/video/mp4/41ajc.mp4": "",
479 | },
480 | },
481 | }
482 |
483 | func TestGetStreamableUrls(t *testing.T) {
484 | for _, pair := range getStreamableUrlsTests {
485 | v, err := getStreamableUrls(pair.value)
486 | if err != nil {
487 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err)
488 | }
489 |
490 | for expectedLink, expectedName := range pair.result {
491 | linkFound := false
492 | for gotLink := range v {
493 | if strings.Contains(gotLink, expectedLink) {
494 | linkFound = true
495 | }
496 | }
497 | if !linkFound {
498 | t.Errorf("For expected %s %s, got %s", expectedLink, expectedName, v)
499 | }
500 | }
501 | }
502 | }
503 |
--------------------------------------------------------------------------------
/models.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import "time"
4 |
5 | type DownloadItem struct {
6 | Link string
7 | Filename string
8 | Time time.Time
9 | }
10 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # PROJECT ABANDONED, [PLEASE USE FORK.](https://github.com/get-got/discord-downloader-go/)
2 |
3 | ---
4 |
5 | # discord-image-downloader-go
6 | [
](https://www.paypal.me/swk)
7 | [](https://goreportcard.com/report/github.com/Seklfreak/discord-image-downloader-go)
8 | [](https://travis-ci.com/Seklfreak/discord-image-downloader-go)
9 |
10 | [**DOWNLOAD THE LATEST RELEASE BUILD**](https://github.com/Seklfreak/discord-image-downloader-go/releases/latest)
11 |
12 | This project is not often maintained. For an actively maintained fork that implements features such as extensive JSON settings with channel-specific configurations, see [**get-got/discord-downloader-go**](https://github.com/get-got/discord-downloader-go)
13 |
14 | ## Discord SelfBots are forbidden!
15 | [Official Statement](https://support.discordapp.com/hc/en-us/articles/115002192352-Automated-user-accounts-self-bots-)
16 | ### You have been warned.
17 |
18 | This is a simple tool which downloads media posted in Discord channels of your choice to a local folder. It handles various sources like Twitter differently to make sure to download the best quality available.
19 |
20 | ## Websites currently supported
21 | - Discord Attachments
22 | - Twitter
23 | - Tistory
24 | - Gfycat
25 | - Instagram
26 | - Imgur
27 | - Google Drive Files and Folders
28 | - Flickr
29 | - Streamable
30 | - Any direct link to an image or video
31 |
32 | ## How to use?
33 | When you run the tool for the first time it creates a `config.ini` file with example values. Edit these values and run the tool for a second time. It should connect to discords api and wait for new messages.
34 |
35 | If you are using a normal user account **without two-factor authentication (2FA)**, simply enter your email and password into the corresponding lines in `config.ini`, under the auth section.
36 |
37 | If you are using **two-factor authentication (2FA) you have to login using your token.** Remove the email and password lines under the auth section in the config file and instead put in `token = `. You can acquire your token from the developer tools in your browser (`localStorage.token`) or discord client (`Ctrl+Shift+I` (Windows) or `Cmd+Option+I` (Mac), click Application, click Local Storage, click `https://discordapp.com`, and find "token" and paste the value).
38 |
39 | If you wish to use a **bot account (not a user account)**, go to https://discord.com/developers/applications and create an application, then create a bot in the `Bot` tab in application settings. The bot tab will show you your token. You can invite to your server(s) by going to the `OAuth2` tab in application settings, check `bot`, and copy+paste the url into your browser. **In the `config.ini`, add "Bot " before your token. (example: `token = Bot mytokenhere`)**
40 |
41 | ## How to download old files?
42 | By default, the tool only downloads new links posted while the tool is running. You can also set up the tool to download the complete history of a channel. To do this you have to run this tool with a separate discord account. Send your second account a dm on your primary account and get the channel id from the direct message channel. Now add this channel id to the config by adding the following lines:
43 | ```
44 | [interactive channels]
45 | =
46 | ```
47 | After this is done restart the tool and send `history` as a DM to your second account. The bot will ask for the channel id of the channel you want to download and start the downloads. You can view all available commands by sending `help`.
48 |
49 | ### Where do I get the Channel ID?
50 | Enable Developer Mode (in Discord Appearance settings) and right click the channel you need, and click Copy ID.
51 |
52 | **OR,** Open discord in your browser and go to the channel you want to monitor. In your address bar should be a URL like `https://discordapp.com/channels/1234/5678`. The number after the last slash is the channel ID, in this case, `5678`.
53 |
54 | ### Where do I get the Channel ID for Direct Messages?
55 | 1. Inspect Element in the Discord client (`Ctrl+Shift+I` for Windows or `Cmd+Option+I` for Mac)
56 | 1. Go to the `Elements` tab on the left.
57 | 1. Click this icon  (the arrow going into a box) and then click on the avatar for the persons DMs you want to grab the ID for.
58 | 1. Somewhere slightly above the HTML it takes you to, there should be a line that looks like this 
59 | 1. Copy the number after the `/@me/`. That is your Channel ID to use.
60 |
61 | **OR,** Open discord in your browser and go to the channel you want to monitor. In your address bar should be a URL like `https://discordapp.com/channels/@me/5678`. The number after the last slash is the channel ID, in this case, `5678`.
62 |
--------------------------------------------------------------------------------
/regex.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "regexp"
5 | )
6 |
7 | const (
8 | REGEXP_URL_TWITTER = `^http(s?):\/\/pbs(-[0-9]+)?\.twimg\.com\/media\/[^\./]+\.(jpg|png)((\:[a-z]+)?)$`
9 | REGEXP_URL_TWITTER_PARAMS = `^http(s?):\/\/pbs(-[0-9]+)?\.twimg\.com\/media\/([^\./]+)\?format=(jpg|png)&name=([a-z]+)[a-z=&]*$`
10 | REGEXP_URL_TWITTER_STATUS = `^http(s?):\/\/(www\.)?twitter\.com\/([A-Za-z0-9-_\.]+\/status\/|statuses\/|i\/web\/status\/)([0-9]+)$`
11 | REGEXP_URL_TISTORY = `^http(s?):\/\/t[0-9]+\.daumcdn\.net\/cfile\/tistory\/([A-Z0-9]+?)(\?original)?$`
12 | REGEXP_URL_TISTORY_LEGACY = `^http(s?):\/\/[a-z0-9]+\.uf\.tistory\.com\/(image|original)\/[A-Z0-9]+$`
13 | REGEXP_URL_TISTORY_LEGACY_WITH_CDN = `^http(s)?:\/\/[0-9a-z]+.daumcdn.net\/[a-z]+\/[a-zA-Z0-9\.]+\/\?scode=mtistory&fname=http(s?)%3A%2F%2F[a-z0-9]+\.uf\.tistory\.com%2F(image|original)%2F[A-Z0-9]+$`
14 | REGEXP_URL_GFYCAT = `^http(s?):\/\/gfycat\.com\/(gifs\/detail\/)?[A-Za-z]+$`
15 | REGEXP_URL_INSTAGRAM = `^http(s?):\/\/(www\.)?instagram\.com\/p\/[^/]+\/(\?[^/]+)?$`
16 | REGEXP_URL_IMGUR_SINGLE = `^http(s?):\/\/(i\.)?imgur\.com\/[A-Za-z0-9]+(\.gifv)?$`
17 | REGEXP_URL_IMGUR_ALBUM = `^http(s?):\/\/imgur\.com\/(a\/|gallery\/|r\/[^\/]+\/)[A-Za-z0-9]+(#[A-Za-z0-9]+)?$`
18 | REGEXP_URL_GOOGLEDRIVE = `^http(s?):\/\/drive\.google\.com\/file\/d\/[^/]+\/view$`
19 | REGEXP_URL_GOOGLEDRIVE_FOLDER = `^http(s?):\/\/drive\.google\.com\/(drive\/folders\/|open\?id=)([^/]+)$`
20 | REGEXP_URL_POSSIBLE_TISTORY_SITE = `^http(s)?:\/\/[0-9a-zA-Z\.-]+\/(m\/)?(photo\/)?[0-9]+$`
21 | REGEXP_URL_FLICKR_PHOTO = `^http(s)?:\/\/(www\.)?flickr\.com\/photos\/([0-9]+)@([A-Z0-9]+)\/([0-9]+)(\/)?(\/in\/album-([0-9]+)(\/)?)?$`
22 | REGEXP_URL_FLICKR_ALBUM = `^http(s)?:\/\/(www\.)?flickr\.com\/photos\/(([0-9]+)@([A-Z0-9]+)|[A-Za-z0-9]+)\/(albums\/(with\/)?|(sets\/)?)([0-9]+)(\/)?$`
23 | REGEXP_URL_FLICKR_ALBUM_SHORT = `^http(s)?:\/\/((www\.)?flickr\.com\/gp\/[0-9]+@[A-Z0-9]+\/[A-Za-z0-9]+|flic\.kr\/s\/[a-zA-Z0-9]+)$`
24 | REGEXP_URL_STREAMABLE = `^http(s?):\/\/(www\.)?streamable\.com\/([0-9a-z]+)$`
25 |
26 | REGEXP_FILENAME = `^^[^/\\:*?"<>|]{1,150}\.[A-Za-z0-9]{2,4}$$`
27 | )
28 |
29 | var (
30 | RegexpUrlTwitter *regexp.Regexp
31 | RegexpUrlTwitterParams *regexp.Regexp
32 | RegexpUrlTwitterStatus *regexp.Regexp
33 | RegexpUrlTistory *regexp.Regexp
34 | RegexpUrlTistoryLegacy *regexp.Regexp
35 | RegexpUrlTistoryLegacyWithCDN *regexp.Regexp
36 | RegexpUrlGfycat *regexp.Regexp
37 | RegexpUrlInstagram *regexp.Regexp
38 | RegexpUrlImgurSingle *regexp.Regexp
39 | RegexpUrlImgurAlbum *regexp.Regexp
40 | RegexpUrlGoogleDrive *regexp.Regexp
41 | RegexpUrlGoogleDriveFolder *regexp.Regexp
42 | RegexpUrlPossibleTistorySite *regexp.Regexp
43 | RegexpUrlFlickrPhoto *regexp.Regexp
44 | RegexpUrlFlickrAlbum *regexp.Regexp
45 | RegexpUrlFlickrAlbumShort *regexp.Regexp
46 | RegexpUrlStreamable *regexp.Regexp
47 | )
48 |
49 | func initRegex() error {
50 | var err error
51 | RegexpUrlTwitter, err = regexp.Compile(REGEXP_URL_TWITTER)
52 | if err != nil {
53 | return err
54 | }
55 | RegexpUrlTwitterParams, err = regexp.Compile(REGEXP_URL_TWITTER_PARAMS)
56 | if err != nil {
57 | return err
58 | }
59 | RegexpUrlTwitterStatus, err = regexp.Compile(REGEXP_URL_TWITTER_STATUS)
60 | if err != nil {
61 | return err
62 | }
63 | RegexpUrlTistory, err = regexp.Compile(REGEXP_URL_TISTORY)
64 | if err != nil {
65 | return err
66 | }
67 | RegexpUrlTistoryLegacy, err = regexp.Compile(REGEXP_URL_TISTORY_LEGACY)
68 | if err != nil {
69 | return err
70 | }
71 | RegexpUrlTistoryLegacyWithCDN, err = regexp.Compile(REGEXP_URL_TISTORY_LEGACY_WITH_CDN)
72 | if err != nil {
73 | return err
74 | }
75 | RegexpUrlGfycat, err = regexp.Compile(REGEXP_URL_GFYCAT)
76 | if err != nil {
77 | return err
78 | }
79 | RegexpUrlInstagram, err = regexp.Compile(REGEXP_URL_INSTAGRAM)
80 | if err != nil {
81 | return err
82 | }
83 | RegexpUrlImgurSingle, err = regexp.Compile(REGEXP_URL_IMGUR_SINGLE)
84 | if err != nil {
85 | return err
86 | }
87 | RegexpUrlImgurAlbum, err = regexp.Compile(REGEXP_URL_IMGUR_ALBUM)
88 | if err != nil {
89 | return err
90 | }
91 | RegexpUrlGoogleDrive, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE)
92 | if err != nil {
93 | return err
94 | }
95 | RegexpUrlGoogleDriveFolder, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE_FOLDER)
96 | if err != nil {
97 | return err
98 | }
99 | RegexpUrlPossibleTistorySite, err = regexp.Compile(REGEXP_URL_POSSIBLE_TISTORY_SITE)
100 | if err != nil {
101 | return err
102 | }
103 | RegexpUrlFlickrPhoto, err = regexp.Compile(REGEXP_URL_FLICKR_PHOTO)
104 | if err != nil {
105 | return err
106 | }
107 | RegexpUrlFlickrAlbum, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM)
108 | if err != nil {
109 | return err
110 | }
111 | RegexpUrlStreamable, err = regexp.Compile(REGEXP_URL_STREAMABLE)
112 | if err != nil {
113 | return err
114 | }
115 | RegexpUrlFlickrAlbumShort, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM_SHORT)
116 | if err != nil {
117 | return err
118 | }
119 | RegexpFilename, err = regexp.Compile(REGEXP_FILENAME)
120 | if err != nil {
121 | return err
122 | }
123 |
124 | return nil
125 | }
126 |
--------------------------------------------------------------------------------
/tistory.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "net/url"
5 | "strings"
6 | )
7 |
8 | // getTistoryUrls downloads tistory URLs
9 | // http://t1.daumcdn.net/cfile/tistory/[…] => http://t1.daumcdn.net/cfile/tistory/[…]
10 | // http://t1.daumcdn.net/cfile/tistory/[…]?original => as is
11 | func getTistoryUrls(link string) (map[string]string, error) {
12 | if !strings.HasSuffix(link, "?original") {
13 | link += "?original"
14 | }
15 | return map[string]string{link: ""}, nil
16 | }
17 |
18 | func getLegacyTistoryUrls(link string) (map[string]string, error) {
19 | link = strings.Replace(link, "/image/", "/original/", -1)
20 | return map[string]string{link: ""}, nil
21 | }
22 |
23 | func getTistoryWithCDNUrls(urlI string) (map[string]string, error) {
24 | parameters, _ := url.ParseQuery(urlI)
25 | if val, ok := parameters["fname"]; ok {
26 | if len(val) > 0 {
27 | if RegexpUrlTistoryLegacy.MatchString(val[0]) {
28 | return getLegacyTistoryUrls(val[0])
29 | }
30 | }
31 | }
32 | return nil, nil
33 | }
34 |
--------------------------------------------------------------------------------
/vars.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | const (
4 | VERSION = "1.38"
5 | DATABASE_DIR = "database"
6 | RELEASE_URL = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest"
7 | RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest"
8 | IMGUR_CLIENT_ID = "a39473314df3f59"
9 |
10 | DEFAULT_CONFIG_FILE = "config.ini"
11 |
12 | discordEmojiBaseUrl = "https://cdn.discordapp.com/emojis/"
13 | )
14 |
--------------------------------------------------------------------------------