├── .gitignore ├── .travis.yml ├── LICENSE ├── common.go ├── extract.go ├── go.mod ├── go.sum ├── handler.go ├── main.go ├── main_test.go ├── models.go ├── readme.md ├── regex.go ├── tistory.go └── vars.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | # Output of the go coverage tool, specifically when used with LiteIDE 27 | *.out 28 | 29 | config.ini 30 | crossbuild.bat 31 | 32 | .idea/ 33 | 34 | vendor 35 | credentials.json 36 | .DS_Store 37 | 38 | # Exclude subdirectories (things like database, saved images during testing) 39 | /** 40 | # Exclude build command shortcut(s) 41 | *.cmd -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - "1.15" 4 | install: 5 | - env GO111MODULE=on go mod download 6 | - go get github.com/mitchellh/gox 7 | script: 8 | - env GO111MODULE=on gox -output="bin/discord-image-downloader-go-{{.OS}}-{{.Arch}}" -osarch="windows/amd64 linux/amd64 linux/arm" 9 | deploy: 10 | provider: releases 11 | api_key: 12 | secure: BZnS+tH7kCy2Jpzl5ctIIlQqurIlLvJVw5Pn2tkjEnQlnEQZUW7dEI/eejz1Tw1NNjuxdfJRzgkTNI7sRM6VjtuoTzq772TuxncTv4Oy/t/S3zaBthMMl/8Ug+VW2Py/A6LO8YDVpEaIIenyUTsmPG99HJd7jjW+qCh3npfjvWvf4Qe3FL0cNOH6l9ptA77Fcu1j7yXOUqCpxVtLL8sRe9qTsciUGT8eSAaHLBKbUxcw4TMn4P7G1/6b4tfkm1aQZX5cJROccdYt6FY5z1hNUUUN3e1bn1Hj/iY2zUZm1F4LHC5AcfZFUu/fAGhoLbB1B/hEZ4zrE7RLKaCR1vWCbr54/Xn+uWdDWPL6dbJ3EzCIuDtGDoHAJBfRos9HsCdfizB5A/yg0OS3RscK7Umh+gEznJyQCvAIbndAaryxKeOxVPd6tIdoW0fi0yaWmq9UHKVUt7lLd8rxlGWhGh6/GPLCME7+EAR84RyvP7e6VUHxUSnB3bly3oeJZd/QZpHmrIPqE6K9iobDJyERjNd+Bvuay8bw65NcqR9ZtAYlK7tUYxyEhw23fyybrwTB7ONUu/DDUSLQrtP7YvhtGnxMS+cU4O5GZrkuIP7urUS5KAnYbzb5I3F1Bk8PDLNFEdsGUOKq9U5ywRypmVfSxldGQHvEl4x1CO3E3zJEhkDOIaI= 13 | file: 14 | - "bin/discord-image-downloader-go-windows-amd64.exe" 15 | - "bin/discord-image-downloader-go-linux-amd64" 16 | - "bin/discord-image-downloader-go-linux-arm" 17 | skip_cleanup: true 18 | on: 19 | repo: Seklfreak/discord-image-downloader-go 20 | tags: true 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Sebastian Winkler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strings" 7 | 8 | "github.com/bwmarrin/discordgo" 9 | ) 10 | 11 | // isDiscordEmoji matches https://cdn.discordapp.com/emojis/503141595860959243.gif, and similar URLs/filenames 12 | func isDiscordEmoji(link string) bool { 13 | // always match discord emoji URLs, eg https://cdn.discordapp.com/emojis/340989430460317707.png 14 | if strings.HasPrefix(link, discordEmojiBaseUrl) { 15 | return true 16 | } 17 | 18 | return false 19 | } 20 | 21 | // deduplicateDownloadItems removes duplicates from a slice of *DownloadItem s identified by the Link 22 | func deduplicateDownloadItems(DownloadItems []*DownloadItem) []*DownloadItem { 23 | var result []*DownloadItem 24 | seen := map[string]bool{} 25 | 26 | for _, item := range DownloadItems { 27 | if seen[item.Link] { 28 | continue 29 | } 30 | 31 | seen[item.Link] = true 32 | result = append(result, item) 33 | } 34 | 35 | return result 36 | } 37 | 38 | func updateDiscordStatus() { 39 | if StatusEnabled { 40 | dg.UpdateStatusComplex(discordgo.UpdateStatusData{ 41 | Game: &discordgo.Game{ 42 | Name: fmt.Sprintf("%d %s", countDownloadedImages(), StatusSuffix), 43 | Type: StatusLabel, 44 | }, 45 | Status: StatusType, 46 | }) 47 | } else if StatusType != "online" { 48 | dg.UpdateStatusComplex(discordgo.UpdateStatusData{ 49 | Status: StatusType, 50 | }) 51 | } 52 | } 53 | 54 | func Pagify(text string, delimiter string) []string { 55 | result := make([]string, 0) 56 | textParts := strings.Split(text, delimiter) 57 | currentOutputPart := "" 58 | for _, textPart := range textParts { 59 | if len(currentOutputPart)+len(textPart)+len(delimiter) <= 1992 { 60 | if len(currentOutputPart) > 0 || len(result) > 0 { 61 | currentOutputPart += delimiter + textPart 62 | } else { 63 | currentOutputPart += textPart 64 | } 65 | } else { 66 | result = append(result, currentOutputPart) 67 | currentOutputPart = "" 68 | if len(textPart) <= 1992 { 69 | currentOutputPart = textPart 70 | } 71 | } 72 | } 73 | if currentOutputPart != "" { 74 | result = append(result, currentOutputPart) 75 | } 76 | return result 77 | } 78 | 79 | func filepathExtension(filepath string) string { 80 | if strings.Contains(filepath, "?") { 81 | filepath = strings.Split(filepath, "?")[0] 82 | } 83 | filepath = path.Ext(filepath) 84 | return filepath 85 | } 86 | -------------------------------------------------------------------------------- /extract.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "time" 7 | 8 | "github.com/bwmarrin/discordgo" 9 | "mvdan.cc/xurls/v2" 10 | ) 11 | 12 | func getDownloadLinks(inputURL string, channelID string, interactive bool) (map[string]string, bool) { 13 | if RegexpUrlTwitter.MatchString(inputURL) { 14 | links, err := getTwitterUrls(inputURL) 15 | if err != nil { 16 | fmt.Println("twitter URL failed,", inputURL, ",", err) 17 | } else if len(links) > 0 { 18 | return skipDuplicateLinks(links, channelID, interactive), true 19 | } 20 | } 21 | if RegexpUrlTwitterParams.MatchString(inputURL) { 22 | links, err := getTwitterParamsUrls(inputURL) 23 | if err != nil { 24 | fmt.Println("twitter params URL failed,", inputURL, ",", err) 25 | } else if len(links) > 0 { 26 | return skipDuplicateLinks(links, channelID, interactive), true 27 | } 28 | } 29 | if RegexpUrlTwitterStatus.MatchString(inputURL) { 30 | links, err := getTwitterStatusUrls(inputURL, channelID) 31 | if err != nil { 32 | fmt.Println("twitter status URL failed,", inputURL, ",", err) 33 | } else if len(links) > 0 { 34 | return skipDuplicateLinks(links, channelID, interactive), true 35 | } 36 | } 37 | if RegexpUrlTistory.MatchString(inputURL) { 38 | links, err := getTistoryUrls(inputURL) 39 | if err != nil { 40 | fmt.Println("tistory URL failed,", inputURL, ",", err) 41 | } else if len(links) > 0 { 42 | return skipDuplicateLinks(links, channelID, interactive), true 43 | } 44 | } 45 | if RegexpUrlTistoryLegacy.MatchString(inputURL) { 46 | links, err := getLegacyTistoryUrls(inputURL) 47 | if err != nil { 48 | fmt.Println("legacy tistory URL failed,", inputURL, ",", err) 49 | } else if len(links) > 0 { 50 | return skipDuplicateLinks(links, channelID, interactive), true 51 | } 52 | } 53 | if RegexpUrlGfycat.MatchString(inputURL) { 54 | links, err := getGfycatUrls(inputURL) 55 | if err != nil { 56 | fmt.Println("gfycat URL failed,", inputURL, ",", err) 57 | } else if len(links) > 0 { 58 | return skipDuplicateLinks(links, channelID, interactive), true 59 | } 60 | } 61 | if RegexpUrlInstagram.MatchString(inputURL) { 62 | links, err := getInstagramUrls(inputURL) 63 | if err != nil { 64 | fmt.Println("instagram URL failed,", inputURL, ",", err) 65 | } else if len(links) > 0 { 66 | return skipDuplicateLinks(links, channelID, interactive), true 67 | } 68 | } 69 | if RegexpUrlImgurSingle.MatchString(inputURL) { 70 | links, err := getImgurSingleUrls(inputURL) 71 | if err != nil { 72 | fmt.Println("imgur single URL failed, ", inputURL, ",", err) 73 | } else if len(links) > 0 { 74 | return skipDuplicateLinks(links, channelID, interactive), true 75 | } 76 | } 77 | if RegexpUrlImgurAlbum.MatchString(inputURL) { 78 | links, err := getImgurAlbumUrls(inputURL) 79 | if err != nil { 80 | fmt.Println("imgur album URL failed, ", inputURL, ",", err) 81 | } else if len(links) > 0 { 82 | return skipDuplicateLinks(links, channelID, interactive), true 83 | } 84 | } 85 | if RegexpUrlGoogleDrive.MatchString(inputURL) { 86 | links, err := getGoogleDriveUrls(inputURL) 87 | if err != nil { 88 | fmt.Println("google drive album URL failed, ", inputURL, ",", err) 89 | } else if len(links) > 0 { 90 | return skipDuplicateLinks(links, channelID, interactive), true 91 | } 92 | } 93 | if RegexpUrlFlickrPhoto.MatchString(inputURL) { 94 | links, err := getFlickrPhotoUrls(inputURL) 95 | if err != nil { 96 | fmt.Println("flickr photo URL failed, ", inputURL, ",", err) 97 | } else if len(links) > 0 { 98 | return skipDuplicateLinks(links, channelID, interactive), true 99 | } 100 | } 101 | if RegexpUrlFlickrAlbum.MatchString(inputURL) { 102 | links, err := getFlickrAlbumUrls(inputURL) 103 | if err != nil { 104 | fmt.Println("flickr album URL failed, ", inputURL, ",", err) 105 | } else if len(links) > 0 { 106 | return skipDuplicateLinks(links, channelID, interactive), true 107 | } 108 | } 109 | if RegexpUrlFlickrAlbumShort.MatchString(inputURL) { 110 | links, err := getFlickrAlbumShortUrls(inputURL) 111 | if err != nil { 112 | fmt.Println("flickr album short URL failed, ", inputURL, ",", err) 113 | } else if len(links) > 0 { 114 | return skipDuplicateLinks(links, channelID, interactive), true 115 | } 116 | } 117 | if RegexpUrlStreamable.MatchString(inputURL) { 118 | links, err := getStreamableUrls(inputURL) 119 | if err != nil { 120 | fmt.Println("streamable URL failed, ", inputURL, ",", err) 121 | } else if len(links) > 0 { 122 | return skipDuplicateLinks(links, channelID, interactive), true 123 | } 124 | } 125 | if DownloadTistorySites { 126 | if RegexpUrlPossibleTistorySite.MatchString(inputURL) { 127 | links, err := getPossibleTistorySiteUrls(inputURL) 128 | if err != nil { 129 | fmt.Println("checking for tistory site failed, ", inputURL, ",", err) 130 | } else if len(links) > 0 { 131 | return skipDuplicateLinks(links, channelID, interactive), true 132 | } 133 | } 134 | } 135 | if RegexpUrlGoogleDriveFolder.MatchString(inputURL) { 136 | if interactive { 137 | links, err := getGoogleDriveFolderUrls(inputURL) 138 | if err != nil { 139 | fmt.Println("google drive folder URL failed, ", inputURL, ",", err) 140 | } else if len(links) > 0 { 141 | return skipDuplicateLinks(links, channelID, interactive), true 142 | } 143 | } else { 144 | fmt.Println("google drive folder only accepted in interactive channels") 145 | } 146 | } 147 | 148 | if !interactive && isDiscordEmoji(inputURL) { 149 | fmt.Printf("skipped %s as it is a Discord emoji\n", inputURL) 150 | return nil, true 151 | } 152 | 153 | // try without queries 154 | parsedURL, err := url.Parse(inputURL) 155 | if err == nil { 156 | parsedURL.RawQuery = "" 157 | inputURLWithoutQueries := parsedURL.String() 158 | if inputURLWithoutQueries != inputURL { 159 | foundLinks, match := getDownloadLinks(inputURLWithoutQueries, channelID, interactive) 160 | if match { 161 | 162 | return skipDuplicateLinks(foundLinks, channelID, interactive), true 163 | } 164 | } 165 | } 166 | 167 | return skipDuplicateLinks(map[string]string{inputURL: ""}, channelID, interactive), false 168 | } 169 | 170 | // getDownloadItemsOfMessage will extract all unique download links out of a message 171 | func getDownloadItemsOfMessage(message *discordgo.Message) []*DownloadItem { 172 | var downloadItems []*DownloadItem 173 | 174 | linkTime, err := message.Timestamp.Parse() 175 | if err != nil { 176 | linkTime = time.Now() 177 | } 178 | 179 | rawLinks := getRawLinksOfMessage(message) 180 | for _, rawLink := range rawLinks { 181 | downloadLinks, _ := getDownloadLinks( 182 | rawLink.Link, 183 | message.ChannelID, 184 | false, 185 | ) 186 | for link, filename := range downloadLinks { 187 | if rawLink.Filename != "" { 188 | filename = rawLink.Filename 189 | } 190 | 191 | downloadItems = append(downloadItems, &DownloadItem{ 192 | Link: link, 193 | Filename: filename, 194 | Time: linkTime, 195 | }) 196 | } 197 | } 198 | 199 | downloadItems = deduplicateDownloadItems(downloadItems) 200 | 201 | return downloadItems 202 | } 203 | 204 | // getRawLinksOfMessage will extract all raw links of a message 205 | func getRawLinksOfMessage(message *discordgo.Message) []*DownloadItem { 206 | var links []*DownloadItem 207 | 208 | if message.Author == nil { 209 | message.Author = new(discordgo.User) 210 | } 211 | 212 | for _, attachment := range message.Attachments { 213 | links = append(links, &DownloadItem{ 214 | Link: attachment.URL, 215 | Filename: attachment.Filename, 216 | }) 217 | } 218 | 219 | foundLinks := xurls.Strict().FindAllString(message.Content, -1) 220 | for _, foundLink := range foundLinks { 221 | links = append(links, &DownloadItem{ 222 | Link: foundLink, 223 | }) 224 | } 225 | 226 | for _, embed := range message.Embeds { 227 | if embed.URL != "" { 228 | links = append(links, &DownloadItem{ 229 | Link: embed.URL, 230 | }) 231 | } 232 | 233 | if embed.Description != "" { 234 | foundLinks = xurls.Strict().FindAllString(embed.Description, -1) 235 | for _, foundLink := range foundLinks { 236 | links = append(links, &DownloadItem{ 237 | Link: foundLink, 238 | }) 239 | } 240 | } 241 | 242 | if embed.Image != nil && embed.Image.URL != "" { 243 | links = append(links, &DownloadItem{ 244 | Link: embed.Image.URL, 245 | }) 246 | } 247 | 248 | if embed.Video != nil && embed.Video.URL != "" { 249 | links = append(links, &DownloadItem{ 250 | Link: embed.Video.URL, 251 | }) 252 | } 253 | } 254 | 255 | return links 256 | } 257 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Seklfreak/discord-image-downloader-go 2 | 3 | go 1.15 4 | 5 | require ( 6 | bou.ke/monkey v1.0.1 // indirect 7 | github.com/ChimeraCoder/anaconda v2.0.0+incompatible 8 | github.com/ChimeraCoder/tokenbucket v0.0.0-20131201223612-c5a927568de7 // indirect 9 | github.com/HouzuoGuo/tiedot v0.0.0-20190123063644-be8ab1f1598e 10 | github.com/Jeffail/gabs v1.2.0 11 | github.com/PuerkitoBio/goquery v1.5.0 12 | github.com/azr/backoff v0.0.0-20160115115103-53511d3c7330 // indirect 13 | github.com/bwmarrin/discordgo v0.22.0 14 | github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc // indirect 15 | github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect 16 | github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17 // indirect 17 | github.com/hashicorp/go-version v1.1.0 18 | github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff // indirect 19 | golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 20 | golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914 21 | google.golang.org/api v0.2.0 22 | gopkg.in/ini.v1 v1.42.0 23 | mvdan.cc/xurls/v2 v2.2.0 24 | ) 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg= 2 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= 4 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 5 | git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= 6 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 7 | github.com/ChimeraCoder/anaconda v2.0.0+incompatible h1:F0eD7CHXieZ+VLboCD5UAqCeAzJZxcr90zSCcuJopJs= 8 | github.com/ChimeraCoder/anaconda v2.0.0+incompatible/go.mod h1:TCt3MijIq3Qqo9SBtuW/rrM4x7rDfWqYWHj8T7hLcLg= 9 | github.com/ChimeraCoder/tokenbucket v0.0.0-20131201223612-c5a927568de7 h1:r+EmXjfPosKO4wfiMLe1XQictsIlhErTufbWUsjOTZs= 10 | github.com/ChimeraCoder/tokenbucket v0.0.0-20131201223612-c5a927568de7/go.mod h1:b2EuEMLSG9q3bZ95ql1+8oVqzzrTNSiOQqSXWFBzxeI= 11 | github.com/HouzuoGuo/tiedot v0.0.0-20190123063644-be8ab1f1598e h1:ZlkqcGyUHfuEnRRUbjFQBSiPceYOwMxcXiiEQ2dzWdg= 12 | github.com/HouzuoGuo/tiedot v0.0.0-20190123063644-be8ab1f1598e/go.mod h1:J2FcoVwTshOscfh8D4LCCVRoHJJQTeCAEkeRSVGnLQs= 13 | github.com/Jeffail/gabs v1.2.0 h1:uFhoIVTtsX7hV2RxNgWad8gMU+8OJdzFbOathJdhD3o= 14 | github.com/Jeffail/gabs v1.2.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= 15 | github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= 16 | github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= 17 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 18 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 19 | github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= 20 | github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 21 | github.com/azr/backoff v0.0.0-20160115115103-53511d3c7330 h1:ekDALXAVvY/Ub1UtNta3inKQwZ/jMB/zpOtD8rAYh78= 22 | github.com/azr/backoff v0.0.0-20160115115103-53511d3c7330/go.mod h1:nH+k0SvAt3HeiYyOlJpLLv1HG1p7KWP7qU9QPp2/pCo= 23 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 24 | github.com/bwmarrin/discordgo v0.22.0 h1:uBxY1HmlVCsW1IuaPjpCGT6A2DBwRn0nvOguQIxDdFM= 25 | github.com/bwmarrin/discordgo v0.22.0/go.mod h1:c1WtWUGN6nREDmzIpyTp/iD3VYt4Fpx+bVyfBG7JE+M= 26 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 27 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 28 | github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc h1:tP7tkU+vIsEOKiK+l/NSLN4uUtkyuxc6hgYpQeCWAeI= 29 | github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc/go.mod h1:ORH5Qp2bskd9NzSfKqAF7tKfONsEkCarTE5ESr/RVBw= 30 | github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad h1:Qk76DOWdOp+GlyDKBAG3Klr9cn7N+LcYc82AZ2S7+cA= 31 | github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad/go.mod h1:mPKfmRa823oBIgl2r20LeMSpTAteW5j7FLkc0vjmzyQ= 32 | github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17 h1:GOfMz6cRgTJ9jWV0qAezv642OhPnKEG7gtUjJSdStHE= 33 | github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17/go.mod h1:HfkOCN6fkKKaPSAeNq/er3xObxTW4VLeY6UUK895gLQ= 34 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 35 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 36 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 37 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 38 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 39 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 40 | github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= 41 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 42 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 43 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 44 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 45 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 46 | github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= 47 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 48 | github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= 49 | github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= 50 | github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 51 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 52 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 53 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 54 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 55 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 56 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 57 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 58 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 59 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 60 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 61 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 62 | github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= 63 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 64 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 65 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 66 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 67 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 68 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 69 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 70 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 71 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 72 | github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 73 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 74 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 75 | github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= 76 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 77 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 78 | go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A= 79 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 80 | golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 81 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= 82 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 83 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 84 | golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 85 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 86 | golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 87 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 88 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 89 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 90 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 91 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 92 | golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 93 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 94 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 95 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 96 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 97 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 98 | golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 h1:78jEq2G3J16aXneH23HSnTQQTCwMHoyO8VEiUH+bpPM= 99 | golang.org/x/net v0.0.0-20190322120337-addf6b3196f6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 100 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 101 | golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 102 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 103 | golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914 h1:jIOcLT9BZzyJ9ce+IwwZ+aF9yeCqzrR+NrD68a/SHKw= 104 | golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 105 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 106 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 107 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 108 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 109 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 110 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 111 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 112 | golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 113 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 114 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 115 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 116 | golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 117 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 118 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 119 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 120 | google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 121 | google.golang.org/api v0.2.0 h1:B5VXkdjt7K2Gm6fGBC9C9a1OAKJDT95cTqwet+2zib0= 122 | google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU= 123 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 124 | google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 125 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 126 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 127 | google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= 128 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 129 | google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= 130 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 131 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 132 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 133 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 134 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 135 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 136 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 137 | gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= 138 | gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 139 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 140 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 141 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 142 | honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 143 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 144 | mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A= 145 | mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= 146 | -------------------------------------------------------------------------------- /handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/bwmarrin/discordgo" 9 | "mvdan.cc/xurls/v2" 10 | ) 11 | 12 | // helpHandler displays the help message 13 | func helpHandler(message *discordgo.Message) { 14 | dg.ChannelMessageSend( 15 | message.ChannelID, 16 | "**** to download a link\n"+ 17 | "**version** to find out the version\n"+ 18 | "**stats** to view stats\n"+ 19 | "**channels** to list active channels\n"+ 20 | "**history** to download a full channel history\n"+ 21 | "**help** to open this help\n", 22 | ) 23 | } 24 | 25 | // versionHandler displays the current version 26 | func versionHandler(message *discordgo.Message) { 27 | result := fmt.Sprintf("discord-image-downloder-go **v%s**\n", VERSION) 28 | 29 | if !isLatestRelease() { 30 | result += fmt.Sprintf("**update available on <%s>**", RELEASE_URL) 31 | } else { 32 | result += "version is up to date" 33 | } 34 | 35 | dg.ChannelMessageSend(message.ChannelID, result) 36 | } 37 | 38 | // channelsHandler displays enabled channels 39 | func channelsHandler(message *discordgo.Message) { 40 | var err error 41 | var channel *discordgo.Channel 42 | var channelRecipientUsername string 43 | 44 | result := "**Channels**\n" 45 | for channelID, channelFolder := range ChannelWhitelist { 46 | channel, err = dg.State.Channel(channelID) 47 | if err != nil { 48 | continue 49 | } 50 | 51 | switch channel.Type { 52 | case discordgo.ChannelTypeDM: 53 | channelRecipientUsername = "" 54 | for _, recipient := range channel.Recipients { 55 | channelRecipientUsername = recipient.Username + ", " 56 | } 57 | channelRecipientUsername = strings.TrimRight(channelRecipientUsername, ", ") 58 | if channelRecipientUsername == "" { 59 | channelRecipientUsername = "N/A" 60 | } 61 | result += fmt.Sprintf("@%s (`#%s`): `%s`\n", channelRecipientUsername, channel.ID, channelFolder) 62 | default: 63 | result += fmt.Sprintf("<#%s> (`#%s`): `%s`\n", channel.ID, channel.ID, channelFolder) 64 | } 65 | 66 | } 67 | result += "**Interactive Channels**\n" 68 | for channelID, channelFolder := range InteractiveChannelWhitelist { 69 | channel, err = dg.State.Channel(channelID) 70 | if err != nil { 71 | continue 72 | } 73 | 74 | switch channel.Type { 75 | case discordgo.ChannelTypeDM: 76 | channelRecipientUsername = "" 77 | for _, recipient := range channel.Recipients { 78 | channelRecipientUsername = recipient.Username + ", " 79 | } 80 | channelRecipientUsername = strings.TrimRight(channelRecipientUsername, ", ") 81 | if channelRecipientUsername == "" { 82 | channelRecipientUsername = "N/A" 83 | } 84 | result += fmt.Sprintf("@%s (`#%s`): `%s`\n", channelRecipientUsername, channel.ID, channelFolder) 85 | default: 86 | result += fmt.Sprintf("<#%s> (`#%s`): `%s`\n", channel.ID, channel.ID, channelFolder) 87 | } 88 | } 89 | 90 | for _, page := range Pagify(result, "\n") { 91 | dg.ChannelMessageSend(message.ChannelID, page) 92 | } 93 | } 94 | 95 | func statsHandler(message *discordgo.Message) { 96 | channelStats := make(map[string]int) 97 | userStats := make(map[string]int) 98 | userGuilds := make(map[string]string) 99 | 100 | var i int 101 | myDB.Use("Downloads").ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) { 102 | downloadedImage := findDownloadedImageById(id) 103 | channelStats[downloadedImage.ChannelId] += 1 104 | userStats[downloadedImage.UserId] += 1 105 | if _, ok := userGuilds[downloadedImage.UserId]; !ok { 106 | channel, err := dg.State.Channel(downloadedImage.ChannelId) 107 | if err == nil && channel.GuildID != "" { 108 | userGuilds[downloadedImage.UserId] = channel.GuildID 109 | } 110 | } 111 | i++ 112 | return true 113 | }) 114 | channelStatsSorted := sortStringIntMapByValue(channelStats) 115 | userStatsSorted := sortStringIntMapByValue(userStats) 116 | replyMessage := fmt.Sprintf("I downloaded **%d** pictures in **%d** channels by **%d** users\n", i, len(channelStats), len(userStats)) 117 | 118 | replyMessage += "**channel breakdown**\n" 119 | for _, downloads := range channelStatsSorted { 120 | channel, err := dg.State.Channel(downloads.Key) 121 | if err == nil { 122 | if channel.Type == discordgo.ChannelTypeDM { 123 | channelRecipientUsername := "N/A" 124 | for _, recipient := range channel.Recipients { 125 | channelRecipientUsername = recipient.Username 126 | } 127 | replyMessage += fmt.Sprintf("@%s (`#%s`): **%d** downloads\n", channelRecipientUsername, downloads.Key, downloads.Value) 128 | } else { 129 | guild, err := dg.State.Guild(channel.GuildID) 130 | if err == nil { 131 | replyMessage += fmt.Sprintf("#%s/%s (`#%s`): **%d** downloads\n", guild.Name, channel.Name, downloads.Key, downloads.Value) 132 | } else { 133 | fmt.Println(err) 134 | } 135 | } 136 | } else { 137 | fmt.Println(err) 138 | } 139 | } 140 | replyMessage += "**user breakdown**\n" 141 | var userI int 142 | for _, downloads := range userStatsSorted { 143 | userI++ 144 | if userI > 10 { 145 | replyMessage += "_only the top 10 users get shown_\n" 146 | break 147 | } 148 | if guildId, ok := userGuilds[downloads.Key]; ok { 149 | user, err := dg.State.Member(guildId, downloads.Key) 150 | if err == nil { 151 | replyMessage += fmt.Sprintf("@%s: **%d** downloads\n", user.User.Username, downloads.Value) 152 | } else { 153 | replyMessage += fmt.Sprintf("@`%s`: **%d** downloads\n", downloads.Key, downloads.Value) 154 | } 155 | } else { 156 | replyMessage += fmt.Sprintf("@`%s`: **%d** downloads\n", downloads.Key, downloads.Value) 157 | } 158 | } 159 | 160 | for _, page := range Pagify(replyMessage, "\n") { 161 | dg.ChannelMessageSend(message.ChannelID, page) 162 | } 163 | } 164 | 165 | func historyHandler(message *discordgo.Message) { 166 | i := 0 167 | _, historyCommandIsSet := historyCommandActive[message.ChannelID] 168 | if !historyCommandIsSet || historyCommandActive[message.ChannelID] == "" { 169 | historyCommandActive[message.ChannelID] = "" 170 | 171 | idArray := strings.Split(message.Content, ",") 172 | for _, channelValue := range idArray { 173 | channelValue = strings.TrimSpace(channelValue) 174 | if folder, ok := ChannelWhitelist[channelValue]; ok { 175 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("downloading to `%s`", folder)) 176 | historyCommandActive[message.ChannelID] = "downloading" 177 | lastBefore := "" 178 | var lastBeforeTime time.Time 179 | MessageRequestingLoop: 180 | for true { 181 | if lastBeforeTime != (time.Time{}) { 182 | fmt.Printf("[%s] Requesting 100 more messages, (before %s)\n", time.Now().Format(time.Stamp), lastBeforeTime) 183 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Requesting 100 more messages, (before %s)\n", lastBeforeTime)) 184 | } 185 | messages, err := dg.ChannelMessages(channelValue, 100, lastBefore, "", "") 186 | if err == nil { 187 | if len(messages) <= 0 { 188 | delete(historyCommandActive, message.ChannelID) 189 | break MessageRequestingLoop 190 | } 191 | lastBefore = messages[len(messages)-1].ID 192 | lastBeforeTime, err = messages[len(messages)-1].Timestamp.Parse() 193 | if err != nil { 194 | fmt.Println(err) 195 | } 196 | for _, msg := range messages { 197 | fileTime := time.Now() 198 | if msg.Timestamp != "" { 199 | fileTime, err = msg.Timestamp.Parse() 200 | if err != nil { 201 | fmt.Println(err) 202 | } 203 | } 204 | if historyCommandActive[message.ChannelID] == "cancel" { 205 | delete(historyCommandActive, message.ChannelID) 206 | dg.ChannelMessageSend(message.ChannelID, "cancelled history downloading") 207 | break MessageRequestingLoop 208 | } 209 | for _, iAttachment := range msg.Attachments { 210 | if len(findDownloadedImageByUrl(iAttachment.URL)) == 0 { 211 | i++ 212 | startDownload(iAttachment.URL, iAttachment.Filename, folder, msg.ChannelID, msg.Author.ID, fileTime) 213 | } 214 | } 215 | foundUrls := xurls.Strict().FindAllString(msg.Content, -1) 216 | for _, iFoundUrl := range foundUrls { 217 | links, _ := getDownloadLinks(iFoundUrl, msg.ChannelID, false) 218 | for link, filename := range links { 219 | if len(findDownloadedImageByUrl(link)) == 0 { 220 | i++ 221 | startDownload(link, filename, folder, msg.ChannelID, msg.Author.ID, fileTime) 222 | } 223 | } 224 | } 225 | } 226 | } else { 227 | dg.ChannelMessageSend(message.ChannelID, err.Error()) 228 | fmt.Println(err) 229 | delete(historyCommandActive, message.ChannelID) 230 | break MessageRequestingLoop 231 | } 232 | } 233 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("done, %d download links started!", i)) 234 | } else { 235 | dg.ChannelMessageSend(message.ChannelID, "Please tell me one or multiple Channel IDs (separated by commas)\nPlease make sure the channels have been whitelisted before submitting.") 236 | } 237 | } 238 | } else if historyCommandActive[message.ChannelID] == "downloading" && strings.ToLower(message.Content) == "cancel" { 239 | historyCommandActive[message.ChannelID] = "cancel" 240 | } 241 | } 242 | 243 | func defaultHandler(message *discordgo.Message) { 244 | folderName := InteractiveChannelWhitelist[message.ChannelID] 245 | 246 | if link, ok := interactiveChannelLinkTemp[message.ChannelID]; ok { 247 | fileTime := time.Now() 248 | var err error 249 | if message.Timestamp != "" { 250 | fileTime, err = message.Timestamp.Parse() 251 | if err != nil { 252 | fmt.Println(err) 253 | } 254 | } 255 | if message.Content == "." { 256 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> started", link)) 257 | dg.ChannelTyping(message.ChannelID) 258 | delete(interactiveChannelLinkTemp, message.ChannelID) 259 | links, _ := getDownloadLinks(link, message.ChannelID, true) 260 | for linkR, filename := range links { 261 | startDownload(linkR, filename, folderName, message.ChannelID, message.Author.ID, fileTime) 262 | } 263 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> finished", link)) 264 | } else if strings.ToLower(message.Content) == "cancel" { 265 | delete(interactiveChannelLinkTemp, message.ChannelID) 266 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> cancelled", link)) 267 | } else if IsValid(message.Content) { 268 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> started", link)) 269 | dg.ChannelTyping(message.ChannelID) 270 | delete(interactiveChannelLinkTemp, message.ChannelID) 271 | links, _ := getDownloadLinks(link, message.ChannelID, true) 272 | for linkR, filename := range links { 273 | startDownload(linkR, filename, message.Content, message.ChannelID, message.Author.ID, fileTime) 274 | } 275 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Download of <%s> finished", link)) 276 | } else { 277 | dg.ChannelMessageSend(message.ChannelID, "invalid path") 278 | } 279 | } else { 280 | _ = folderName 281 | foundLinks := false 282 | for _, iAttachment := range message.Attachments { 283 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Where do you want to save <%s>?\nType **.** for default path or **cancel** to cancel the download %s", iAttachment.URL, folderName)) 284 | interactiveChannelLinkTemp[message.ChannelID] = iAttachment.URL 285 | foundLinks = true 286 | } 287 | foundUrls := xurls.Strict().FindAllString(message.Content, -1) 288 | for _, iFoundUrl := range foundUrls { 289 | dg.ChannelMessageSend(message.ChannelID, fmt.Sprintf("Where do you want to save <%s>?\nType **.** for default path or **cancel** to cancel the download %s", iFoundUrl, folderName)) 290 | interactiveChannelLinkTemp[message.ChannelID] = iFoundUrl 291 | foundLinks = true 292 | } 293 | if foundLinks == false { 294 | dg.ChannelMessageSend(message.ChannelID, "unable to find valid link") 295 | } 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "mime" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "os/signal" 14 | "path" 15 | "path/filepath" 16 | "regexp" 17 | "sort" 18 | "strconv" 19 | "strings" 20 | "syscall" 21 | "time" 22 | 23 | "github.com/ChimeraCoder/anaconda" 24 | "github.com/HouzuoGuo/tiedot/db" 25 | "github.com/Jeffail/gabs" 26 | "github.com/PuerkitoBio/goquery" 27 | "github.com/bwmarrin/discordgo" 28 | "github.com/hashicorp/go-version" 29 | "golang.org/x/net/context" 30 | "golang.org/x/net/html" 31 | "golang.org/x/oauth2/google" 32 | "google.golang.org/api/drive/v3" 33 | "google.golang.org/api/googleapi" 34 | "gopkg.in/ini.v1" 35 | ) 36 | 37 | var ( 38 | ChannelWhitelist map[string]string 39 | InteractiveChannelWhitelist map[string]string 40 | dg *discordgo.Session 41 | DownloadTistorySites bool 42 | interactiveChannelLinkTemp map[string]string 43 | DiscordUserId string 44 | myDB *db.DB 45 | historyCommandActive map[string]string 46 | MaxDownloadRetries int 47 | flickrApiKey string 48 | twitterClient *anaconda.TwitterApi 49 | DownloadTimeout int 50 | SendNoticesToInteractiveChannels bool 51 | StatusEnabled bool 52 | StatusType string 53 | StatusLabel discordgo.GameType 54 | StatusSuffix string 55 | clientCredentialsJson string 56 | DriveService *drive.Service 57 | RegexpFilename *regexp.Regexp 58 | ) 59 | 60 | type GfycatObject struct { 61 | GfyItem struct { 62 | Mp4URL string `json:"mp4Url"` 63 | } `json:"gfyItem"` 64 | } 65 | 66 | type ImgurAlbumObject struct { 67 | Data []struct { 68 | Link string 69 | } 70 | } 71 | 72 | func main() { 73 | fmt.Printf("> discord-image-downloader-go v%s -- discordgo v%s\n", VERSION, discordgo.VERSION) 74 | if !isLatestRelease() { 75 | fmt.Printf("update available on %s !\n", RELEASE_URL) 76 | } 77 | 78 | var err error 79 | 80 | var configFile string 81 | flag.StringVar(&configFile, "config", DEFAULT_CONFIG_FILE, "config file to read from") 82 | 83 | flag.Parse() 84 | 85 | if configFile == "" { 86 | configFile = DEFAULT_CONFIG_FILE 87 | } 88 | 89 | fmt.Printf("reading config from %s\n", configFile) 90 | cfg, err := ini.Load(configFile) 91 | if err != nil { 92 | fmt.Println("unable to read config file", err) 93 | cfg = ini.Empty() 94 | } 95 | 96 | if (!cfg.Section("auth").HasKey("email") || 97 | !cfg.Section("auth").HasKey("password")) && 98 | !cfg.Section("auth").HasKey("token") { 99 | cfg.Section("auth").NewKey("email", "your@email.com") 100 | cfg.Section("auth").NewKey("password", "your password") 101 | cfg.Section("general").NewKey("skip edits", "true") 102 | cfg.Section("general").NewKey("download tistory sites", "false") 103 | cfg.Section("general").NewKey("max download retries", "5") 104 | cfg.Section("general").NewKey("download timeout", "60") 105 | cfg.Section("general").NewKey("send notices to interactive channels", "false") 106 | cfg.Section("status").NewKey("status enabled", "true") 107 | cfg.Section("status").NewKey("status type", "online") 108 | cfg.Section("status").NewKey("status label", fmt.Sprint(discordgo.GameTypeWatching)) 109 | cfg.Section("status").NewKey("status suffix", "downloaded pictures") 110 | cfg.Section("channels").NewKey("channelid1", "C:\\full\\path\\1") 111 | cfg.Section("channels").NewKey("channelid2", "C:\\full\\path\\2") 112 | cfg.Section("channels").NewKey("channelid3", "C:\\full\\path\\3") 113 | cfg.Section("flickr").NewKey("api key", "your flickr api key") 114 | cfg.Section("twitter").NewKey("consumer key", "your consumer key") 115 | cfg.Section("twitter").NewKey("consumer secret", "your consumer secret") 116 | cfg.Section("twitter").NewKey("access token", "your access token") 117 | cfg.Section("twitter").NewKey("access token secret", "your access token secret") 118 | err = cfg.SaveTo("config.ini") 119 | 120 | if err != nil { 121 | fmt.Println("unable to write config file", err) 122 | return 123 | } 124 | fmt.Println("Wrote config file, please fill out and restart the program") 125 | return 126 | } 127 | 128 | myDB, err = db.OpenDB(DATABASE_DIR) 129 | if err != nil { 130 | fmt.Println("unable to create db", err) 131 | return 132 | } 133 | if myDB.Use("Downloads") == nil { 134 | fmt.Println("Creating new database...") 135 | if err := myDB.Create("Downloads"); err != nil { 136 | fmt.Println("unable to create db", err) 137 | return 138 | } 139 | if err := myDB.Use("Downloads").Index([]string{"Url"}); err != nil { 140 | fmt.Println("unable to create index", err) 141 | return 142 | } 143 | } 144 | 145 | ChannelWhitelist = cfg.Section("channels").KeysHash() 146 | InteractiveChannelWhitelist = cfg.Section("interactive channels").KeysHash() 147 | interactiveChannelLinkTemp = make(map[string]string) 148 | historyCommandActive = make(map[string]string) 149 | flickrApiKey = cfg.Section("flickr").Key("api key").MustString("yourflickrapikey") 150 | twitterConsumerKey := cfg.Section("twitter").Key("consumer key").MustString("your consumer key") 151 | twitterConsumerSecret := cfg.Section("twitter").Key("consumer secret").MustString("your consumer secret") 152 | twitterAccessToken := cfg.Section("twitter").Key("access token").MustString("your access token") 153 | twitterAccessTokenSecret := cfg.Section("twitter").Key("access token secret").MustString("your access token secret") 154 | if twitterAccessToken != "" && 155 | twitterAccessTokenSecret != "" && 156 | twitterConsumerKey != "" && 157 | twitterConsumerSecret != "" { 158 | twitterClient = anaconda.NewTwitterApiWithCredentials( 159 | twitterAccessToken, 160 | twitterAccessTokenSecret, 161 | twitterConsumerKey, 162 | twitterConsumerSecret, 163 | ) 164 | } 165 | 166 | err = initRegex() 167 | if err != nil { 168 | fmt.Println("error initialising regex,", err.Error()) 169 | return 170 | } 171 | 172 | if cfg.Section("auth").HasKey("token") { 173 | dg, err = discordgo.New(cfg.Section("auth").Key("token").String()) 174 | } else { 175 | dg, err = discordgo.New( 176 | cfg.Section("auth").Key("email").String(), 177 | cfg.Section("auth").Key("password").String()) 178 | } 179 | if err != nil { 180 | // Newer discordgo throws this error for some reason with Email/Password login 181 | if err.Error() != "Unable to fetch discord authentication token. " { 182 | fmt.Println("error creating Discord session,", err) 183 | return 184 | } 185 | } 186 | 187 | dg.AddHandler(messageCreate) 188 | 189 | if cfg.Section("general").HasKey("skip edits") { 190 | if cfg.Section("general").Key("skip edits").MustBool() == false { 191 | dg.AddHandler(messageUpdate) 192 | } 193 | } 194 | 195 | DownloadTistorySites = cfg.Section("general").Key("download tistory sites").MustBool() 196 | MaxDownloadRetries = cfg.Section("general").Key("max download retries").MustInt(3) 197 | DownloadTimeout = cfg.Section("general").Key("download timeout").MustInt(60) 198 | SendNoticesToInteractiveChannels = cfg.Section("general").Key("send notices to interactive channels").MustBool(false) 199 | 200 | StatusEnabled = cfg.Section("status").Key("status enabled").MustBool(true) 201 | StatusType = cfg.Section("status").Key("status type").MustString("online") 202 | StatusLabel = discordgo.GameType(cfg.Section("status").Key("status label").MustInt(int(discordgo.GameTypeWatching))) 203 | StatusSuffix = cfg.Section("status").Key("status suffix").MustString("downloaded pictures") 204 | 205 | // setup google drive client 206 | clientCredentialsJson = cfg.Section("google").Key("client credentials json").MustString("") 207 | if clientCredentialsJson != "" { 208 | ctx := context.Background() 209 | authJson, err := ioutil.ReadFile(clientCredentialsJson) 210 | if err != nil { 211 | fmt.Println("error opening google credentials json,", err) 212 | } else { 213 | config, err := google.JWTConfigFromJSON(authJson, drive.DriveReadonlyScope) 214 | if err != nil { 215 | fmt.Println("error parsing google credentials json,", err) 216 | } else { 217 | client := config.Client(ctx) 218 | DriveService, err = drive.New(client) 219 | if err != nil { 220 | fmt.Println("error setting up google drive client,", err) 221 | } 222 | } 223 | } 224 | } 225 | dg.LogLevel = -1 // to ignore dumb wsapi error 226 | err = dg.Open() 227 | if err != nil { 228 | fmt.Println("error opening connection,", err) 229 | return 230 | } 231 | dg.LogLevel = 0 // reset 232 | 233 | u, err := dg.User("@me") 234 | if err != nil { 235 | fmt.Println("error obtaining account details,", err) 236 | } 237 | 238 | fmt.Printf("Client is now connected as %s. Press CTRL-C to exit.\n", 239 | u.Username) 240 | DiscordUserId = u.ID 241 | 242 | updateDiscordStatus() 243 | 244 | // keep program running until CTRL-C is pressed. 245 | sc := make(chan os.Signal, 1) 246 | signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt) 247 | <-sc 248 | fmt.Println("Closing database...") 249 | myDB.Close() 250 | fmt.Println("Logging out of Discord...") 251 | dg.Close() 252 | fmt.Println("Exiting...") 253 | return 254 | } 255 | 256 | func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) { 257 | handleDiscordMessage(m.Message) 258 | } 259 | 260 | func messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) { 261 | if m.EditedTimestamp != discordgo.Timestamp("") { 262 | handleDiscordMessage(m.Message) 263 | } 264 | } 265 | 266 | func skipDuplicateLinks(linkList map[string]string, channelID string, interactive bool) map[string]string { 267 | if interactive == false { 268 | newList := make(map[string]string, 0) 269 | for link, filename := range linkList { 270 | downloadedImages := findDownloadedImageByUrl(link) 271 | isMatched := false 272 | for _, downloadedImage := range downloadedImages { 273 | if downloadedImage.ChannelId == channelID { 274 | isMatched = true 275 | } 276 | } 277 | if isMatched == false { 278 | newList[link] = filename 279 | } else { 280 | fmt.Println("url already downloaded in this channel:", link) 281 | } 282 | } 283 | return newList 284 | } 285 | return linkList 286 | } 287 | 288 | func handleDiscordMessage(m *discordgo.Message) { 289 | if folderName, ok := ChannelWhitelist[m.ChannelID]; ok { 290 | // download from whitelisted channels 291 | downloadItems := getDownloadItemsOfMessage(m) 292 | 293 | for _, downloadItem := range downloadItems { 294 | startDownload( 295 | downloadItem.Link, 296 | downloadItem.Filename, 297 | folderName, 298 | m.ChannelID, 299 | m.Author.ID, 300 | downloadItem.Time, 301 | ) 302 | } 303 | 304 | } else if _, ok := InteractiveChannelWhitelist[m.ChannelID]; ok { 305 | // handle interactive channel 306 | 307 | // skip messages from the Bot 308 | if m.Author == nil || m.Author.ID == DiscordUserId { 309 | return 310 | } 311 | 312 | dg.ChannelTyping(m.ChannelID) 313 | 314 | args := strings.Fields(m.Content) 315 | if len(args) <= 0 { 316 | return 317 | } 318 | 319 | _, historyCommandIsActive := historyCommandActive[m.ChannelID] 320 | 321 | switch strings.ToLower(args[0]) { 322 | case "help": 323 | helpHandler(m) 324 | case "version": 325 | versionHandler(m) 326 | case "channels": 327 | channelsHandler(m) 328 | case "stats": 329 | statsHandler(m) 330 | case "history": 331 | historyHandler(m) 332 | default: 333 | if historyCommandIsActive { 334 | historyHandler(m) 335 | return 336 | } 337 | 338 | defaultHandler(m) 339 | } 340 | } 341 | } 342 | 343 | type GithubReleaseApiObject struct { 344 | TagName string `json:"tag_name"` 345 | } 346 | 347 | func isLatestRelease() bool { 348 | githubReleaseApiObject := new(GithubReleaseApiObject) 349 | getJson(RELEASE_API_URL, githubReleaseApiObject) 350 | currentVer, err := version.NewVersion(VERSION) 351 | if err != nil { 352 | fmt.Println(err) 353 | return true 354 | } 355 | lastVer, err := version.NewVersion(githubReleaseApiObject.TagName) 356 | if err != nil { 357 | fmt.Println(err) 358 | return true 359 | } 360 | if lastVer.GreaterThan(currentVer) { 361 | return false 362 | } 363 | return true 364 | } 365 | 366 | // http://stackoverflow.com/a/35240286/1443726 367 | func IsValid(fp string) bool { 368 | // Check if file already exists 369 | if _, err := os.Stat(fp); err == nil { 370 | return true 371 | } 372 | 373 | // Attempt to create it 374 | var d []byte 375 | if err := ioutil.WriteFile(fp, d, 0644); err == nil { 376 | os.Remove(fp) // And delete it 377 | return true 378 | } 379 | 380 | return false 381 | } 382 | 383 | func getTwitterUrls(url string) (map[string]string, error) { 384 | parts := strings.Split(url, ":") 385 | if len(parts) < 2 { 386 | return nil, errors.New("unable to parse twitter url") 387 | } 388 | return map[string]string{"https:" + parts[1] + ":orig": filenameFromUrl(parts[1])}, nil 389 | } 390 | 391 | func getTwitterParamsUrls(url string) (map[string]string, error) { 392 | matches := RegexpUrlTwitterParams.FindStringSubmatch(url) 393 | 394 | return map[string]string{ 395 | "https://pbs.twimg.com/media/" + matches[3] + "." + matches[4] + ":orig": matches[3] + "." + matches[4], 396 | }, nil 397 | } 398 | 399 | func getTwitterStatusUrls(url string, channelID string) (map[string]string, error) { 400 | if twitterClient == nil { 401 | return nil, errors.New("invalid twitter api keys set") 402 | } 403 | 404 | matches := RegexpUrlTwitterStatus.FindStringSubmatch(url) 405 | statusId, err := strconv.ParseInt(matches[4], 10, 64) 406 | if err != nil { 407 | return nil, err 408 | } 409 | 410 | tweet, err := twitterClient.GetTweet(statusId, nil) 411 | if err != nil { 412 | return nil, err 413 | } 414 | 415 | links := make(map[string]string) 416 | for _, tweetMedia := range tweet.ExtendedEntities.Media { 417 | if len(tweetMedia.VideoInfo.Variants) > 0 { 418 | var lastVideoVariant anaconda.Variant 419 | for _, videoVariant := range tweetMedia.VideoInfo.Variants { 420 | if videoVariant.Bitrate >= lastVideoVariant.Bitrate { 421 | lastVideoVariant = videoVariant 422 | } 423 | } 424 | if lastVideoVariant.Url != "" { 425 | links[lastVideoVariant.Url] = "" 426 | } 427 | } else { 428 | foundUrls, _ := getDownloadLinks(tweetMedia.Media_url_https, channelID, false) 429 | for foundUrlKey, foundUrlValue := range foundUrls { 430 | links[foundUrlKey] = foundUrlValue 431 | } 432 | } 433 | } 434 | for _, tweetUrl := range tweet.Entities.Urls { 435 | foundUrls, _ := getDownloadLinks(tweetUrl.Expanded_url, channelID, false) 436 | for foundUrlKey, foundUrlValue := range foundUrls { 437 | links[foundUrlKey] = foundUrlValue 438 | } 439 | } 440 | 441 | return links, nil 442 | } 443 | 444 | func getGfycatUrls(url string) (map[string]string, error) { 445 | parts := strings.Split(url, "/") 446 | if len(parts) < 3 { 447 | return nil, errors.New("unable to parse gfycat url") 448 | } else { 449 | gfycatId := parts[len(parts)-1] 450 | gfycatObject := new(GfycatObject) 451 | getJson("https://api.gfycat.com/v1/gfycats/"+gfycatId, gfycatObject) 452 | gfycatUrl := gfycatObject.GfyItem.Mp4URL 453 | if url == "" { 454 | return nil, errors.New("failed to read response from gfycat") 455 | } 456 | return map[string]string{gfycatUrl: ""}, nil 457 | } 458 | } 459 | 460 | func getInstagramUrls(url string) (map[string]string, error) { 461 | username, shortcode := getInstagramInfo(url) 462 | filename := fmt.Sprintf("instagram %s - %s", username, shortcode) 463 | // if instagram video 464 | videoUrl := getInstagramVideoUrl(url) 465 | if videoUrl != "" { 466 | return map[string]string{videoUrl: filename + filepathExtension(videoUrl)}, nil 467 | } 468 | // if instagram album 469 | albumUrls := getInstagramAlbumUrls(url) 470 | if len(albumUrls) > 0 { 471 | //fmt.Println("is instagram album") 472 | links := make(map[string]string) 473 | for i, albumUrl := range albumUrls { 474 | links[albumUrl] = filename + " " + strconv.Itoa(i+1) + filepathExtension(albumUrl) 475 | } 476 | return links, nil 477 | } 478 | // if instagram picture 479 | afterLastSlash := strings.LastIndex(url, "/") 480 | mediaUrl := url[:afterLastSlash] 481 | mediaUrl += strings.Replace(strings.Replace(url[afterLastSlash:], "?", "&", -1), "/", "/media/?size=l", -1) 482 | return map[string]string{mediaUrl: filename + ".jpg"}, nil 483 | } 484 | 485 | func getInstagramInfo(url string) (string, string) { 486 | resp, err := http.Get(url) 487 | 488 | if err != nil { 489 | return "N/A", "N/A" 490 | } 491 | 492 | defer resp.Body.Close() 493 | z := html.NewTokenizer(resp.Body) 494 | 495 | ParseLoop: 496 | for { 497 | tt := z.Next() 498 | switch { 499 | case tt == html.ErrorToken: 500 | break ParseLoop 501 | } 502 | if tt == html.StartTagToken || tt == html.SelfClosingTagToken { 503 | t := z.Token() 504 | for _, a := range t.Attr { 505 | if a.Key == "type" { 506 | if a.Val == "text/javascript" { 507 | z.Next() 508 | content := string(z.Text()) 509 | if strings.Contains(content, "window._sharedData = ") { 510 | content = strings.Replace(content, "window._sharedData = ", "", 1) 511 | content = content[:len(content)-1] 512 | jsonParsed, err := gabs.ParseJSON([]byte(content)) 513 | if err != nil { 514 | fmt.Println("error parsing instagram json: ", err) 515 | continue ParseLoop 516 | } 517 | entryChildren, err := jsonParsed.Path("entry_data.PostPage").Children() 518 | if err != nil { 519 | fmt.Println("unable to find entries children: ", err) 520 | continue ParseLoop 521 | } 522 | for _, entryChild := range entryChildren { 523 | shortcode := entryChild.Path("graphql.shortcode_media.shortcode").Data().(string) 524 | username := entryChild.Path("graphql.shortcode_media.owner.username").Data().(string) 525 | return username, shortcode 526 | } 527 | } 528 | } 529 | } 530 | } 531 | } 532 | } 533 | return "N/A", "N/A" 534 | } 535 | 536 | func getImgurSingleUrls(url string) (map[string]string, error) { 537 | url = regexp.MustCompile(`(r\/[^\/]+\/)`).ReplaceAllString(url, "") // remove subreddit url 538 | fmt.Println(url) 539 | url = strings.Replace(url, "imgur.com/", "imgur.com/download/", -1) 540 | url = strings.Replace(url, ".gifv", "", -1) 541 | return map[string]string{url: ""}, nil 542 | } 543 | 544 | func getImgurAlbumUrls(url string) (map[string]string, error) { 545 | url = regexp.MustCompile(`(#[A-Za-z0-9]+)?$`).ReplaceAllString(url, "") // remove anchor 546 | afterLastSlash := strings.LastIndex(url, "/") 547 | albumId := url[afterLastSlash+1:] 548 | headers := make(map[string]string) 549 | headers["Authorization"] = "Client-ID " + IMGUR_CLIENT_ID 550 | imgurAlbumObject := new(ImgurAlbumObject) 551 | getJsonWithHeaders("https://api.imgur.com/3/album/"+albumId+"/images", imgurAlbumObject, headers) 552 | links := make(map[string]string) 553 | for _, v := range imgurAlbumObject.Data { 554 | links[v.Link] = "" 555 | } 556 | if len(links) <= 0 { 557 | return getImgurSingleUrls(url) 558 | } 559 | fmt.Printf("[%s] Found imgur album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url) 560 | return links, nil 561 | } 562 | 563 | func getGoogleDriveUrls(url string) (map[string]string, error) { 564 | parts := strings.Split(url, "/") 565 | if len(parts) != 7 { 566 | return nil, errors.New("unable to parse google drive url") 567 | } 568 | fileId := parts[len(parts)-2] 569 | return map[string]string{"https://drive.google.com/uc?export=download&id=" + fileId: ""}, nil 570 | } 571 | 572 | func getGoogleDriveFolderUrls(url string) (map[string]string, error) { 573 | matches := RegexpUrlGoogleDriveFolder.FindStringSubmatch(url) 574 | if len(matches) < 4 || matches[3] == "" { 575 | return nil, errors.New("unable to find google drive folder ID in link") 576 | } 577 | if DriveService.BasePath == "" { 578 | return nil, errors.New("please set up google credentials") 579 | } 580 | googleDriveFolderID := matches[3] 581 | 582 | links := make(map[string]string) 583 | 584 | driveQuery := fmt.Sprintf("\"%s\" in parents", googleDriveFolderID) 585 | driveFields := "nextPageToken, files(id)" 586 | result, err := DriveService.Files.List().Q(driveQuery).Fields(googleapi.Field(driveFields)).PageSize(1000).Do() 587 | if err != nil { 588 | fmt.Println("driveQuery:", driveQuery) 589 | fmt.Println("driveFields:", driveFields) 590 | fmt.Println("err:", err) 591 | return nil, err 592 | } 593 | for _, file := range result.Files { 594 | fileUrl := "https://drive.google.com/uc?export=download&id=" + file.Id 595 | links[fileUrl] = "" 596 | } 597 | 598 | for { 599 | if result.NextPageToken == "" { 600 | break 601 | } 602 | result, err = DriveService.Files.List().Q(driveQuery).Fields(googleapi.Field(driveFields)).PageSize(1000).PageToken(result.NextPageToken).Do() 603 | if err != nil { 604 | return nil, err 605 | } 606 | for _, file := range result.Files { 607 | links[file.Id] = "" 608 | } 609 | } 610 | return links, nil 611 | } 612 | 613 | type FlickrPhotoSizeObject struct { 614 | Label string `json:"label"` 615 | Width int `json:"width,int,string"` 616 | Height int `json:"height,int,string"` 617 | Source string `json:"source"` 618 | URL string `json:"url"` 619 | Media string `json:"media"` 620 | } 621 | 622 | type FlickrPhotoObject struct { 623 | Sizes struct { 624 | Canblog int `json:"canblog"` 625 | Canprint int `json:"canprint"` 626 | Candownload int `json:"candownload"` 627 | Size []FlickrPhotoSizeObject `json:"size"` 628 | } `json:"sizes"` 629 | Stat string `json:"stat"` 630 | } 631 | 632 | func getFlickrUrlFromPhotoId(photoId string) string { 633 | reqUrl := fmt.Sprintf("https://www.flickr.com/services/rest/?format=json&nojsoncallback=1&method=%s&api_key=%s&photo_id=%s", 634 | "flickr.photos.getSizes", flickrApiKey, photoId) 635 | flickrPhoto := new(FlickrPhotoObject) 636 | getJson(reqUrl, flickrPhoto) 637 | var bestSize FlickrPhotoSizeObject 638 | for _, size := range flickrPhoto.Sizes.Size { 639 | if bestSize.Label == "" { 640 | bestSize = size 641 | } else { 642 | if size.Width > bestSize.Width || size.Height > bestSize.Height { 643 | bestSize = size 644 | } 645 | } 646 | } 647 | return bestSize.Source 648 | } 649 | 650 | func getFlickrPhotoUrls(url string) (map[string]string, error) { 651 | if flickrApiKey == "" || flickrApiKey == "yourflickrapikey" || flickrApiKey == "your flickr api key" { 652 | return nil, errors.New("invalid flickr api key set") 653 | } 654 | matches := RegexpUrlFlickrPhoto.FindStringSubmatch(url) 655 | photoId := matches[5] 656 | if photoId == "" { 657 | return nil, errors.New("unable to get photo id from url") 658 | } 659 | return map[string]string{getFlickrUrlFromPhotoId(photoId): ""}, nil 660 | } 661 | 662 | type FlickrAlbumObject struct { 663 | Photoset struct { 664 | ID string `json:"id"` 665 | Primary string `json:"primary"` 666 | Owner string `json:"owner"` 667 | Ownername string `json:"ownername"` 668 | Photo []struct { 669 | ID string `json:"id"` 670 | Secret string `json:"secret"` 671 | Server string `json:"server"` 672 | Farm int `json:"farm"` 673 | Title string `json:"title"` 674 | Isprimary string `json:"isprimary"` 675 | Ispublic int `json:"ispublic"` 676 | Isfriend int `json:"isfriend"` 677 | Isfamily int `json:"isfamily"` 678 | } `json:"photo"` 679 | Page int `json:"page"` 680 | PerPage int `json:"per_page"` 681 | Perpage int `json:"perpage"` 682 | Pages int `json:"pages"` 683 | Total string `json:"total"` 684 | Title string `json:"title"` 685 | } `json:"photoset"` 686 | Stat string `json:"stat"` 687 | } 688 | 689 | func getFlickrAlbumUrls(url string) (map[string]string, error) { 690 | if flickrApiKey == "" || flickrApiKey == "yourflickrapikey" { 691 | return nil, errors.New("invalid flickr api key set") 692 | } 693 | matches := RegexpUrlFlickrAlbum.FindStringSubmatch(url) 694 | if len(matches) < 10 || matches[9] == "" { 695 | return nil, errors.New("unable to find flickr album ID in link") 696 | } 697 | albumId := matches[9] 698 | if albumId == "" { 699 | return nil, errors.New("unable to get album id from url") 700 | } 701 | reqUrl := fmt.Sprintf("https://www.flickr.com/services/rest/?format=json&nojsoncallback=1&method=%s&api_key=%s&photoset_id=%s&per_page=500", 702 | "flickr.photosets.getPhotos", flickrApiKey, albumId) 703 | flickrAlbum := new(FlickrAlbumObject) 704 | getJson(reqUrl, flickrAlbum) 705 | links := make(map[string]string) 706 | for _, photo := range flickrAlbum.Photoset.Photo { 707 | links[getFlickrUrlFromPhotoId(photo.ID)] = "" 708 | } 709 | fmt.Printf("[%s] Found flickr album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url) 710 | return links, nil 711 | } 712 | 713 | func getFlickrAlbumShortUrls(url string) (map[string]string, error) { 714 | result, err := http.Get(url) 715 | if err != nil { 716 | return nil, errors.New("error getting long url from shortened flickr album url: " + err.Error()) 717 | } 718 | if RegexpUrlFlickrAlbum.MatchString(result.Request.URL.String()) { 719 | return getFlickrAlbumUrls(result.Request.URL.String()) 720 | } 721 | return nil, errors.New("got invalid url while trying to get long url from short flickr album url") 722 | } 723 | 724 | type StreamableObject struct { 725 | Status int `json:"status"` 726 | Title string `json:"title"` 727 | Files struct { 728 | Mp4 struct { 729 | URL string `json:"url"` 730 | Width int `json:"width"` 731 | Height int `json:"height"` 732 | } `json:"mp4"` 733 | Mp4Mobile struct { 734 | URL string `json:"url"` 735 | Width int `json:"width"` 736 | Height int `json:"height"` 737 | } `json:"mp4-mobile"` 738 | } `json:"files"` 739 | URL string `json:"url"` 740 | ThumbnailURL string `json:"thumbnail_url"` 741 | Message interface{} `json:"message"` 742 | } 743 | 744 | func getStreamableUrls(url string) (map[string]string, error) { 745 | matches := RegexpUrlStreamable.FindStringSubmatch(url) 746 | shortcode := matches[3] 747 | if shortcode == "" { 748 | return nil, errors.New("unable to get shortcode from url") 749 | } 750 | reqUrl := fmt.Sprintf("https://api.streamable.com/videos/%s", shortcode) 751 | streamable := new(StreamableObject) 752 | getJson(reqUrl, streamable) 753 | if streamable.Status != 2 || streamable.Files.Mp4.URL == "" { 754 | return nil, errors.New("streamable object has no download candidate") 755 | } 756 | link := streamable.Files.Mp4.URL 757 | if !strings.HasPrefix(link, "http") { 758 | link = "https:" + link 759 | } 760 | links := make(map[string]string) 761 | links[link] = "" 762 | return links, nil 763 | } 764 | 765 | func getPossibleTistorySiteUrls(url string) (map[string]string, error) { 766 | client := new(http.Client) 767 | request, err := http.NewRequest("HEAD", url, nil) 768 | if err != nil { 769 | return nil, err 770 | } 771 | request.Header.Add("Accept-Encoding", "identity") 772 | request.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36") 773 | respHead, err := client.Do(request) 774 | if err != nil { 775 | return nil, err 776 | } 777 | 778 | contentType := "" 779 | for headerKey, headerValue := range respHead.Header { 780 | if headerKey == "Content-Type" { 781 | contentType = headerValue[0] 782 | } 783 | } 784 | if !strings.Contains(contentType, "text/html") { 785 | return nil, nil 786 | } 787 | 788 | request, err = http.NewRequest("GET", url, nil) 789 | if err != nil { 790 | return nil, err 791 | } 792 | request.Header.Add("Accept-Encoding", "identity") 793 | request.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36") 794 | resp, err := client.Do(request) 795 | if err != nil { 796 | return nil, err 797 | } 798 | 799 | doc, err := goquery.NewDocumentFromResponse(resp) 800 | if err != nil { 801 | return nil, err 802 | } 803 | 804 | var links = make(map[string]string) 805 | 806 | doc.Find(".article img, #content img, div[role=main] img, .section_blogview img").Each(func(i int, s *goquery.Selection) { 807 | foundUrl, exists := s.Attr("src") 808 | if exists { 809 | isTistoryCdnUrl := RegexpUrlTistoryLegacyWithCDN.MatchString(foundUrl) 810 | isTistoryUrl := RegexpUrlTistoryLegacy.MatchString(foundUrl) 811 | if isTistoryCdnUrl == true { 812 | finalTistoryUrls, _ := getTistoryWithCDNUrls(foundUrl) 813 | if len(finalTistoryUrls) > 0 { 814 | for finalTistoryUrl := range finalTistoryUrls { 815 | foundFilename := s.AttrOr("filename", "") 816 | links[finalTistoryUrl] = foundFilename 817 | } 818 | } 819 | } else if isTistoryUrl == true { 820 | finalTistoryUrls, _ := getLegacyTistoryUrls(foundUrl) 821 | if len(finalTistoryUrls) > 0 { 822 | for finalTistoryUrl := range finalTistoryUrls { 823 | foundFilename := s.AttrOr("filename", "") 824 | links[finalTistoryUrl] = foundFilename 825 | } 826 | } 827 | } 828 | } 829 | }) 830 | 831 | if len(links) > 0 { 832 | fmt.Printf("[%s] Found tistory album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url) 833 | } 834 | return links, nil 835 | } 836 | 837 | func getJson(url string, target interface{}) error { 838 | r, err := http.Get(url) 839 | if err != nil { 840 | return err 841 | } 842 | defer r.Body.Close() 843 | 844 | return json.NewDecoder(r.Body).Decode(target) 845 | } 846 | 847 | func getJsonWithHeaders(url string, target interface{}, headers map[string]string) error { 848 | client := &http.Client{} 849 | req, _ := http.NewRequest("GET", url, nil) 850 | 851 | for k, v := range headers { 852 | req.Header.Set(k, v) 853 | } 854 | 855 | r, err := client.Do(req) 856 | if err != nil { 857 | return err 858 | } 859 | defer r.Body.Close() 860 | 861 | return json.NewDecoder(r.Body).Decode(target) 862 | } 863 | 864 | func getInstagramVideoUrl(url string) string { 865 | resp, err := http.Get(url) 866 | 867 | if err != nil { 868 | return "" 869 | } 870 | 871 | defer resp.Body.Close() 872 | z := html.NewTokenizer(resp.Body) 873 | 874 | for { 875 | tt := z.Next() 876 | switch { 877 | case tt == html.ErrorToken: 878 | return "" 879 | } 880 | if tt == html.StartTagToken || tt == html.SelfClosingTagToken { 881 | t := z.Token() 882 | if t.Data == "meta" { 883 | for _, a := range t.Attr { 884 | if a.Key == "property" { 885 | if a.Val == "og:video" || a.Val == "og:video:secure_url" { 886 | for _, at := range t.Attr { 887 | if at.Key == "content" { 888 | return at.Val 889 | } 890 | } 891 | } 892 | } 893 | } 894 | } 895 | } 896 | } 897 | } 898 | 899 | func getInstagramAlbumUrls(url string) []string { 900 | var links []string 901 | resp, err := http.Get(url) 902 | 903 | if err != nil { 904 | return links 905 | } 906 | 907 | defer resp.Body.Close() 908 | z := html.NewTokenizer(resp.Body) 909 | 910 | ParseLoop: 911 | for { 912 | tt := z.Next() 913 | switch { 914 | case tt == html.ErrorToken: 915 | break ParseLoop 916 | } 917 | if tt == html.StartTagToken || tt == html.SelfClosingTagToken { 918 | t := z.Token() 919 | for _, a := range t.Attr { 920 | if a.Key == "type" { 921 | if a.Val == "text/javascript" { 922 | z.Next() 923 | content := string(z.Text()) 924 | if strings.Contains(content, "window._sharedData = ") { 925 | content = strings.Replace(content, "window._sharedData = ", "", 1) 926 | content = content[:len(content)-1] 927 | jsonParsed, err := gabs.ParseJSON([]byte(content)) 928 | if err != nil { 929 | fmt.Println("error parsing instagram json: ", err) 930 | continue ParseLoop 931 | } 932 | entryChildren, err := jsonParsed.Path("entry_data.PostPage").Children() 933 | if err != nil { 934 | fmt.Println("unable to find entries children: ", err) 935 | continue ParseLoop 936 | } 937 | for _, entryChild := range entryChildren { 938 | albumChildren, err := entryChild.Path("graphql.shortcode_media.edge_sidecar_to_children.edges").Children() 939 | if err != nil { 940 | continue ParseLoop 941 | } 942 | for _, albumChild := range albumChildren { 943 | link, ok := albumChild.Path("node.display_url").Data().(string) 944 | if ok { 945 | links = append(links, link) 946 | } 947 | } 948 | } 949 | } 950 | } 951 | } 952 | } 953 | } 954 | } 955 | if len(links) > 0 { 956 | fmt.Printf("[%s] Found instagram album with %d images (url: %s)\n", time.Now().Format(time.Stamp), len(links), url) 957 | } 958 | 959 | return links 960 | } 961 | 962 | func filenameFromUrl(dUrl string) string { 963 | base := path.Base(dUrl) 964 | parts := strings.Split(base, "?") 965 | return parts[0] 966 | } 967 | 968 | func startDownload(dUrl string, filename string, path string, channelId string, userId string, fileTime time.Time) { 969 | success := false 970 | for i := 0; i < MaxDownloadRetries; i++ { 971 | success = downloadFromUrl(dUrl, filename, path, channelId, userId, fileTime) 972 | if success == true { 973 | break 974 | } else { 975 | time.Sleep(5 * time.Second) 976 | } 977 | } 978 | if success == false { 979 | fmt.Println("Gave up on downloading", dUrl) 980 | if SendNoticesToInteractiveChannels == true { 981 | for channelId := range InteractiveChannelWhitelist { 982 | content := fmt.Sprintf("Gave up on downloading %s, no success after %d retries", dUrl, MaxDownloadRetries) 983 | _, err := dg.ChannelMessageSend(channelId, content) 984 | if err != nil { 985 | fmt.Println("Failed to send notice to", channelId, "-", err) 986 | } 987 | } 988 | } 989 | } 990 | } 991 | 992 | func downloadFromUrl(dUrl string, filename string, path string, channelId string, userId string, fileTime time.Time) bool { 993 | err := os.MkdirAll(path, 0755) 994 | if err != nil { 995 | fmt.Println("Error while creating folder", path, "-", err) 996 | return false 997 | } 998 | 999 | timeout := time.Duration(time.Duration(DownloadTimeout) * time.Second) 1000 | client := &http.Client{ 1001 | Timeout: timeout, 1002 | } 1003 | request, err := http.NewRequest("GET", dUrl, nil) 1004 | if err != nil { 1005 | fmt.Println("Error while downloading", dUrl, "-", err) 1006 | return false 1007 | } 1008 | request.Header.Add("Accept-Encoding", "identity") 1009 | response, err := client.Do(request) 1010 | if err != nil { 1011 | fmt.Println("Error while downloading", dUrl, "-", err) 1012 | return false 1013 | } 1014 | defer response.Body.Close() 1015 | 1016 | if filename == "" { 1017 | filename = filenameFromUrl(response.Request.URL.String()) 1018 | for key, iHeader := range response.Header { 1019 | if key == "Content-Disposition" { 1020 | _, params, err := mime.ParseMediaType(iHeader[0]) 1021 | if err == nil { 1022 | newFilename, err := url.QueryUnescape(params["filename"]) 1023 | if err != nil { 1024 | newFilename = params["filename"] 1025 | } 1026 | if newFilename != "" { 1027 | filename = newFilename 1028 | } 1029 | } 1030 | } 1031 | } 1032 | } 1033 | 1034 | bodyOfResp, err := ioutil.ReadAll(response.Body) 1035 | if err != nil { 1036 | fmt.Println("Could not read response", dUrl, "-", err) 1037 | return false 1038 | } 1039 | 1040 | contentType := http.DetectContentType(bodyOfResp) 1041 | 1042 | // check for valid filename, if not, replace with generic filename 1043 | if !RegexpFilename.MatchString(filename) { 1044 | filename = time.Now().Format("2006-01-02 15-04-05") 1045 | possibleExtension, _ := mime.ExtensionsByType(contentType) 1046 | if len(possibleExtension) > 0 { 1047 | filename += possibleExtension[0] 1048 | } 1049 | } 1050 | 1051 | completePath := path + string(os.PathSeparator) + filename 1052 | if _, err := os.Stat(completePath); err == nil { 1053 | tmpPath := completePath 1054 | i := 1 1055 | for { 1056 | completePath = tmpPath[0:len(tmpPath)-len(filepathExtension(tmpPath))] + 1057 | "-" + strconv.Itoa(i) + filepathExtension(tmpPath) 1058 | if _, err := os.Stat(completePath); os.IsNotExist(err) { 1059 | break 1060 | } 1061 | i = i + 1 1062 | } 1063 | fmt.Printf("[%s] Saving possible duplicate (filenames match): %s to %s\n", time.Now().Format(time.Stamp), tmpPath, completePath) 1064 | } 1065 | 1066 | extension := filepath.Ext(filename) 1067 | contentTypeParts := strings.Split(contentType, "/") 1068 | if t := contentTypeParts[0]; t != "image" && t != "video" && t != "audio" && 1069 | !(t == "application" && isAudioFile(filename)) && 1070 | strings.ToLower(extension) != ".mov" && 1071 | strings.ToLower(extension) != ".mp4" && 1072 | strings.ToLower(extension) != ".webm" { 1073 | fmt.Println("No image, video, or audio found at", dUrl) 1074 | return true 1075 | } 1076 | 1077 | err = ioutil.WriteFile(completePath, bodyOfResp, 0644) 1078 | if err != nil { 1079 | fmt.Println("Error while writing to disk", dUrl, "-", err) 1080 | return false 1081 | } 1082 | 1083 | err = os.Chtimes(completePath, fileTime, fileTime) 1084 | if err != nil { 1085 | fmt.Println("Error while changing date", dUrl, "-", err) 1086 | } 1087 | 1088 | sourceChannelName := channelId 1089 | sourceGuildName := "N/A" 1090 | sourceChannel, _ := dg.State.Channel(channelId) 1091 | if sourceChannel != nil && sourceChannel.Name != "" { 1092 | sourceChannelName = sourceChannel.Name 1093 | sourceGuild, _ := dg.State.Guild(sourceChannel.GuildID) 1094 | if sourceGuild != nil && sourceGuild.Name != "" { 1095 | sourceGuildName = sourceGuild.Name 1096 | } 1097 | } 1098 | 1099 | fmt.Printf("[%s] Saved URL %s to %s from #%s/%s\n", 1100 | time.Now().Format(time.Stamp), dUrl, completePath, sourceChannelName, sourceGuildName) 1101 | err = insertDownloadedImage(&DownloadedImage{Url: dUrl, Time: time.Now(), Destination: completePath, ChannelId: channelId, UserId: userId}) 1102 | if err != nil { 1103 | fmt.Println("Error while writing to database", err) 1104 | } 1105 | 1106 | updateDiscordStatus() 1107 | return true 1108 | } 1109 | 1110 | func isAudioFile(f string) bool { 1111 | switch strings.ToLower(path.Ext(f)) { 1112 | case ".mp3", ".wav", ".aif": 1113 | return true 1114 | default: 1115 | return false 1116 | } 1117 | } 1118 | 1119 | type DownloadedImage struct { 1120 | Url string 1121 | Time time.Time 1122 | Destination string 1123 | ChannelId string 1124 | UserId string 1125 | } 1126 | 1127 | func insertDownloadedImage(downloadedImage *DownloadedImage) error { 1128 | _, err := myDB.Use("Downloads").Insert(map[string]interface{}{ 1129 | "Url": downloadedImage.Url, 1130 | "Time": downloadedImage.Time.String(), 1131 | "Destination": downloadedImage.Destination, 1132 | "ChannelId": downloadedImage.ChannelId, 1133 | "UserId": downloadedImage.UserId, 1134 | }) 1135 | return err 1136 | } 1137 | 1138 | func findDownloadedImageById(id int) *DownloadedImage { 1139 | downloads := myDB.Use("Downloads") 1140 | //var query interface{} 1141 | //json.Unmarshal([]byte(fmt.Sprintf(`[{"eq": "%d", "in": ["Id"]}]`, id)), &query) 1142 | //queryResult := make(map[int]struct{}) 1143 | //db.EvalQuery(query, myDB.Use("Downloads"), &queryResult) 1144 | 1145 | readBack, err := downloads.Read(id) 1146 | if err != nil { 1147 | fmt.Println(err) 1148 | } 1149 | timeT, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", readBack["Time"].(string)) 1150 | return &DownloadedImage{ 1151 | Url: readBack["Url"].(string), 1152 | Time: timeT, 1153 | Destination: readBack["Destination"].(string), 1154 | ChannelId: readBack["ChannelId"].(string), 1155 | UserId: readBack["UserId"].(string), 1156 | } 1157 | } 1158 | 1159 | func findDownloadedImageByUrl(url string) []*DownloadedImage { 1160 | var query interface{} 1161 | json.Unmarshal([]byte(fmt.Sprintf(`[{"eq": "%s", "in": ["Url"]}]`, url)), &query) 1162 | queryResult := make(map[int]struct{}) 1163 | db.EvalQuery(query, myDB.Use("Downloads"), &queryResult) 1164 | 1165 | downloadedImages := make([]*DownloadedImage, 0) 1166 | for id := range queryResult { 1167 | downloadedImages = append(downloadedImages, findDownloadedImageById(id)) 1168 | } 1169 | return downloadedImages 1170 | } 1171 | 1172 | func countDownloadedImages() int { 1173 | i := 0 1174 | myDB.Use("Downloads").ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) { 1175 | // fmt.Printf("%v\n", findDownloadedImageById(id)) 1176 | i++ 1177 | return true 1178 | }) 1179 | return i 1180 | // fmt.Println(myDB.Use("Downloads").ApproxDocCount()) TODO? 1181 | } 1182 | 1183 | // http://stackoverflow.com/a/18695740/1443726 1184 | func sortStringIntMapByValue(m map[string]int) PairList { 1185 | pl := make(PairList, len(m)) 1186 | i := 0 1187 | for k, v := range m { 1188 | pl[i] = Pair{k, v} 1189 | i++ 1190 | } 1191 | sort.Sort(sort.Reverse(pl)) 1192 | return pl 1193 | } 1194 | 1195 | type Pair struct { 1196 | Key string 1197 | Value int 1198 | } 1199 | 1200 | type PairList []Pair 1201 | 1202 | func (p PairList) Len() int { return len(p) } 1203 | func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value } 1204 | func (p PairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 1205 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "regexp" 7 | "strings" 8 | "testing" 9 | 10 | "gopkg.in/ini.v1" 11 | ) 12 | 13 | func init() { 14 | RegexpUrlTwitter, _ = regexp.Compile(REGEXP_URL_TWITTER) 15 | RegexpUrlTistoryLegacy, _ = regexp.Compile(REGEXP_URL_TISTORY_LEGACY) 16 | RegexpUrlTistoryLegacyWithCDN, _ = regexp.Compile(REGEXP_URL_TISTORY_LEGACY_WITH_CDN) 17 | RegexpUrlGfycat, _ = regexp.Compile(REGEXP_URL_GFYCAT) 18 | RegexpUrlInstagram, _ = regexp.Compile(REGEXP_URL_INSTAGRAM) 19 | RegexpUrlImgurSingle, _ = regexp.Compile(REGEXP_URL_IMGUR_SINGLE) 20 | RegexpUrlImgurAlbum, _ = regexp.Compile(REGEXP_URL_IMGUR_ALBUM) 21 | RegexpUrlGoogleDrive, _ = regexp.Compile(REGEXP_URL_GOOGLEDRIVE) 22 | RegexpUrlPossibleTistorySite, _ = regexp.Compile(REGEXP_URL_POSSIBLE_TISTORY_SITE) 23 | RegexpUrlFlickrPhoto, _ = regexp.Compile(REGEXP_URL_FLICKR_PHOTO) 24 | RegexpUrlFlickrAlbum, _ = regexp.Compile(REGEXP_URL_FLICKR_ALBUM) 25 | RegexpUrlStreamable, _ = regexp.Compile(REGEXP_URL_STREAMABLE) 26 | flickrApiKey = os.Getenv("FLICKR_API_KEY") 27 | 28 | var err error 29 | cfg, err := ini.Load("config.ini") 30 | if err == nil { 31 | flickrApiKey = cfg.Section("flickr").Key("api key").MustString("yourflickrapikey") 32 | } 33 | } 34 | 35 | type urlsTestpair struct { 36 | value string 37 | result map[string]string 38 | } 39 | 40 | var getTwitterUrlsTests = []urlsTestpair{ 41 | { 42 | "https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig", 43 | map[string]string{"https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig": "CulDBM6VYAA-YhY.jpg"}, 44 | }, 45 | { 46 | "https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg", 47 | map[string]string{"https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig": "CulDBM6VYAA-YhY.jpg"}, 48 | }, 49 | { 50 | "http://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg", 51 | map[string]string{"https://pbs.twimg.com/media/CulDBM6VYAA-YhY.jpg:orig": "CulDBM6VYAA-YhY.jpg"}, 52 | }, 53 | } 54 | 55 | func TestGetTwitterUrls(t *testing.T) { 56 | for _, pair := range getTwitterUrlsTests { 57 | v, err := getTwitterUrls(pair.value) 58 | if err != nil { 59 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 60 | } 61 | if !reflect.DeepEqual(v, pair.result) { 62 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 63 | } 64 | } 65 | } 66 | 67 | var getTistoryUrlsTests = []urlsTestpair{ 68 | { 69 | "http://cfile25.uf.tistory.com/original/235CA739582E86992EFC4E", 70 | map[string]string{"http://cfile25.uf.tistory.com/original/235CA739582E86992EFC4E": ""}, 71 | }, 72 | { 73 | "http://cfile25.uf.tistory.com/image/235CA739582E86992EFC4E", 74 | map[string]string{"http://cfile25.uf.tistory.com/original/235CA739582E86992EFC4E": ""}, 75 | }, 76 | } 77 | 78 | func TestGetTistoryUrls(t *testing.T) { 79 | for _, pair := range getTistoryUrlsTests { 80 | v, err := getLegacyTistoryUrls(pair.value) 81 | if err != nil { 82 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 83 | } 84 | if !reflect.DeepEqual(v, pair.result) { 85 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 86 | } 87 | } 88 | } 89 | 90 | var getGfycatUrlsTests = []urlsTestpair{ 91 | { 92 | "https://gfycat.com/SandyChiefBoubou", 93 | map[string]string{"https://fat.gfycat.com/SandyChiefBoubou.mp4": ""}, 94 | }, 95 | } 96 | 97 | func TestGetGfycatUrls(t *testing.T) { 98 | for _, pair := range getGfycatUrlsTests { 99 | v, err := getGfycatUrls(pair.value) 100 | if err != nil { 101 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 102 | } 103 | if !reflect.DeepEqual(v, pair.result) { 104 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 105 | } 106 | } 107 | } 108 | 109 | var getInstagramUrlsPictureTests = []urlsTestpair{ 110 | { 111 | "https://www.instagram.com/p/BHhDAmhAz33/?taken-by=s_sohye", 112 | map[string]string{"https://www.instagram.com/p/BHhDAmhAz33/media/?size=l&taken-by=s_sohye": "instagram s_sohye - BHhDAmhAz33.jpg"}, 113 | }, 114 | { 115 | "https://www.instagram.com/p/BHhDAmhAz33/", 116 | map[string]string{"https://www.instagram.com/p/BHhDAmhAz33/media/?size=l": "instagram s_sohye - BHhDAmhAz33.jpg"}, 117 | }, 118 | } 119 | 120 | var getInstagramUrlsVideoTests = []urlsTestpair{ 121 | { 122 | "https://www.instagram.com/p/BL2_ZIHgYTp/?taken-by=s_sohye", 123 | map[string]string{"14811404_233311497085396_338650092456116224_n.mp4": "instagram s_sohye - BL2_ZIHgYTp.mp4"}, 124 | }, 125 | { 126 | "https://www.instagram.com/p/BL2_ZIHgYTp/", 127 | map[string]string{"14811404_233311497085396_338650092456116224_n.mp4": "instagram s_sohye - BL2_ZIHgYTp.mp4"}, 128 | }, 129 | } 130 | 131 | var getInstagramUrlsAlbumTests = []urlsTestpair{ 132 | { 133 | "https://www.instagram.com/p/BRiCc0VjULk/?taken-by=gfriendofficial", 134 | map[string]string{ 135 | "17265460_395888184109957_3500310922180689920_n.jpg": "instagram gfriendofficial - BRiCc0VjULk", 136 | "17265456_267171360360765_8110946520456495104_n.jpg": "instagram gfriendofficial - BRiCc0VjULk", 137 | "17265327_1394797493912862_2677004307588448256_n.jpg": "instagram gfriendofficial - BRiCc0VjULk"}, 138 | }, 139 | { 140 | "https://www.instagram.com/p/BRhheSPjaQ3/", 141 | map[string]string{ 142 | "17125875_306909746390523_8184965703367917568_n.jpg": "instagram gfriendofficial - BRhheSPjaQ3", 143 | "17266053_188727064951899_2485556569865977856_n.jpg": "instagram gfriendofficial - BRhheSPjaQ3"}, 144 | }, 145 | } 146 | 147 | func TestGetInstagramUrls(t *testing.T) { 148 | for _, pair := range getInstagramUrlsPictureTests { 149 | v, err := getInstagramUrls(pair.value) 150 | if err != nil { 151 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 152 | } 153 | if !reflect.DeepEqual(v, pair.result) { 154 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 155 | } 156 | } 157 | for _, pair := range getInstagramUrlsVideoTests { 158 | v, err := getInstagramUrls(pair.value) 159 | if err != nil { 160 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 161 | } 162 | for keyResult, valueResult := range pair.result { 163 | for keyExpected, valueExpected := range v { 164 | if strings.Contains(keyResult, keyExpected) || valueResult != valueExpected { // CDN location can vary 165 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 166 | } 167 | } 168 | } 169 | } 170 | for _, pair := range getInstagramUrlsAlbumTests { 171 | v, err := getInstagramUrls(pair.value) 172 | if err != nil { 173 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 174 | } 175 | for keyResult, valueResult := range pair.result { 176 | for keyExpected, valueExpected := range v { 177 | if strings.Contains(keyResult, keyExpected) || strings.Contains(valueResult, valueExpected) { // CDN location can vary 178 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 179 | } 180 | } 181 | } 182 | } 183 | } 184 | 185 | var getImgurSingleUrlsTests = []urlsTestpair{ 186 | { 187 | "http://imgur.com/viZictl", 188 | map[string]string{"http://imgur.com/download/viZictl": ""}, 189 | }, 190 | { 191 | "https://imgur.com/viZictl", 192 | map[string]string{"https://imgur.com/download/viZictl": ""}, 193 | }, 194 | { 195 | "https://i.imgur.com/viZictl.jpg", 196 | map[string]string{"https://i.imgur.com/download/viZictl.jpg": ""}, 197 | }, 198 | { 199 | "http://imgur.com/uYwt2VV", 200 | map[string]string{"http://imgur.com/download/uYwt2VV": ""}, 201 | }, 202 | { 203 | "http://i.imgur.com/uYwt2VV.gifv", 204 | map[string]string{"http://i.imgur.com/download/uYwt2VV": ""}, 205 | }, 206 | } 207 | 208 | func TestGetImgurSingleUrls(t *testing.T) { 209 | for _, pair := range getImgurSingleUrlsTests { 210 | v, err := getImgurSingleUrls(pair.value) 211 | if err != nil { 212 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 213 | } 214 | if !reflect.DeepEqual(v, pair.result) { 215 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 216 | } 217 | } 218 | } 219 | 220 | var getImgurAlbumUrlsTests = []urlsTestpair{ 221 | { 222 | "http://imgur.com/a/ALTpi", 223 | map[string]string{ 224 | "https://i.imgur.com/FKoguPh.jpg": "", 225 | "https://i.imgur.com/5FNL6Pe.jpg": "", 226 | "https://i.imgur.com/YA0V0g9.jpg": "", 227 | "https://i.imgur.com/Uc2iDhD.jpg": "", 228 | "https://i.imgur.com/J9JRSSJ.jpg": "", 229 | "https://i.imgur.com/Xrx0uyE.jpg": "", 230 | "https://i.imgur.com/3xDSq1O.jpg": "", 231 | }, 232 | }, 233 | } 234 | 235 | func TestGetImgurAlbumUrls(t *testing.T) { 236 | for _, pair := range getImgurAlbumUrlsTests { 237 | v, err := getImgurAlbumUrls(pair.value) 238 | if err != nil { 239 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 240 | } 241 | for expectedLink, expectedName := range pair.result { 242 | linkFound := false 243 | for gotLink, gotName := range v { 244 | if expectedLink == gotLink && expectedName == gotName { 245 | linkFound = true 246 | } 247 | } 248 | if !linkFound { 249 | t.Errorf("For expected %s %s, got %s", expectedLink, expectedName, v) 250 | } 251 | } 252 | } 253 | } 254 | 255 | var getGoogleDriveUrlsTests = []urlsTestpair{ 256 | { 257 | "https://drive.google.com/file/d/0B8TnwsJqlFllSUtvUEhoSU40WkE/view", 258 | map[string]string{"https://drive.google.com/uc?export=download&id=0B8TnwsJqlFllSUtvUEhoSU40WkE": ""}, 259 | }, 260 | } 261 | 262 | func TestGetGoogleDriveUrls(t *testing.T) { 263 | for _, pair := range getGoogleDriveUrlsTests { 264 | v, err := getGoogleDriveUrls(pair.value) 265 | if err != nil { 266 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 267 | } 268 | if !reflect.DeepEqual(v, pair.result) { 269 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 270 | } 271 | } 272 | } 273 | 274 | var getTistoryWithCDNUrlsTests = []urlsTestpair{ 275 | { 276 | "http://img1.daumcdn.net/thumb/R720x0.q80/?scode=mtistory&fname=http%3A%2F%2Fcfile24.uf.tistory.com%2Fimage%2F2658554B580BDC4C0924CA", 277 | map[string]string{"http://cfile24.uf.tistory.com/original/2658554B580BDC4C0924CA": ""}, 278 | }, 279 | } 280 | 281 | func TestGetTistoryWithCDNUrls(t *testing.T) { 282 | for _, pair := range getTistoryWithCDNUrlsTests { 283 | v, err := getTistoryWithCDNUrls(pair.value) 284 | if err != nil { 285 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 286 | } 287 | if !reflect.DeepEqual(v, pair.result) { 288 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 289 | } 290 | } 291 | } 292 | 293 | var getPossibleTistorySiteUrlsTests = []urlsTestpair{ 294 | { 295 | "http://soonduck.tistory.com/482", 296 | map[string]string{ 297 | "a": "", 298 | "b": "", 299 | "c": "", 300 | "d": "", 301 | "e": "", 302 | }, 303 | }, 304 | { 305 | "http://soonduck.tistory.com/m/482", 306 | map[string]string{ 307 | "a": "", 308 | "b": "", 309 | "c": "", 310 | "d": "", 311 | "e": "", 312 | }, 313 | }, 314 | { 315 | "http://slmn.de/123", 316 | map[string]string{}, 317 | }, 318 | } 319 | 320 | func TestGetPossibleTistorySiteUrls(t *testing.T) { 321 | for _, pair := range getPossibleTistorySiteUrlsTests { 322 | v, err := getPossibleTistorySiteUrls(pair.value) 323 | if err != nil { 324 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 325 | } 326 | if len(pair.result) > len(v) { // only check that received amount of posts >= expected amount of posts 327 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 328 | } 329 | } 330 | } 331 | 332 | var getFlickrUrlFromPhotoIdTests = []map[string]string{ 333 | { 334 | "value": "31065043320", 335 | "result": "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg", 336 | }, 337 | } 338 | 339 | func TestGetFlickrUrlFromPhotoId(t *testing.T) { 340 | for _, pair := range getFlickrUrlFromPhotoIdTests { 341 | v := getFlickrUrlFromPhotoId(pair["value"]) 342 | if v != pair["result"] { 343 | t.Errorf("For %s, expected %s, got %s", pair["value"], pair["result"], v) 344 | } 345 | } 346 | } 347 | 348 | var getFlickrPhotoUrlsTests = []urlsTestpair{ 349 | { 350 | "https://www.flickr.com/photos/137385017@N08/31065043320/in/album-72157677350305446/", 351 | map[string]string{ 352 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 353 | }, 354 | }, 355 | { 356 | "https://www.flickr.com/photos/137385017@N08/31065043320/in/album-72157677350305446", 357 | map[string]string{ 358 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 359 | }, 360 | }, 361 | { 362 | "https://www.flickr.com/photos/137385017@N08/31065043320/", 363 | map[string]string{ 364 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 365 | }, 366 | }, 367 | { 368 | "https://www.flickr.com/photos/137385017@N08/31065043320", 369 | map[string]string{ 370 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 371 | }, 372 | }, 373 | } 374 | 375 | func TestGetFlickrPhotoUrls(t *testing.T) { 376 | for _, pair := range getFlickrPhotoUrlsTests { 377 | v, err := getFlickrPhotoUrls(pair.value) 378 | if err != nil { 379 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 380 | } 381 | if !reflect.DeepEqual(v, pair.result) { 382 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 383 | } 384 | } 385 | } 386 | 387 | var getFlickrAlbumUrlsTests = []urlsTestpair{ 388 | { 389 | "https://www.flickr.com/photos/137385017@N08/albums/72157677350305446/", 390 | map[string]string{ 391 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 392 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "", 393 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "", 394 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "", 395 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "", 396 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "", 397 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "", 398 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "", 399 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "", 400 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "", 401 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "", 402 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "", 403 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "", 404 | }, 405 | }, 406 | { 407 | "https://www.flickr.com/photos/137385017@N08/albums/72157677350305446", 408 | map[string]string{ 409 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 410 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "", 411 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "", 412 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "", 413 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "", 414 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "", 415 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "", 416 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "", 417 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "", 418 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "", 419 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "", 420 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "", 421 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "", 422 | }, 423 | }, 424 | { 425 | "https://www.flickr.com/photos/137385017@N08/albums/with/72157677350305446/", 426 | map[string]string{ 427 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 428 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "", 429 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "", 430 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "", 431 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "", 432 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "", 433 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "", 434 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "", 435 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "", 436 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "", 437 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "", 438 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "", 439 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "", 440 | }, 441 | }, 442 | { 443 | "https://www.flickr.com/photos/137385017@N08/albums/with/72157677350305446", 444 | map[string]string{ 445 | "https://farm6.staticflickr.com/5521/31065043320_cd03a9a448_b.jpg": "", 446 | "https://farm6.staticflickr.com/5651/31434767515_49f88ee12e_b.jpg": "", 447 | "https://farm6.staticflickr.com/5750/31434766825_529fd08071_b.jpg": "", 448 | "https://farm6.staticflickr.com/5811/31319456971_37c8c4708a_b.jpg": "", 449 | "https://farm6.staticflickr.com/5494/30627074913_b7f810fc26_b.jpg": "", 450 | "https://farm6.staticflickr.com/5539/31065042720_d76f643b28_b.jpg": "", 451 | "https://farm6.staticflickr.com/5813/31434765285_94b85d5e8c_b.jpg": "", 452 | "https://farm6.staticflickr.com/5600/31065044090_eca63bd5a5_b.jpg": "", 453 | "https://farm6.staticflickr.com/5733/31434764435_350825477e_b.jpg": "", 454 | "https://farm6.staticflickr.com/5715/30627073573_b86e4b2c22_b.jpg": "", 455 | "https://farm6.staticflickr.com/5758/31289864222_5e3cca7e72_b.jpg": "", 456 | "https://farm6.staticflickr.com/5801/30627076673_5a32f3e562_b.jpg": "", 457 | "https://farm6.staticflickr.com/5538/31319458901_088858d7f1_b.jpg": "", 458 | }, 459 | }, 460 | } 461 | 462 | func TestGetFlickrAlbumUrls(t *testing.T) { 463 | for _, pair := range getFlickrAlbumUrlsTests { 464 | v, err := getFlickrAlbumUrls(pair.value) 465 | if err != nil { 466 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 467 | } 468 | if !reflect.DeepEqual(v, pair.result) { 469 | t.Errorf("For %s, expected %s, got %s", pair.value, pair.result, v) 470 | } 471 | } 472 | } 473 | 474 | var getStreamableUrlsTests = []urlsTestpair{ 475 | { 476 | "http://streamable.com/41ajc", 477 | map[string]string{ 478 | "streamable.com/video/mp4/41ajc.mp4": "", 479 | }, 480 | }, 481 | } 482 | 483 | func TestGetStreamableUrls(t *testing.T) { 484 | for _, pair := range getStreamableUrlsTests { 485 | v, err := getStreamableUrls(pair.value) 486 | if err != nil { 487 | t.Errorf("For %v, expected %v, got %v", pair.value, nil, err) 488 | } 489 | 490 | for expectedLink, expectedName := range pair.result { 491 | linkFound := false 492 | for gotLink := range v { 493 | if strings.Contains(gotLink, expectedLink) { 494 | linkFound = true 495 | } 496 | } 497 | if !linkFound { 498 | t.Errorf("For expected %s %s, got %s", expectedLink, expectedName, v) 499 | } 500 | } 501 | } 502 | } 503 | -------------------------------------------------------------------------------- /models.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | type DownloadItem struct { 6 | Link string 7 | Filename string 8 | Time time.Time 9 | } 10 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # PROJECT ABANDONED, [PLEASE USE FORK.](https://github.com/get-got/discord-downloader-go/) 2 | 3 | --- 4 | 5 | # discord-image-downloader-go 6 | [](https://www.paypal.me/swk) 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/Seklfreak/discord-image-downloader-go)](https://goreportcard.com/report/github.com/Seklfreak/discord-image-downloader-go) 8 | [![Build Status](https://travis-ci.com/Seklfreak/discord-image-downloader-go.svg?branch=master)](https://travis-ci.com/Seklfreak/discord-image-downloader-go) 9 | 10 | [**DOWNLOAD THE LATEST RELEASE BUILD**](https://github.com/Seklfreak/discord-image-downloader-go/releases/latest) 11 | 12 | This project is not often maintained. For an actively maintained fork that implements features such as extensive JSON settings with channel-specific configurations, see [**get-got/discord-downloader-go**](https://github.com/get-got/discord-downloader-go) 13 | 14 | ## Discord SelfBots are forbidden! 15 | [Official Statement](https://support.discordapp.com/hc/en-us/articles/115002192352-Automated-user-accounts-self-bots-) 16 | ### You have been warned. 17 | 18 | This is a simple tool which downloads media posted in Discord channels of your choice to a local folder. It handles various sources like Twitter differently to make sure to download the best quality available. 19 | 20 | ## Websites currently supported 21 | - Discord Attachments 22 | - Twitter 23 | - Tistory 24 | - Gfycat 25 | - Instagram 26 | - Imgur 27 | - Google Drive Files and Folders 28 | - Flickr 29 | - Streamable 30 | - Any direct link to an image or video 31 | 32 | ## How to use? 33 | When you run the tool for the first time it creates a `config.ini` file with example values. Edit these values and run the tool for a second time. It should connect to discords api and wait for new messages. 34 | 35 | If you are using a normal user account **without two-factor authentication (2FA)**, simply enter your email and password into the corresponding lines in `config.ini`, under the auth section. 36 | 37 | If you are using **two-factor authentication (2FA) you have to login using your token.** Remove the email and password lines under the auth section in the config file and instead put in `token = `. You can acquire your token from the developer tools in your browser (`localStorage.token`) or discord client (`Ctrl+Shift+I` (Windows) or `Cmd+Option+I` (Mac), click Application, click Local Storage, click `https://discordapp.com`, and find "token" and paste the value). 38 | 39 | If you wish to use a **bot account (not a user account)**, go to https://discord.com/developers/applications and create an application, then create a bot in the `Bot` tab in application settings. The bot tab will show you your token. You can invite to your server(s) by going to the `OAuth2` tab in application settings, check `bot`, and copy+paste the url into your browser. **In the `config.ini`, add "Bot " before your token. (example: `token = Bot mytokenhere`)** 40 | 41 | ## How to download old files? 42 | By default, the tool only downloads new links posted while the tool is running. You can also set up the tool to download the complete history of a channel. To do this you have to run this tool with a separate discord account. Send your second account a dm on your primary account and get the channel id from the direct message channel. Now add this channel id to the config by adding the following lines: 43 | ``` 44 | [interactive channels] 45 | = 46 | ``` 47 | After this is done restart the tool and send `history` as a DM to your second account. The bot will ask for the channel id of the channel you want to download and start the downloads. You can view all available commands by sending `help`. 48 | 49 | ### Where do I get the Channel ID? 50 | Enable Developer Mode (in Discord Appearance settings) and right click the channel you need, and click Copy ID. 51 | 52 | **OR,** Open discord in your browser and go to the channel you want to monitor. In your address bar should be a URL like `https://discordapp.com/channels/1234/5678`. The number after the last slash is the channel ID, in this case, `5678`. 53 | 54 | ### Where do I get the Channel ID for Direct Messages? 55 | 1. Inspect Element in the Discord client (`Ctrl+Shift+I` for Windows or `Cmd+Option+I` for Mac) 56 | 1. Go to the `Elements` tab on the left. 57 | 1. Click this icon ![arrow going into box](https://i.imgur.com/PkDOCyZ.png) (the arrow going into a box) and then click on the avatar for the persons DMs you want to grab the ID for. 58 | 1. Somewhere slightly above the HTML it takes you to, there should be a line that looks like this ![](https://i.imgur.com/614rZnX.png) 59 | 1. Copy the number after the `/@me/`. That is your Channel ID to use. 60 | 61 | **OR,** Open discord in your browser and go to the channel you want to monitor. In your address bar should be a URL like `https://discordapp.com/channels/@me/5678`. The number after the last slash is the channel ID, in this case, `5678`. 62 | -------------------------------------------------------------------------------- /regex.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | const ( 8 | REGEXP_URL_TWITTER = `^http(s?):\/\/pbs(-[0-9]+)?\.twimg\.com\/media\/[^\./]+\.(jpg|png)((\:[a-z]+)?)$` 9 | REGEXP_URL_TWITTER_PARAMS = `^http(s?):\/\/pbs(-[0-9]+)?\.twimg\.com\/media\/([^\./]+)\?format=(jpg|png)&name=([a-z]+)[a-z=&]*$` 10 | REGEXP_URL_TWITTER_STATUS = `^http(s?):\/\/(www\.)?twitter\.com\/([A-Za-z0-9-_\.]+\/status\/|statuses\/|i\/web\/status\/)([0-9]+)$` 11 | REGEXP_URL_TISTORY = `^http(s?):\/\/t[0-9]+\.daumcdn\.net\/cfile\/tistory\/([A-Z0-9]+?)(\?original)?$` 12 | REGEXP_URL_TISTORY_LEGACY = `^http(s?):\/\/[a-z0-9]+\.uf\.tistory\.com\/(image|original)\/[A-Z0-9]+$` 13 | REGEXP_URL_TISTORY_LEGACY_WITH_CDN = `^http(s)?:\/\/[0-9a-z]+.daumcdn.net\/[a-z]+\/[a-zA-Z0-9\.]+\/\?scode=mtistory&fname=http(s?)%3A%2F%2F[a-z0-9]+\.uf\.tistory\.com%2F(image|original)%2F[A-Z0-9]+$` 14 | REGEXP_URL_GFYCAT = `^http(s?):\/\/gfycat\.com\/(gifs\/detail\/)?[A-Za-z]+$` 15 | REGEXP_URL_INSTAGRAM = `^http(s?):\/\/(www\.)?instagram\.com\/p\/[^/]+\/(\?[^/]+)?$` 16 | REGEXP_URL_IMGUR_SINGLE = `^http(s?):\/\/(i\.)?imgur\.com\/[A-Za-z0-9]+(\.gifv)?$` 17 | REGEXP_URL_IMGUR_ALBUM = `^http(s?):\/\/imgur\.com\/(a\/|gallery\/|r\/[^\/]+\/)[A-Za-z0-9]+(#[A-Za-z0-9]+)?$` 18 | REGEXP_URL_GOOGLEDRIVE = `^http(s?):\/\/drive\.google\.com\/file\/d\/[^/]+\/view$` 19 | REGEXP_URL_GOOGLEDRIVE_FOLDER = `^http(s?):\/\/drive\.google\.com\/(drive\/folders\/|open\?id=)([^/]+)$` 20 | REGEXP_URL_POSSIBLE_TISTORY_SITE = `^http(s)?:\/\/[0-9a-zA-Z\.-]+\/(m\/)?(photo\/)?[0-9]+$` 21 | REGEXP_URL_FLICKR_PHOTO = `^http(s)?:\/\/(www\.)?flickr\.com\/photos\/([0-9]+)@([A-Z0-9]+)\/([0-9]+)(\/)?(\/in\/album-([0-9]+)(\/)?)?$` 22 | REGEXP_URL_FLICKR_ALBUM = `^http(s)?:\/\/(www\.)?flickr\.com\/photos\/(([0-9]+)@([A-Z0-9]+)|[A-Za-z0-9]+)\/(albums\/(with\/)?|(sets\/)?)([0-9]+)(\/)?$` 23 | REGEXP_URL_FLICKR_ALBUM_SHORT = `^http(s)?:\/\/((www\.)?flickr\.com\/gp\/[0-9]+@[A-Z0-9]+\/[A-Za-z0-9]+|flic\.kr\/s\/[a-zA-Z0-9]+)$` 24 | REGEXP_URL_STREAMABLE = `^http(s?):\/\/(www\.)?streamable\.com\/([0-9a-z]+)$` 25 | 26 | REGEXP_FILENAME = `^^[^/\\:*?"<>|]{1,150}\.[A-Za-z0-9]{2,4}$$` 27 | ) 28 | 29 | var ( 30 | RegexpUrlTwitter *regexp.Regexp 31 | RegexpUrlTwitterParams *regexp.Regexp 32 | RegexpUrlTwitterStatus *regexp.Regexp 33 | RegexpUrlTistory *regexp.Regexp 34 | RegexpUrlTistoryLegacy *regexp.Regexp 35 | RegexpUrlTistoryLegacyWithCDN *regexp.Regexp 36 | RegexpUrlGfycat *regexp.Regexp 37 | RegexpUrlInstagram *regexp.Regexp 38 | RegexpUrlImgurSingle *regexp.Regexp 39 | RegexpUrlImgurAlbum *regexp.Regexp 40 | RegexpUrlGoogleDrive *regexp.Regexp 41 | RegexpUrlGoogleDriveFolder *regexp.Regexp 42 | RegexpUrlPossibleTistorySite *regexp.Regexp 43 | RegexpUrlFlickrPhoto *regexp.Regexp 44 | RegexpUrlFlickrAlbum *regexp.Regexp 45 | RegexpUrlFlickrAlbumShort *regexp.Regexp 46 | RegexpUrlStreamable *regexp.Regexp 47 | ) 48 | 49 | func initRegex() error { 50 | var err error 51 | RegexpUrlTwitter, err = regexp.Compile(REGEXP_URL_TWITTER) 52 | if err != nil { 53 | return err 54 | } 55 | RegexpUrlTwitterParams, err = regexp.Compile(REGEXP_URL_TWITTER_PARAMS) 56 | if err != nil { 57 | return err 58 | } 59 | RegexpUrlTwitterStatus, err = regexp.Compile(REGEXP_URL_TWITTER_STATUS) 60 | if err != nil { 61 | return err 62 | } 63 | RegexpUrlTistory, err = regexp.Compile(REGEXP_URL_TISTORY) 64 | if err != nil { 65 | return err 66 | } 67 | RegexpUrlTistoryLegacy, err = regexp.Compile(REGEXP_URL_TISTORY_LEGACY) 68 | if err != nil { 69 | return err 70 | } 71 | RegexpUrlTistoryLegacyWithCDN, err = regexp.Compile(REGEXP_URL_TISTORY_LEGACY_WITH_CDN) 72 | if err != nil { 73 | return err 74 | } 75 | RegexpUrlGfycat, err = regexp.Compile(REGEXP_URL_GFYCAT) 76 | if err != nil { 77 | return err 78 | } 79 | RegexpUrlInstagram, err = regexp.Compile(REGEXP_URL_INSTAGRAM) 80 | if err != nil { 81 | return err 82 | } 83 | RegexpUrlImgurSingle, err = regexp.Compile(REGEXP_URL_IMGUR_SINGLE) 84 | if err != nil { 85 | return err 86 | } 87 | RegexpUrlImgurAlbum, err = regexp.Compile(REGEXP_URL_IMGUR_ALBUM) 88 | if err != nil { 89 | return err 90 | } 91 | RegexpUrlGoogleDrive, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE) 92 | if err != nil { 93 | return err 94 | } 95 | RegexpUrlGoogleDriveFolder, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE_FOLDER) 96 | if err != nil { 97 | return err 98 | } 99 | RegexpUrlPossibleTistorySite, err = regexp.Compile(REGEXP_URL_POSSIBLE_TISTORY_SITE) 100 | if err != nil { 101 | return err 102 | } 103 | RegexpUrlFlickrPhoto, err = regexp.Compile(REGEXP_URL_FLICKR_PHOTO) 104 | if err != nil { 105 | return err 106 | } 107 | RegexpUrlFlickrAlbum, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM) 108 | if err != nil { 109 | return err 110 | } 111 | RegexpUrlStreamable, err = regexp.Compile(REGEXP_URL_STREAMABLE) 112 | if err != nil { 113 | return err 114 | } 115 | RegexpUrlFlickrAlbumShort, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM_SHORT) 116 | if err != nil { 117 | return err 118 | } 119 | RegexpFilename, err = regexp.Compile(REGEXP_FILENAME) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | return nil 125 | } 126 | -------------------------------------------------------------------------------- /tistory.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | ) 7 | 8 | // getTistoryUrls downloads tistory URLs 9 | // http://t1.daumcdn.net/cfile/tistory/[…] => http://t1.daumcdn.net/cfile/tistory/[…] 10 | // http://t1.daumcdn.net/cfile/tistory/[…]?original => as is 11 | func getTistoryUrls(link string) (map[string]string, error) { 12 | if !strings.HasSuffix(link, "?original") { 13 | link += "?original" 14 | } 15 | return map[string]string{link: ""}, nil 16 | } 17 | 18 | func getLegacyTistoryUrls(link string) (map[string]string, error) { 19 | link = strings.Replace(link, "/image/", "/original/", -1) 20 | return map[string]string{link: ""}, nil 21 | } 22 | 23 | func getTistoryWithCDNUrls(urlI string) (map[string]string, error) { 24 | parameters, _ := url.ParseQuery(urlI) 25 | if val, ok := parameters["fname"]; ok { 26 | if len(val) > 0 { 27 | if RegexpUrlTistoryLegacy.MatchString(val[0]) { 28 | return getLegacyTistoryUrls(val[0]) 29 | } 30 | } 31 | } 32 | return nil, nil 33 | } 34 | -------------------------------------------------------------------------------- /vars.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const ( 4 | VERSION = "1.38" 5 | DATABASE_DIR = "database" 6 | RELEASE_URL = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest" 7 | RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest" 8 | IMGUR_CLIENT_ID = "a39473314df3f59" 9 | 10 | DEFAULT_CONFIG_FILE = "config.ini" 11 | 12 | discordEmojiBaseUrl = "https://cdn.discordapp.com/emojis/" 13 | ) 14 | --------------------------------------------------------------------------------