├── .gitignore ├── go.mod ├── go.sum ├── directory.go ├── README.md ├── main.go ├── coub-dl.go ├── coub_type.go └── parser.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | backups/ 3 | .DS_Store 4 | coub-backup 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/yamamushi/coub-backup 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= 2 | github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 8 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 9 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 10 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 11 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= 12 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 13 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 14 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 15 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 18 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 19 | -------------------------------------------------------------------------------- /directory.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | ) 7 | 8 | func DirectorySetup(user string, dir string) (err error) { 9 | // Creates a directory named user in the dir directory. 10 | // If the directory already exists, it is not created. 11 | return CreateDirectory(dir + user) 12 | } 13 | 14 | // CreateDirectory creates a directory if it does not exist, and returns an error if it cannot be created and does not already exist. 15 | func CreateDirectory(path string) error { 16 | //path = strings.ReplaceAll(path, "/", "\\/") 17 | 18 | if _, err := os.Stat(path); os.IsNotExist(err) { 19 | err = os.MkdirAll(path, 0755) 20 | if err != nil { 21 | return err 22 | } 23 | } 24 | return nil 25 | } 26 | 27 | // FileExists returns true if the file exists, false otherwise. 28 | func FileExists(path string) bool { 29 | if _, err := os.Stat(path); os.IsNotExist(err) { 30 | return false 31 | } 32 | return true 33 | } 34 | 35 | func CreateCoubDir(dir string, coub Coub) (outdir string, err error) { 36 | 37 | //dir = strings.TrimRight(dir, "/") 38 | CoubYear := coub.CreatedAt.Year() 39 | 40 | err = CreateDirectory(dir + "/" + strconv.Itoa(CoubYear)) 41 | if err != nil { 42 | return "", err 43 | } 44 | 45 | // convert month to string 46 | CoubMonthString := coub.CreatedAt.Month().String() 47 | 48 | err = CreateDirectory(dir + "/" + strconv.Itoa(CoubYear) + "/" + CoubMonthString) 49 | if err != nil { 50 | return "", err 51 | } 52 | 53 | //CoubFileSafeTitle := url.QueryEscape(coub.Title) 54 | //Previous: outdir = dir + "/" + strconv.Itoa(CoubYear) + "/" + CoubMonthString + "/" + strconv.Itoa(coub.ID) 55 | //outdir = dir + "/" + strconv.Itoa(CoubYear) + "/" + CoubMonthString + "/" + CoubFileSafeTitle 56 | 57 | outdir = dir + "/" + strconv.Itoa(CoubYear) + "/" + CoubMonthString + "/" + strconv.Itoa(coub.ID) 58 | err = CreateDirectory(outdir) 59 | if err != nil { 60 | return "", err 61 | } 62 | 63 | return outdir, nil 64 | } 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # coub-backup 2 | 3 | # Table of Contents 4 | 1. [About](#about) 5 | 2. [Usage](#usage) 6 | 3. [Contributing](#contributing) 7 | 4. [License](#license) 8 | 9 | ## About 10 | coub-backup is a simple tool to automate the downloading of coubs from [coub.com](https://coub.com). 11 | 12 | This tool will parse a user profile for all of their coubs, and assemble directory structure by year and month for each coub. 13 | 14 | It will generate an info text file that looks like: 15 | 16 | ```text 17 | Title: Atrium - Week End 18 | Created At: 2021-05-31 04:37:17 +0000 UTC 19 | Duration: 8.44 20 | Views: 2754 21 | Recoubs: 4 22 | Source: map[has_embed:true service_name:YouTube type:Youtube url:https://youtu.be/TUYmmvFPEro] 23 | Tags: synthesizer, synthwave, synth, 80s dancing, dancing, classics, live, 80s music, eighties, 80s, disco, italodisco, italo, 80, palpala, italo disco, guillermo gustavo gallardo, guille music video, week end, atrium 24 | 25 | ``` 26 | 27 | Additional metadata is also generated for each coub, including all original coub information. This metadata is stored in a JSON file. 28 | 29 | ## Usage: 30 | 31 | If building from source, with Golang >v.1.17.0, use `go build` in the main directory to get coub-backup.exe. 32 | 33 | ### Backup a user's coubs 34 | 35 | `coub-backup -directory= ` 36 | 37 | ### Backup a community's coubs (max of 500 pages) 38 | 39 | `coub-backup -directory= -community= -pages=10` 40 | 41 | Supported options: *animals-pets, mashup, music, blogging, standup-jokes, movies, anime, gaming, cartoons, art, live-pictures, news, sports, science-technology, food-kitchen, celebrity, nature-travel, fashion, dance, cars, memes, nsfw* 42 | 43 | ### Backup a Best Of for a given year (between 2012 and 2022) 44 | 45 | `coub-backup -directory= -bestof=` 46 | 47 | ### Backup the day's coub of the day 48 | 49 | `coub-backup -directory= -day=true` 50 | 51 | ### Backup the day's featured coubs 52 | 53 | `coub-backup -directory= -featured=true` 54 | 55 | ## Contributing 56 | 57 | It's highly recommended to join the discord group before submitting issues or pull requests: [https://discord.gg/Z7WYmbaGpU](https://discord.gg/Z7WYmbaGpU) 58 | 59 | ## License 60 | 61 | Unless otherwise specified, all content is licensed under the [GPLV3](https://www.gnu.org/licenses/gpl-3.0.en.html) license. -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "os" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | // Parse the command line flags. 13 | // They are set if the short or long option is given, e.g.: 14 | // `go run main.go -v` or `go run main.go --verbose` 15 | // sets the verbose flag. 16 | directory := flag.String("directory", "./backups/", "select output directory") 17 | bestof := flag.String("bestof", "", "select best of to download") 18 | community := flag.String("community", "", "select community to download") 19 | pages := flag.Int("pages", 10, "select number of COMMUNITY pages to download") 20 | featured := flag.Bool("featured", false, "select featured to download") 21 | day := flag.Bool("day", false, "select day to download") 22 | flag.Parse() 23 | 24 | if *bestof != "" { 25 | year, err := strconv.Atoi(*bestof) 26 | if err != nil { 27 | log.Fatal("Best of must be a number") 28 | } 29 | 30 | // if the year is less than 2012 then quit 31 | if year < 2012 { 32 | log.Fatal("Year must be greater than 2012") 33 | } 34 | 35 | log.Println("Attempting to backup Coub videos for best of year: ", year) 36 | log.Println("Output directory:", *directory) 37 | // append a / to the end of the directory if it doesn't exist 38 | if (*directory)[len(*directory)-1] != '/' { 39 | *directory += "/" 40 | } 41 | 42 | err = RunBestOf(*directory, strconv.Itoa(year)) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | } else if *community != "" { 47 | log.Println("Attempting to backup Coub videos for community: ", *community) 48 | log.Println("Note: There is a maximum of 500 pages that can be downloaded, of 10 coubs per page.") 49 | log.Println("Output directory:", *directory) 50 | 51 | if *pages > 500 { 52 | *pages = 500 53 | } else if *pages < 1 { 54 | *pages = 1 55 | } 56 | 57 | err := RunCommunity(*directory, *community, *pages) 58 | if err != nil { 59 | log.Fatal(err) 60 | } 61 | } else if *featured || *day { 62 | log.Println("Attempting to backup Coub videos for featured or day") 63 | log.Println("Output directory:", *directory) 64 | 65 | if *featured { 66 | log.Println("Attempting to backup Coub videos for featured") 67 | err := RunFeatured(*directory) 68 | if err != nil { 69 | log.Fatal(err) 70 | } 71 | } else if *day { 72 | log.Println("Attempting to backup Coub videos for day") 73 | err := RunDay(*directory) 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | } 78 | } else { 79 | 80 | // user string is the first non-flag argument 81 | user := flag.Arg(0) // get the first non-flag argument 82 | 83 | log.Println("Attempting to backup Coub videos for user:", user) 84 | log.Println("Output directory:", *directory) 85 | 86 | err := Run(user, *directory) 87 | if err != nil { 88 | log.Fatal(err) 89 | } 90 | } 91 | 92 | } 93 | 94 | func RunCommunity(dir string, community string, pages int) (err error) { 95 | switch community { 96 | case "animals-pets": 97 | break 98 | case "mashup": 99 | break 100 | case "music": 101 | break 102 | case "blogging": 103 | break 104 | case "standup-jokes": 105 | break 106 | case "movies": 107 | break 108 | case "anime": 109 | break 110 | case "gaming": 111 | break 112 | case "cartoons": 113 | break 114 | case "art": 115 | break 116 | case "live-pictures": 117 | break 118 | case "news": 119 | break 120 | case "sports": 121 | break 122 | case "science-technology": 123 | break 124 | case "food-kitchen": 125 | break 126 | case "celebrity": 127 | break 128 | case "nature-travel": 129 | break 130 | case "fashion": 131 | break 132 | case "dance": 133 | break 134 | case "cars": 135 | break 136 | case "memes": 137 | break 138 | case "nsfw": 139 | break 140 | default: 141 | log.Println("Unrecognized community, please chose from one of the following:") 142 | log.Println("animals-pets, mashup, music, blogging, standup-jokes, movies, anime, gaming, cartoons, art, live-pictures, news, sports, science-technology, food-kitchen, celebrity, nature-travel, fashion, dance, cars, memes, nsfw") 143 | } 144 | 145 | log.Println("Creating directory if not exists:", dir) 146 | err = DirectorySetup(community, dir) 147 | if err != nil { 148 | return err 149 | } 150 | 151 | outputDir := dir + community + "/" 152 | 153 | // if file doesn't exist 154 | if !FileExists(outputDir + community + ".json") { 155 | err = GenerateCommunityInfoFile(outputDir, community, pages) 156 | if err != nil { 157 | return err 158 | } 159 | } else { 160 | log.Println("Coub info file already exists for community: ", community, ", skipping generation.") 161 | log.Println("Proceeding to downloads for ", community) 162 | } 163 | 164 | log.Println("Beginning downloads in 5 seconds...") 165 | time.Sleep(5 * time.Second) 166 | 167 | err = ReadCoub(outputDir, community) 168 | if err != nil { 169 | return err 170 | } 171 | return nil 172 | } 173 | 174 | func RunBestOf(dir string, year string) (err error) { 175 | log.Println("Creating directory if not exists:", dir) 176 | err = DirectorySetup("bestof-"+year, dir) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | outputDir := dir + "bestof-" + year + "/" 182 | 183 | // if file doesn't exist 184 | if !FileExists(outputDir + "bestof.json") { 185 | err = GenerateBestOfInfoFile(outputDir, year) 186 | if err != nil { 187 | return err 188 | } 189 | } else { 190 | log.Println("Coub info file already exists for bestof ", year, ", skipping generation.") 191 | log.Println("Proceeding to downloads for bestof ", year) 192 | } 193 | 194 | log.Println("Beginning downloads in 5 seconds...") 195 | time.Sleep(5 * time.Second) 196 | 197 | err = ReadCoub(outputDir, "bestof") 198 | if err != nil { 199 | return err 200 | } 201 | return nil 202 | } 203 | 204 | func Run(user string, dir string) (err error) { 205 | log.Println("Creating directory if not exists:", dir+user) 206 | err = DirectorySetup(user, dir) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | outputDir := dir + user + "/" 212 | 213 | // if file doesn't exist 214 | if !FileExists(outputDir + user + ".json") { 215 | err = GenerateInfoFile(outputDir, user) 216 | if err != nil { 217 | return err 218 | } 219 | } else { 220 | log.Println("Coub info file already exists for user:", user, ", skipping generation.") 221 | log.Println("Proceeding to downloads for ", user) 222 | } 223 | 224 | log.Println("Beginning downloads in 5 seconds...") 225 | time.Sleep(5 * time.Second) 226 | 227 | err = ReadCoub(outputDir, user) 228 | if err != nil { 229 | return err 230 | } 231 | 232 | return nil 233 | } 234 | 235 | func RunFeatured(dir string) (err error) { 236 | log.Println("Creating directory if not exists:", dir+"featured") 237 | err = DirectorySetup("featured", dir) 238 | if err != nil { 239 | return err 240 | } 241 | 242 | outputDir := dir + "featured" + "/" 243 | 244 | // if file exists, delete it 245 | if FileExists(outputDir + "featured.json") { 246 | err = os.Remove(outputDir + "featured.json") 247 | if err != nil { 248 | return err 249 | } 250 | } 251 | err = GenerateFeaturedInfoFile(outputDir) 252 | if err != nil { 253 | return err 254 | } 255 | 256 | log.Println("Beginning downloads in 5 seconds...") 257 | time.Sleep(5 * time.Second) 258 | 259 | err = ReadCoub(outputDir, "featured") 260 | if err != nil { 261 | return err 262 | } 263 | 264 | return nil 265 | } 266 | 267 | func RunDay(dir string) (err error) { 268 | log.Println("Creating directory if not exists:", dir+"coub-of-the-day") 269 | err = DirectorySetup("coub-of-the-day", dir) 270 | if err != nil { 271 | return err 272 | } 273 | 274 | outputDir := dir + "coub-of-the-day" + "/" 275 | 276 | // if file exists, delete it 277 | if FileExists(outputDir + "coub-of-the-day.json") { 278 | err = os.Remove(outputDir + "coub-of-the-day.json") 279 | if err != nil { 280 | return err 281 | } 282 | } 283 | err = GenerateCoubOfDayInfoFile(outputDir) 284 | if err != nil { 285 | return err 286 | } 287 | 288 | log.Println("Beginning downloads in 5 seconds...") 289 | time.Sleep(5 * time.Second) 290 | 291 | err = ReadCoub(outputDir, "coub-of-the-day") 292 | if err != nil { 293 | return err 294 | } 295 | 296 | return nil 297 | } 298 | -------------------------------------------------------------------------------- /coub-dl.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const DownloadInterval = time.Millisecond * 100 16 | 17 | // ReadCoub Accepts a Coub struct 18 | // It generates a directory for the coub, creates the info file for it 19 | // And finally downloads all data for it 20 | func ReadCoub(rootdir string, user string) (err error) { 21 | 22 | // rootdir should be the path to the user directory 23 | // From there we will create our sub directories 24 | 25 | coubs, err := GetNonRecoubs(rootdir, user) 26 | if err != nil { 27 | return err 28 | } 29 | log.Println("Total Coubs to process: " + strconv.Itoa(len(coubs))) 30 | 31 | var wg sync.WaitGroup 32 | for i, coub := range coubs { 33 | coub.Title = strings.TrimSpace(coub.Title) 34 | 35 | log.Println("Processing Coub: " + coub.Title) 36 | // Create the directory for the coub 37 | outdir, err := CreateCoubDir(rootdir, coub) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | // Create the info file for the coub 43 | err = CreateCoubInfoFiles(outdir, coub) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | // Download all data for the coub 49 | wg.Add(1) 50 | go func(coubID int) { 51 | log.Println("Downloading Coub: " + coubs[coubID].Title) 52 | err = DownloadCoubData(&wg, outdir, coubs[coubID]) 53 | if err != nil { 54 | log.Println("Error downloading coub: "+coubs[coubID].Title, err) 55 | } 56 | }(i) 57 | time.Sleep(time.Second * 1) 58 | 59 | // every 5 coubs, wait for the goroutines to finish 60 | if i%5 == 0 { 61 | wg.Wait() 62 | } 63 | } 64 | wg.Wait() 65 | log.Println("All found coubs downloaded") 66 | return nil 67 | } 68 | 69 | func GetNonRecoubs(dir string, user string) (coubs []Coub, err error) { 70 | // Open the json file for the user 71 | jsonFile, err := os.Open(dir + "/" + user + ".json") 72 | if err != nil { 73 | return nil, err 74 | } 75 | defer jsonFile.Close() 76 | 77 | var tmpCoubs []Coub 78 | 79 | // Unmarshal the json file into a Coubs struct 80 | err = json.NewDecoder(jsonFile).Decode(&tmpCoubs) 81 | //log.Print(len(tmpCoubs)) 82 | for _, coub := range tmpCoubs { 83 | if coub.Type != "Coub::Recoub" { 84 | coubs = append(coubs, coub) 85 | } 86 | } 87 | return coubs, nil 88 | } 89 | 90 | func CreateCoubInfoFiles(dir string, coub Coub) (err error) { 91 | // First we dump the coub struct into a json file 92 | outputFile, _ := json.MarshalIndent(coub, "", " ") 93 | err = ioutil.WriteFile(dir+"/metadata.json", outputFile, 0644) 94 | if err != nil { 95 | return err 96 | } 97 | 98 | infoFile, err := os.Create(dir + "/info.txt") 99 | if err != nil { 100 | fmt.Println("Unable to open file: %s", err) 101 | } 102 | 103 | _, err = infoFile.WriteString("Title: " + coub.Title + "\n") 104 | if err != nil { 105 | return err 106 | } 107 | _, err = infoFile.WriteString("Created At: " + coub.CreatedAt.String() + "\n") 108 | if err != nil { 109 | return err 110 | } 111 | 112 | _, err = infoFile.WriteString("Duration: " + fmt.Sprintf("%.2f", coub.Duration) + "\n") 113 | if err != nil { 114 | return err 115 | } 116 | 117 | _, err = infoFile.WriteString("Views: " + strconv.Itoa(coub.ViewsCount) + "\n") 118 | if err != nil { 119 | return err 120 | } 121 | 122 | _, err = infoFile.WriteString("Recoubs: " + strconv.Itoa(coub.RecoubsCount) + "\n") 123 | if err != nil { 124 | return err 125 | } 126 | 127 | _, err = infoFile.WriteString("Source: " + fmt.Sprintf("%v", coub.ExternalDownload) + "\n") 128 | 129 | _, err = infoFile.WriteString("Tags: ") 130 | 131 | for i, tag := range coub.Tags { 132 | if i == len(coub.Tags)-1 { 133 | _, err = infoFile.WriteString(tag.Title + "\n") 134 | } else { 135 | _, err = infoFile.WriteString(tag.Title + ", ") 136 | } 137 | if err != nil { 138 | return err 139 | } 140 | } 141 | 142 | err = infoFile.Close() 143 | if err != nil { 144 | return err 145 | } 146 | 147 | return nil 148 | } 149 | 150 | func DownloadCoubData(PoolWG *sync.WaitGroup, rootdir string, coub Coub) (err error) { 151 | defer PoolWG.Done() 152 | 153 | var wg sync.WaitGroup 154 | 155 | wg.Add(1) 156 | go func() { 157 | log.Print("Downloading Frames for Coub: " + coub.Title) 158 | err = DownloadFirstFrameVersions(&wg, rootdir, coub) 159 | if err != nil { 160 | log.Println("Error Downloading First Frame Versions: " + err.Error()) 161 | } 162 | }() 163 | 164 | wg.Add(1) 165 | go func() { 166 | log.Print("Downloading Images for Coub: " + coub.Title) 167 | err = DownloadImageVersions(&wg, rootdir, coub) 168 | if err != nil { 169 | log.Println("Error Downloading Image Versions: " + err.Error()) 170 | } 171 | }() 172 | 173 | wg.Add(1) 174 | go func() { 175 | log.Print("Downloading Media Files for Coub: " + coub.Title) 176 | err = DownloadFileVersions(&wg, rootdir, coub) 177 | if err != nil { 178 | log.Println("Error Downloading File Versions: " + err.Error()) 179 | } 180 | }() 181 | 182 | wg.Wait() 183 | log.Println("Finished Downloading Coub: " + coub.Title) 184 | return nil 185 | } 186 | 187 | func DownloadFileVersions(wg *sync.WaitGroup, filepath string, coub Coub) (err error) { 188 | defer wg.Done() 189 | 190 | url := coub.FileVersions.HTML5.Video.Med.URL 191 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 192 | if err != nil { 193 | log.Println("Error downloading Medium Quality HTML5 Video for: " + coub.Title + ": " + err.Error()) 194 | } 195 | time.Sleep(DownloadInterval) 196 | 197 | url = coub.FileVersions.HTML5.Video.High.URL 198 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 199 | if err != nil { 200 | log.Println("Error downloading High Quality HTML5 Video for: " + coub.Title + ": " + err.Error()) 201 | } 202 | time.Sleep(DownloadInterval) 203 | 204 | url = coub.FileVersions.HTML5.Video.Higher.URL 205 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 206 | if err != nil { 207 | log.Println("Error downloading Higher Quality HTML5 Video for: " + coub.Title + ": " + err.Error()) 208 | } 209 | time.Sleep(DownloadInterval) 210 | 211 | url = coub.FileVersions.HTML5.Audio.High.URL 212 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 213 | if err != nil { 214 | log.Println("Error downloading Higher Quality HTML5 Audio for: " + coub.Title + ": " + err.Error()) 215 | } 216 | time.Sleep(DownloadInterval) 217 | 218 | url = coub.FileVersions.HTML5.Audio.Med.URL 219 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 220 | if err != nil { 221 | log.Println("Error downloading Medium Quality HTML5 Audio for: " + coub.Title + ": " + err.Error()) 222 | } 223 | time.Sleep(DownloadInterval) 224 | 225 | // We do not download mobile versions, because they are the same as the medium quality HTML5 versions 226 | /* 227 | url = coub.FileVersions.Mobile.Video 228 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 229 | if err != nil { 230 | log.Println("Error downloading Mobile Video for" + coub.Title + ": " + err.Error()) 231 | } 232 | time.Sleep(DownloadInterval) 233 | 234 | url = coub.FileVersions.Mobile.Audio[0] 235 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 236 | if err != nil { 237 | log.Println("Error downloading Mobile Audio for" + coub.Title + ": " + err.Error()) 238 | } 239 | time.Sleep(DownloadInterval) 240 | */ 241 | 242 | url = coub.FileVersions.Share.Default 243 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 244 | if err != nil { 245 | log.Println("Error downloading Default Share File for: " + coub.Title + ": " + err.Error()) 246 | } 247 | 248 | url = coub.FileVersions.Share.Default 249 | err = DownloadFile(filepath+"/"+strconv.Itoa(coub.ID)+".mp4", url) 250 | if err != nil { 251 | log.Println("Error downloading (renamed) Default Share File for: " + coub.Title + ": " + err.Error()) 252 | } 253 | 254 | return nil 255 | } 256 | 257 | func DownloadImageVersions(wg *sync.WaitGroup, filepath string, coub Coub) (err error) { 258 | defer wg.Done() 259 | 260 | template := coub.ImageVersions.Template 261 | for _, version := range coub.ImageVersions.Versions { 262 | url := strings.Replace(template, "%{version}", version, -1) 263 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 264 | if err != nil { 265 | return err 266 | } 267 | time.Sleep(time.Second * 1) 268 | } 269 | return nil 270 | } 271 | 272 | func DownloadFirstFrameVersions(wg *sync.WaitGroup, filepath string, coub Coub) (err error) { 273 | defer wg.Done() 274 | 275 | template := coub.FirstFrameVersions.Template 276 | for _, version := range coub.FirstFrameVersions.Versions { 277 | url := strings.Replace(template, "%{version}", version, -1) 278 | err = DownloadFile(filepath+"/"+FileNameFromURL(url), url) 279 | if err != nil { 280 | return err 281 | } 282 | time.Sleep(time.Second * 1) 283 | } 284 | return nil 285 | } 286 | -------------------------------------------------------------------------------- /coub_type.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | type CoubInfo struct { 6 | Page int `json:"page"` 7 | PerPage int `json:"per_page"` 8 | TotalPages int `json:"total_pages"` 9 | Coubs []Coub `json:"coubs"` 10 | } 11 | 12 | type Coub struct { 13 | Flag bool `json:"flag"` 14 | Abuses interface{} `json:"abuses"` 15 | RecoubsByUsersChannels []interface{} `json:"recoubs_by_users_channels"` 16 | Favourite bool `json:"favourite"` 17 | PromotedID interface{} `json:"promoted_id"` 18 | Recoub bool `json:"recoub"` 19 | Like bool `json:"like"` 20 | Dislike bool `json:"dislike"` 21 | Reaction interface{} `json:"reaction"` 22 | InMyBest2015 bool `json:"in_my_best2015"` 23 | ID int `json:"id"` 24 | Type string `json:"type"` 25 | Permalink string `json:"permalink"` 26 | Title string `json:"title"` 27 | VisibilityType string `json:"visibility_type"` 28 | OriginalVisibilityType string `json:"original_visibility_type"` 29 | ChannelID int `json:"channel_id"` 30 | CreatedAt time.Time `json:"created_at"` 31 | UpdatedAt time.Time `json:"updated_at"` 32 | IsDone bool `json:"is_done"` 33 | ViewsCount int `json:"views_count"` 34 | Cotd interface{} `json:"cotd"` 35 | CotdAt interface{} `json:"cotd_at"` 36 | VisibleOnExploreRoot bool `json:"visible_on_explore_root"` 37 | VisibleOnExplore bool `json:"visible_on_explore"` 38 | Featured bool `json:"featured"` 39 | Published bool `json:"published"` 40 | PublishedAt time.Time `json:"published_at"` 41 | Reversed bool `json:"reversed"` 42 | FromEditorV2 bool `json:"from_editor_v2"` 43 | IsEditable bool `json:"is_editable"` 44 | OriginalSound bool `json:"original_sound"` 45 | HasSound bool `json:"has_sound"` 46 | RecoubTo interface{} `json:"recoub_to"` 47 | FileVersions struct { 48 | HTML5 struct { 49 | Video struct { 50 | Higher struct { 51 | URL string `json:"url"` 52 | Size int `json:"size"` 53 | } `json:"higher"` 54 | High struct { 55 | URL string `json:"url"` 56 | Size int `json:"size"` 57 | } `json:"high"` 58 | Med struct { 59 | URL string `json:"url"` 60 | Size int `json:"size"` 61 | } `json:"med"` 62 | } `json:"video"` 63 | Audio struct { 64 | High struct { 65 | URL string `json:"url"` 66 | Size int `json:"size"` 67 | } `json:"high"` 68 | Med struct { 69 | URL string `json:"url"` 70 | Size int `json:"size"` 71 | } `json:"med"` 72 | SampleDuration float64 `json:"sample_duration"` 73 | } `json:"audio"` 74 | } `json:"html5"` 75 | Mobile struct { 76 | Video string `json:"video"` 77 | Audio []string `json:"audio"` 78 | } `json:"mobile"` 79 | Share struct { 80 | Default string `json:"default"` 81 | } `json:"share"` 82 | } `json:"file_versions"` 83 | AudioVersions struct { 84 | } `json:"audio_versions"` 85 | ImageVersions struct { 86 | Template string `json:"template"` 87 | Versions []string `json:"versions"` 88 | } `json:"image_versions"` 89 | FirstFrameVersions struct { 90 | Template string `json:"template"` 91 | Versions []string `json:"versions"` 92 | } `json:"first_frame_versions"` 93 | Dimensions struct { 94 | Big []int `json:"big"` 95 | Med []int `json:"med"` 96 | } `json:"dimensions"` 97 | SiteWH []int `json:"site_w_h"` 98 | PageWH []int `json:"page_w_h"` 99 | SiteWHSmall []int `json:"site_w_h_small"` 100 | Size []int `json:"size"` 101 | AgeRestricted bool `json:"age_restricted"` 102 | AgeRestrictedByAdmin bool `json:"age_restricted_by_admin"` 103 | NotSafeForWork bool `json:"not_safe_for_work"` 104 | AllowReuse bool `json:"allow_reuse"` 105 | DontCrop bool `json:"dont_crop"` 106 | Banned bool `json:"banned"` 107 | GlobalSafe bool `json:"global_safe"` 108 | AudioFileURL interface{} `json:"audio_file_url"` 109 | ExternalDownload interface{} `json:"external_download"` 110 | Application interface{} `json:"application"` 111 | Channel struct { 112 | ID int `json:"id"` 113 | Permalink string `json:"permalink"` 114 | Title string `json:"title"` 115 | Description string `json:"description"` 116 | IFollowHim bool `json:"i_follow_him"` 117 | FollowsByUsersChannels []interface{} `json:"follows_by_users_channels"` 118 | FollowersCount int `json:"followers_count"` 119 | FollowingCount int `json:"following_count"` 120 | AvatarVersions struct { 121 | Template string `json:"template"` 122 | Versions []string `json:"versions"` 123 | } `json:"avatar_versions"` 124 | } `json:"channel"` 125 | File interface{} `json:"file"` 126 | Picture string `json:"picture"` 127 | TimelinePicture string `json:"timeline_picture"` 128 | SmallPicture string `json:"small_picture"` 129 | SharingPicture interface{} `json:"sharing_picture"` 130 | PercentDone int `json:"percent_done"` 131 | Tags []struct { 132 | ID int `json:"id"` 133 | Title string `json:"title"` 134 | Value string `json:"value"` 135 | } `json:"tags"` 136 | Categories []struct { 137 | ID int `json:"id"` 138 | Title string `json:"title"` 139 | Permalink string `json:"permalink"` 140 | SubscriptionsCount int `json:"subscriptions_count"` 141 | BigImageURL string `json:"big_image_url"` 142 | SmallImageURL string `json:"small_image_url"` 143 | MedImageURL string `json:"med_image_url"` 144 | Visible bool `json:"visible"` 145 | } `json:"categories"` 146 | Communities []struct { 147 | ID int `json:"id"` 148 | Title string `json:"title"` 149 | Permalink string `json:"permalink"` 150 | SubscriptionsCount int `json:"subscriptions_count"` 151 | BigImageURL string `json:"big_image_url"` 152 | SmallImageURL string `json:"small_image_url"` 153 | MedImageURL string `json:"med_image_url"` 154 | ISubscribed bool `json:"i_subscribed"` 155 | CommunityNotificationsEnabled bool `json:"community_notifications_enabled"` 156 | Description struct { 157 | ID int `json:"id"` 158 | Description string `json:"description"` 159 | Rules []interface{} `json:"rules"` 160 | DescriptionHTML string `json:"description_html"` 161 | RulesHTML []interface{} `json:"rules_html"` 162 | } `json:"description"` 163 | } `json:"communities"` 164 | RecoubsCount int `json:"recoubs_count"` 165 | RemixesCount int `json:"remixes_count"` 166 | LikesCount int `json:"likes_count"` 167 | DislikesCount int `json:"dislikes_count"` 168 | //RawVideoID int `json:"raw_video_id"` 169 | UploadedByIosApp bool `json:"uploaded_by_ios_app"` 170 | UploadedByAndroidApp bool `json:"uploaded_by_android_app"` 171 | MediaBlocks struct { 172 | //UploadedRawVideos []interface{} `json:"uploaded_raw_videos"` 173 | ExternalRawVideos []struct { 174 | ID int `json:"id"` 175 | Title string `json:"title"` 176 | URL string `json:"url"` 177 | Image string `json:"image"` 178 | ImageRetina string `json:"image_retina"` 179 | Meta struct { 180 | Service string `json:"service"` 181 | Duration string `json:"duration"` 182 | } `json:"meta"` 183 | Duration float64 `json:"duration"` 184 | //RawVideoID int `json:"raw_video_id"` 185 | HasEmbed bool `json:"has_embed"` 186 | } `json:"external_raw_videos"` 187 | RemixedFromCoubs []interface{} `json:"remixed_from_coubs"` 188 | ExternalVideo struct { 189 | ID int `json:"id"` 190 | Title string `json:"title"` 191 | URL string `json:"url"` 192 | Image string `json:"image"` 193 | ImageRetina string `json:"image_retina"` 194 | Meta struct { 195 | Service string `json:"service"` 196 | Duration string `json:"duration"` 197 | } `json:"meta"` 198 | Duration float64 `json:"duration"` 199 | //RawVideoID int `json:"raw_video_id"` 200 | HasEmbed bool `json:"has_embed"` 201 | } `json:"external_video"` 202 | } `json:"media_blocks"` 203 | RawVideoThumbnailURL string `json:"raw_video_thumbnail_url"` 204 | RawVideoTitle string `json:"raw_video_title"` 205 | VideoBlockBanned bool `json:"video_block_banned"` 206 | Duration float64 `json:"duration"` 207 | PromoWinner bool `json:"promo_winner"` 208 | PromoWinnerRecoubers interface{} `json:"promo_winner_recoubers"` 209 | EditorialInfo struct { 210 | } `json:"editorial_info"` 211 | PromoHint interface{} `json:"promo_hint"` 212 | BeelineBest2014 interface{} `json:"beeline_best_2014"` 213 | FromWebEditor bool `json:"from_web_editor"` 214 | NormalizeSound bool `json:"normalize_sound"` 215 | NormalizeChangeAllowed bool `json:"normalize_change_allowed"` 216 | Best2015Addable bool `json:"best2015_addable"` 217 | AhmadPromo interface{} `json:"ahmad_promo"` 218 | PromoData interface{} `json:"promo_data"` 219 | AudioCopyrightClaim interface{} `json:"audio_copyright_claim"` 220 | AdsDisabled bool `json:"ads_disabled"` 221 | IsSafeForAds bool `json:"is_safe_for_ads"` 222 | MegafonContents []interface{} `json:"megafon_contents"` 223 | PositionOnPage int `json:"position_on_page"` 224 | } 225 | 226 | type ExternalDownload struct { 227 | Type string `json:"type"` 228 | ServiceName string `json:"service_name"` 229 | URL string `json:"url"` 230 | HasEmbed bool `json:"has_embed"` 231 | } 232 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | const MaxRetries = 3 16 | 17 | // RetrieveProfile a complete array of CoubInfo structs for a given profile 18 | func RetrieveProfile(username string) (coubs []Coub, err error) { 19 | // Get first page of profile info with an http.Get request 20 | log.Print("Retrieving first page to parse channel info") 21 | 22 | timelineFirstPage := "https://coub.com/api/v2/timeline/channel/" + username + "?page=1&per_page=25" 23 | res, err := http.Get(timelineFirstPage) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | b, err := io.ReadAll(res.Body) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | var Coubs []Coub 34 | 35 | var CoubInfoPage CoubInfo 36 | err = json.Unmarshal(b, &CoubInfoPage) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 42 | 43 | log.Print("Pages to parse: ", CoubInfoPage.TotalPages) 44 | 45 | // Get all pages of profile info with an http.Get request 46 | for i := 2; i <= CoubInfoPage.TotalPages; i++ { 47 | log.Print("Retrieving page ", i) 48 | 49 | timelinePage := "https://coub.com/api/v2/timeline/channel/" + username + "?page=" + strconv.Itoa(i) + "&per_page=25" 50 | res, err := http.Get(timelinePage) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | b, err := io.ReadAll(res.Body) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | var CoubInfoPage CoubInfo 61 | err = json.Unmarshal(b, &CoubInfoPage) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 67 | time.Sleep(DownloadInterval) 68 | } 69 | 70 | return Coubs, nil 71 | } 72 | 73 | func GenerateFeaturedInfoFile(outputDir string) (err error) { 74 | log.Print("Getting Coub featured data for today") 75 | coubs, err := RetrieveFeatured() 76 | if err != nil { 77 | return err 78 | } 79 | 80 | outputFile, _ := json.MarshalIndent(coubs, "", " ") 81 | _ = ioutil.WriteFile(outputDir+"featured.json", outputFile, 0644) 82 | 83 | return nil 84 | } 85 | 86 | func GenerateCoubOfDayInfoFile(outputDir string) (err error) { 87 | log.Print("Getting Coub of The Day data") 88 | coubs, err := RetrieveCoubOfDay() 89 | if err != nil { 90 | return err 91 | } 92 | 93 | outputFile, _ := json.MarshalIndent(coubs, "", " ") 94 | _ = ioutil.WriteFile(outputDir+"coub-of-day.json", outputFile, 0644) 95 | 96 | return nil 97 | } 98 | 99 | func GenerateCommunityInfoFile(outputDir string, community string, pages int) (err error) { 100 | log.Print("Getting Coub community data for ", community) 101 | coubs, err := RetrieveCommunity(community, pages) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | outputFile, _ := json.MarshalIndent(coubs, "", " ") 107 | _ = ioutil.WriteFile(outputDir+community+".json", outputFile, 0644) 108 | 109 | return nil 110 | } 111 | 112 | func GenerateBestOfInfoFile(outputDir string, year string) (err error) { 113 | log.Print("Getting Coub best of data") 114 | coubs, err := RetrieveBestOf(year) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | outputFile, _ := json.MarshalIndent(coubs, "", " ") 120 | _ = ioutil.WriteFile(outputDir+"bestof.json", outputFile, 0644) 121 | 122 | return nil 123 | } 124 | 125 | func GenerateInfoFile(outputDir string, user string) (err error) { 126 | log.Print("Getting Coub user data for: ", user) 127 | coubs, err := RetrieveProfile(user) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | outputFile, _ := json.MarshalIndent(coubs, "", " ") 133 | _ = ioutil.WriteFile(outputDir+user+".json", outputFile, 0644) 134 | 135 | return nil 136 | } 137 | 138 | func DownloadFile(filepath string, url string) (err error) { 139 | if url == "" { 140 | return nil 141 | } 142 | // Check if a file already exists at filepath 143 | if _, err := os.Stat(filepath); err == nil { 144 | //log.Print("File already exists at ", filepath) 145 | return nil 146 | } 147 | 148 | // Get the data and retry if it fails 149 | var resp *http.Response 150 | for i := 0; i < MaxRetries; i++ { 151 | resp, err = http.Get(url) 152 | if err != nil { 153 | log.Print("Failed to download file: ", err, "retrying...") 154 | if i == MaxRetries-1 { 155 | return err 156 | } 157 | continue 158 | } 159 | break 160 | } 161 | defer resp.Body.Close() 162 | 163 | // Create the file 164 | out, err := os.Create(filepath) 165 | if err != nil { 166 | return err 167 | } 168 | defer out.Close() 169 | 170 | // Write the body to file 171 | _, err = io.Copy(out, resp.Body) 172 | return err 173 | } 174 | 175 | func FileNameFromURL(url string) (filename string) { 176 | filename = url[strings.LastIndex(url, "/")+1:] 177 | return 178 | } 179 | 180 | // RetrieveBestOf a complete array of CoubInfo structs for the best of category 181 | func RetrieveBestOf(year string) (coubs []Coub, err error) { 182 | // Get first page of profile info with an http.Get request 183 | oldAPI := false 184 | log.Print("Retrieving first page to parse best of info") 185 | var yearID string 186 | switch year { 187 | case "2021": 188 | yearID = "62" 189 | case "2020": 190 | yearID = "53" 191 | case "2019": 192 | yearID = "14" 193 | case "2018": 194 | yearID = "2018" 195 | oldAPI = true 196 | case "2017": 197 | yearID = "2017" 198 | oldAPI = true 199 | case "2016": 200 | yearID = "2016" 201 | oldAPI = true 202 | case "2015": 203 | yearID = "2015" 204 | oldAPI = true 205 | case "2014": 206 | yearID = "2014" 207 | oldAPI = true 208 | case "2013": 209 | yearID = "2013" 210 | oldAPI = true 211 | case "2012": 212 | yearID = "2012" 213 | oldAPI = true 214 | } 215 | var timelineFirstPage string 216 | if !oldAPI { 217 | timelineFirstPage = "https://coub.com/api/v2/best/" + yearID + "/coubs?type=coubs&page=1" 218 | } else { 219 | timelineFirstPage = "https://coub.com/api/v2/best/" + yearID + "/likes?page=1" 220 | } 221 | res, err := http.Get(timelineFirstPage) 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | b, err := io.ReadAll(res.Body) 227 | if err != nil { 228 | return nil, err 229 | } 230 | 231 | var Coubs []Coub 232 | 233 | var CoubInfoPage CoubInfo 234 | err = json.Unmarshal(b, &CoubInfoPage) 235 | if err != nil { 236 | return nil, err 237 | } 238 | 239 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 240 | 241 | log.Print("Pages to parse: ", CoubInfoPage.TotalPages) 242 | 243 | // Get all pages of profile info with an http.Get request 244 | for i := 2; i <= CoubInfoPage.TotalPages; i++ { 245 | log.Print("Retrieving page ", i) 246 | 247 | var timelinePage string 248 | if !oldAPI { 249 | timelinePage = "https://coub.com/api/v2/best/" + yearID + "/coubs?type=coubs&page=" + strconv.Itoa(i) 250 | } else { 251 | timelinePage = "https://coub.com/api/v2/best/" + yearID + "/likes?page=" + strconv.Itoa(i) 252 | } 253 | res, err := http.Get(timelinePage) 254 | if err != nil { 255 | return nil, err 256 | } 257 | 258 | b, err := io.ReadAll(res.Body) 259 | if err != nil { 260 | return nil, err 261 | } 262 | 263 | var CoubInfoPage CoubInfo 264 | err = json.Unmarshal(b, &CoubInfoPage) 265 | if err != nil { 266 | return nil, err 267 | } 268 | 269 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 270 | } 271 | 272 | return Coubs, nil 273 | } 274 | 275 | // RetrieveCommunity a complete array of CoubInfo structs for a given profile 276 | func RetrieveCommunity(community string, pages int) (coubs []Coub, err error) { 277 | // Get first page of profile info with an http.Get request 278 | log.Print("Retrieving first page to parse category info") 279 | 280 | timelineFirstPage := "https://coub.com/api/v2/timeline/community/" + community + "/daily?page=1" 281 | res, err := http.Get(timelineFirstPage) 282 | if err != nil { 283 | return nil, err 284 | } 285 | 286 | b, err := io.ReadAll(res.Body) 287 | if err != nil { 288 | return nil, err 289 | } 290 | 291 | var Coubs []Coub 292 | 293 | var CoubInfoPage CoubInfo 294 | err = json.Unmarshal(b, &CoubInfoPage) 295 | if err != nil { 296 | return nil, err 297 | } 298 | 299 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 300 | 301 | log.Print("Pages to parse: ", pages) 302 | 303 | // Get all pages of profile info with an http.Get request 304 | for i := 2; i <= CoubInfoPage.TotalPages && i <= pages; i++ { 305 | log.Print("Retrieving page ", i) 306 | 307 | timelinePage := "https://coub.com/api/v2/timeline/community/" + community + "/daily?page=" + strconv.Itoa(i) 308 | res, err := http.Get(timelinePage) 309 | if err != nil { 310 | return nil, err 311 | } 312 | 313 | b, err := io.ReadAll(res.Body) 314 | if err != nil { 315 | return nil, err 316 | } 317 | 318 | var CoubInfoPage CoubInfo 319 | err = json.Unmarshal(b, &CoubInfoPage) 320 | if err != nil { 321 | return nil, err 322 | } 323 | 324 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 325 | } 326 | 327 | return Coubs, nil 328 | } 329 | 330 | // RetrieveFeatured a complete array of CoubInfo structs for a given profile 331 | func RetrieveFeatured() (coubs []Coub, err error) { 332 | // Get first page of profile info with an http.Get request 333 | log.Print("Retrieving first page to parse featured coubs info") 334 | 335 | timelineFirstPage := "https://coub.com/api/v2/timeline/explore?page=1" 336 | res, err := http.Get(timelineFirstPage) 337 | if err != nil { 338 | return nil, err 339 | } 340 | 341 | b, err := io.ReadAll(res.Body) 342 | if err != nil { 343 | return nil, err 344 | } 345 | 346 | var Coubs []Coub 347 | 348 | var CoubInfoPage CoubInfo 349 | err = json.Unmarshal(b, &CoubInfoPage) 350 | if err != nil { 351 | return nil, err 352 | } 353 | 354 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 355 | 356 | log.Print("Pages to parse: ", CoubInfoPage.TotalPages) 357 | 358 | // Get all pages of profile info with an http.Get request 359 | for i := 2; i <= CoubInfoPage.TotalPages; i++ { 360 | log.Print("Retrieving page ", i) 361 | 362 | timelinePage := "https://coub.com/api/v2/timeline/explore?page=" + strconv.Itoa(i) 363 | res, err := http.Get(timelinePage) 364 | if err != nil { 365 | return nil, err 366 | } 367 | 368 | b, err := io.ReadAll(res.Body) 369 | if err != nil { 370 | return nil, err 371 | } 372 | 373 | var CoubInfoPage CoubInfo 374 | err = json.Unmarshal(b, &CoubInfoPage) 375 | if err != nil { 376 | return nil, err 377 | } 378 | 379 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 380 | } 381 | 382 | return Coubs, nil 383 | } 384 | 385 | // RetrieveFeatured a complete array of CoubInfo structs for a given profile 386 | func RetrieveCoubOfDay() (coubs []Coub, err error) { 387 | // Get first page of profile info with an http.Get request 388 | log.Print("Retrieving first page to parse coub of the day info") 389 | 390 | timelineFirstPage := "https://coub.com/api/v2/timeline/explore/coub_of_the_day?page=1" 391 | res, err := http.Get(timelineFirstPage) 392 | if err != nil { 393 | return nil, err 394 | } 395 | 396 | b, err := io.ReadAll(res.Body) 397 | if err != nil { 398 | return nil, err 399 | } 400 | 401 | var Coubs []Coub 402 | 403 | var CoubInfoPage CoubInfo 404 | err = json.Unmarshal(b, &CoubInfoPage) 405 | if err != nil { 406 | return nil, err 407 | } 408 | 409 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 410 | 411 | log.Print("Pages to parse: ", CoubInfoPage.TotalPages) 412 | 413 | // Get all pages of profile info with an http.Get request 414 | for i := 2; i <= CoubInfoPage.TotalPages; i++ { 415 | log.Print("Retrieving page ", i) 416 | 417 | timelinePage := "https://coub.com/api/v2/timeline/explore/coub_of_the_day?page=" + strconv.Itoa(i) 418 | res, err := http.Get(timelinePage) 419 | if err != nil { 420 | return nil, err 421 | } 422 | 423 | b, err := io.ReadAll(res.Body) 424 | if err != nil { 425 | return nil, err 426 | } 427 | 428 | var CoubInfoPage CoubInfo 429 | err = json.Unmarshal(b, &CoubInfoPage) 430 | if err != nil { 431 | return nil, err 432 | } 433 | 434 | Coubs = append(Coubs, CoubInfoPage.Coubs...) 435 | } 436 | 437 | return Coubs, nil 438 | } 439 | 440 | func StripString(s string) string { 441 | var result strings.Builder 442 | for i := 0; i < len(s); i++ { 443 | b := s[i] 444 | if ('a' <= b && b <= 'z') || 445 | ('A' <= b && b <= 'Z') || 446 | ('0' <= b && b <= '9') || 447 | b == ' ' { 448 | result.WriteByte(b) 449 | } 450 | } 451 | return result.String() 452 | } 453 | --------------------------------------------------------------------------------