├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── banner_v1.1.png └── workflows │ ├── codeql-analysis.yml │ └── go.yml ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── README.md ├── Taskfile.yml ├── cmd └── creep │ ├── main.go │ └── main_test.go ├── go.mod └── pkg ├── download ├── download.go └── download_test.go └── flags └── flags.go /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at flyweight@pm.me. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Hi and thanks for you interest in contributing to `creep`! 4 | 5 | This document is still a work in progress. 6 | -------------------------------------------------------------------------------- /.github/banner_v1.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Splode/creep/eb0e617ed31334159da71f4bfa27976677710c3a/.github/banner_v1.1.png -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | # The branches below must be a subset of the branches above 8 | branches: [master] 9 | schedule: 10 | - cron: '0 4 * * 5' 11 | 12 | jobs: 13 | analyze: 14 | name: Analyze 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | # Override automatic language detection by changing the below list 21 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 22 | language: ['go'] 23 | # Learn more... 24 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 25 | 26 | steps: 27 | - name: Checkout repository 28 | uses: actions/checkout@v2 29 | with: 30 | # We must fetch at least the immediate parents so that if this is 31 | # a pull request then we can checkout the head. 32 | fetch-depth: 2 33 | 34 | # If this run was triggered by a pull request event, then checkout 35 | # the head of the pull request instead of the merge commit. 36 | - run: git checkout HEAD^2 37 | if: ${{ github.event_name == 'pull_request' }} 38 | 39 | # Initializes the CodeQL tools for scanning. 40 | - name: Initialize CodeQL 41 | uses: github/codeql-action/init@v1 42 | with: 43 | languages: ${{ matrix.language }} 44 | 45 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 46 | # If this step fails, then you should remove it and run the build manually (see below) 47 | - name: Autobuild 48 | uses: github/codeql-action/autobuild@v1 49 | 50 | # ℹ️ Command-line programs to run using the OS shell. 51 | # 📚 https://git.io/JvXDl 52 | 53 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 54 | # and modify them (or add more) to build your code if your project 55 | # uses a compiled language 56 | 57 | #- run: | 58 | # make bootstrap 59 | # make release 60 | 61 | - name: Perform CodeQL Analysis 62 | uses: github/codeql-action/analyze@v1 63 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | 5 | build: 6 | name: Build 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 1.14 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.14 14 | id: go 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v2 18 | 19 | - name: Get dependencies 20 | run: | 21 | go get -v -t -d ./... 22 | if [ -f Gopkg.toml ]; then 23 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 24 | dep ensure 25 | fi 26 | 27 | - name: Build 28 | run: go build -v ./cmd/creep 29 | 30 | - name: Test-download 31 | run: go test ./pkg/download 32 | 33 | - name: Test-creep 34 | run: go test ./cmd/creep 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | .envrc 4 | .idea 5 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: creep 2 | builds: 3 | - main: ./cmd/creep/main.go 4 | env: 5 | - CGO_ENABLED=0 6 | goos: 7 | - darwin 8 | - linux 9 | - windows 10 | ldflags: 11 | - -s -w -X github.com/Splode/creep/pkg/flags.Version={{ .Version }} 12 | archives: 13 | - replacements: 14 | darwin: macOS 15 | linux: linux 16 | windows: windows 17 | 386: i386 18 | amd64: x86_64 19 | format_overrides: 20 | - goos: windows 21 | format: zip 22 | checksum: 23 | name_template: "{{ .ProjectName }}-checksums.txt" 24 | snapshot: 25 | name_template: "{{ .Tag }}" 26 | changelog: 27 | sort: asc 28 | filters: 29 | exclude: 30 | - '^docs:' 31 | - '^test:' 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Christopher Murphy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # creep 2 | 3 | A specialized image download utility, useful for grabbing massive amounts of random images. 4 | 5 | creep logo 6 | 7 | ![Go](https://github.com/Splode/creep/workflows/Go/badge.svg?branch=master) 8 | 9 | Creep can be used to generate gobs of random image data quickly given a single URL. It has no dependencies or requirements and is cross-platform. 10 | 11 | - [creep](#creep) 12 | - [Install](#install) 13 | - [Prebuilt Binaries](#prebuilt-binaries) 14 | - [Build from Source](#build-from-source) 15 | - [Usage](#usage) 16 | - [Options](#options) 17 | - [Examples](#examples) 18 | - [Sample URLs](#sample-urls) 19 | - [Why](#why) 20 | - [Contributing](#contributing) 21 | - [Author](#author) 22 | - [License](#license) 23 | 24 | ## Install 25 | 26 | ### Prebuilt Binaries 27 | 28 | Install a prebuilt binary from the [releases page](https://github.com/Splode/creep/releases/latest). 29 | 30 | ### Build from Source 31 | 32 | ```bash 33 | go get github.com/splode/creep/cmd/creep 34 | ``` 35 | 36 | ## Usage 37 | 38 | Simply pass in a URL that returns an image to `creep` to download. Pass in a number of images, and `creep` will download them all concurrently. 39 | 40 | ``` 41 | Usage: 42 | creep [FLAGS] [OPTIONS] [URL] 43 | 44 | URL: 45 | The URL of the resource to access (required) 46 | 47 | Options: 48 | -c, --count int The number of times to access the resource (defaults to 1) 49 | -n, --name string The base filename to use as output (defaults to "creep") 50 | -o, --out string The output directory path (defaults to current directory) 51 | -t, --throttle int Number of seconds to wait between downloads (defaults to 0) 52 | 53 | Flags: 54 | -h, --help Prints help information 55 | -v, --version Prints version information 56 | ``` 57 | 58 | ### Options 59 | 60 | `URL` 61 | 62 | Specifies the HTTP URL of the image resource to access. This is the only required argument. 63 | 64 | `--count` 65 | 66 | The number of times to access and download a resource. Defaults to 1. 67 | 68 | `--name` 69 | 70 | The base filename of the downloaded resource. For example, given a `count` of `3`, a `name` of `cat` and `url` that returns `jpg`, `creep` will generate the following list of files: 71 | 72 | ``` 73 | cat-1.jpg 74 | cat-2.jpg 75 | cat-3.jpg 76 | ``` 77 | 78 | Defaults to "creep". 79 | 80 | `--out` 81 | 82 | The directory to save the output. If no directory is given, the current directory will be used. If the given directory does not exist, it will be created. 83 | 84 | `--throttle` 85 | 86 | Throttle downloads by the given number of seconds. Some URLs will return a given image based on the current time, so performing requests in very quick succession will yield duplicate images. If you're receiving duplicate images, it may help to throttle the download rate. Throttling is disabled by default. 87 | 88 | ### Examples 89 | 90 | Download `32` random images to the current directory. 91 | 92 | ```bash 93 | creep -c 32 https://thispersondoesnotexist.com/image 94 | ``` 95 | 96 | Download `64` random images using the base filename `random` to the `downloads` folder, throttling the download rate to `3` seconds. 97 | 98 | ```bash 99 | creep --name=random --out=downloads --count=64 --throttle=3 https://source.unsplash.com/random 100 | ``` 101 | 102 | Download a single random image to the current directory. 103 | 104 | ```bash 105 | creep https://source.unsplash.com/random 106 | ``` 107 | 108 | ### Sample URLs 109 | 110 | The following URLs will serve a random image upon request: 111 | 112 | - Unsplash [https://source.unsplash.com/random](https://source.unsplash.com/random) 113 | - This Person Does Not Exist [https://thispersondoesnotexist.com/image](https://thispersondoesnotexist.com/image) 114 | - Picsum [https://picsum.photos/400](https://picsum.photos/400) 115 | - Lorem Pixel [http://lorempixel.com/400](http://lorempixel.com/400) 116 | - This Cat Does Not Exist [https://thiscatdoesnotexist.com/](https://thiscatdoesnotexist.com/) 117 | - PlaceGOAT [http://placegoat.com/200](http://placegoat.com/200) 118 | - PlaceIMG [https://placeimg.com/640/480/any](https://placeimg.com/640/480/any) 119 | - LoremFlickr [https://loremflickr.com/320/240](https://loremflickr.com/320/240) 120 | - This Artwork Does Not Exist [https://thisartworkdoesnotexist.com](https://thisartworkdoesnotexist.com) 121 | - This Horse Does Not Exist [https://thishorsedoesnotexist.com/](https://thishorsedoesnotexist.com/) 122 | 123 | ## Why 124 | 125 | I frequently find myself needing to seed application data sets with lots of images for testing or demos. Given a few minutes searching for a tool, I wasn't able to find something that suited my requirements, so I built one. 126 | 127 | Why Go and not simply script `curl` or python? Go's concurrency model makes multiple HTTP requests _fast_, and being able to compile to a single, cross-platform binary is handy. Besides, Go's cool. 128 | 129 | ## Contributing 130 | 131 | Contributions are welcome! See [CONTRIBUTING](https://github.com/Splode/creep/blob/master/.github/CONTRIBUTING.md) for details. 132 | 133 | ## Author 134 | 135 | [Christopher Murphy](https://github.com/Splode) 136 | 137 | ## License 138 | 139 | [MIT](https://github.com/Splode/creep/blob/master/LICENSE) 140 | -------------------------------------------------------------------------------- /Taskfile.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | tasks: 4 | build: 5 | deps: [build-dir] 6 | cmds: 7 | - go build -o build ./cmd/creep 8 | 9 | build-dir: 10 | cmds: 11 | - mkdir build 12 | status: 13 | - test -d build 14 | 15 | release-snapshot: 16 | cmds: 17 | - goreleaser release --skip-publish --snapshot --rm-dist 18 | -------------------------------------------------------------------------------- /cmd/creep/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/Splode/creep/pkg/download" 8 | "github.com/Splode/creep/pkg/flags" 9 | ) 10 | 11 | func main() { 12 | config, err := flags.HandleFlags() 13 | if err != nil { 14 | fmt.Fprintf(os.Stderr, "Failed to parse arguments: %s\n", err) 15 | os.Exit(1) 16 | } 17 | 18 | if errs := download.Batch((*download.Config)(config)); errs != nil { 19 | for _, err := range errs { 20 | fmt.Fprintf(os.Stderr, err.Error()) 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cmd/creep/main_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | var ( 14 | binName = "creep" 15 | ) 16 | 17 | func TestMain(m *testing.M) { 18 | fmt.Printf("Building %s...\n", binName) 19 | 20 | if runtime.GOOS == "windows" { 21 | binName += ".exe" 22 | } 23 | 24 | build := exec.Command("go", "build", "-o", binName) 25 | if err := build.Run(); err != nil { 26 | fmt.Fprintf(os.Stderr, "Failed to build tool %s: %s", binName, err) 27 | os.Exit(1) 28 | } 29 | 30 | fmt.Println("Running tests...") 31 | result := m.Run() 32 | 33 | fmt.Println("Cleaning up...") 34 | if err := os.Remove(binName); err != nil { 35 | fmt.Fprintf(os.Stderr, "Failed to remove tool %s: %s", binName, err) 36 | os.Exit(1) 37 | } 38 | 39 | images, err := filepath.Glob("*.jpg") 40 | if err != nil { 41 | fmt.Fprintf(os.Stderr, "Failed to find downloaded image files: %s", err) 42 | } 43 | if len(images) > 0 { 44 | for _, img := range images { 45 | if err := os.Remove(img); err != nil { 46 | fmt.Fprintf(os.Stderr, "Failed to remove image file %s: %s", img, err) 47 | os.Exit(1) 48 | } 49 | } 50 | } 51 | os.Exit(result) 52 | } 53 | 54 | func TestCreepCLI(t *testing.T) { 55 | dir, err := os.Getwd() 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | 60 | cmdPath := filepath.Join(dir, binName) 61 | 62 | t.Run("ErrOnNoArgs", func(t *testing.T) { 63 | cmd := exec.Command(cmdPath) 64 | if err := cmd.Run(); err == nil { 65 | t.Fatal("Expected to receive error, none received") 66 | } 67 | }) 68 | 69 | t.Run("PrintHelp", func(t *testing.T) { 70 | cmd := exec.Command(cmdPath, "-h") 71 | if err := cmd.Run(); err != nil { 72 | t.Fatal(err) 73 | } 74 | }) 75 | 76 | t.Run("PrintVersion", func(t *testing.T) { 77 | cmd := exec.Command(cmdPath, "-v") 78 | out, err := cmd.CombinedOutput() 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | exp := "Master" + "\n" 83 | if exp != string(out) { 84 | t.Errorf("Expected %q, got %q instead\n", exp, string(out)) 85 | } 86 | }) 87 | 88 | t.Run("DownloadSingleImage", func(t *testing.T) { 89 | args := "https://source.unsplash.com/random" 90 | cmd := exec.Command(cmdPath, strings.Split(args, " ")...) 91 | if err := cmd.Run(); err != nil { 92 | t.Fatal(err) 93 | } 94 | }) 95 | 96 | t.Run("DownloadMultipleImages", func(t *testing.T) { 97 | args := "-c 2 https://source.unsplash.com/random" 98 | cmd := exec.Command(cmdPath, strings.Split(args, " ")...) 99 | if err := cmd.Run(); err != nil { 100 | t.Fatal(err) 101 | } 102 | }) 103 | 104 | t.Run("DownloadSingleImageWithName", func(t *testing.T) { 105 | args := "-n test https://source.unsplash.com/random" 106 | cmd := exec.Command(cmdPath, strings.Split(args, " ")...) 107 | if err := cmd.Run(); err != nil { 108 | t.Fatal(err) 109 | } 110 | }) 111 | 112 | t.Run("DownloadSingleImageWithPath", func(t *testing.T) { 113 | args := "-o test https://source.unsplash.com/random" 114 | cmd := exec.Command(cmdPath, strings.Split(args, " ")...) 115 | if err := cmd.Run(); err != nil { 116 | t.Fatal(err) 117 | } 118 | if err := os.RemoveAll("test"); err != nil { 119 | fmt.Fprintf(os.Stderr, "Failed to remove test directory: %q", err) 120 | os.Exit(1) 121 | } 122 | }) 123 | 124 | t.Run("DownloadMultipleImagesWithThrottle", func(t *testing.T) { 125 | args := "-c 2 -t 4 https://source.unsplash.com/random" 126 | cmd := exec.Command(cmdPath, strings.Split(args, " ")...) 127 | if err := cmd.Run(); err != nil { 128 | t.Fatal(err) 129 | } 130 | }) 131 | } 132 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Splode/creep 2 | 3 | go 1.14 4 | -------------------------------------------------------------------------------- /pkg/download/download.go: -------------------------------------------------------------------------------- 1 | package download 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | var mimes = map[string]string{ 15 | "image/.jpg": "jpg", 16 | "image/jpeg": "jpg", 17 | "image/jpg": "jpg", 18 | "image/png": "png", 19 | } 20 | 21 | // Config represents the command-line configuration options. 22 | type Config struct { 23 | Count uint 24 | Name string 25 | Out string 26 | Throttle uint 27 | URL string 28 | } 29 | 30 | // Batch downloads a batch of images given a set of options. 31 | func Batch(config *Config) (errs []error) { 32 | var wg sync.WaitGroup 33 | wg.Add(int(config.Count)) 34 | 35 | for i := 1; i <= int(config.Count); i++ { 36 | file := fmt.Sprintf("%s-%d", config.Name, i) 37 | outPath := filepath.Join(config.Out, file) 38 | if config.Throttle > 0 && i > 1 { 39 | var s string 40 | if config.Throttle > 1 { 41 | s = "seconds" 42 | } else { 43 | s = "second" 44 | } 45 | fmt.Printf("Throttling for %d %s...\n", config.Throttle, s) 46 | time.Sleep(time.Second * time.Duration(config.Throttle)) 47 | } 48 | fmt.Printf("Downloading %s to %s...\n", config.URL, outPath) 49 | go func(wg *sync.WaitGroup) { 50 | if err := ImageFile(outPath, config.URL); err != nil { 51 | errs = append(errs, fmt.Errorf("Failed to download %s: %s\n", file, err.Error())) 52 | } else { 53 | fmt.Printf("Successfully downloaded %s to %s\n", file, outPath) 54 | } 55 | wg.Done() 56 | }(&wg) 57 | } 58 | wg.Wait() 59 | return errs 60 | } 61 | 62 | // ImageFile saves the request body from a given URL to the provided filepath. 63 | func ImageFile(filepath, url string) (err error) { 64 | // get data 65 | res, err := http.Get(url) 66 | if err != nil { 67 | return err 68 | } 69 | defer func() { 70 | if err := res.Body.Close(); err != nil { 71 | err = err.(error) 72 | } 73 | }() 74 | 75 | // check server response 76 | if res.StatusCode != http.StatusOK { 77 | return fmt.Errorf("bad status: %s", res.Status) 78 | } 79 | 80 | // attempt to get file ext 81 | ext, err := getExtHeader(res) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | // create file 87 | path := fmt.Sprintf("%s.%s", filepath, ext) 88 | out, err := os.Create(path) 89 | if err != nil { 90 | return err 91 | } 92 | defer func() { 93 | if err := out.Close(); err != nil { 94 | err = err.(error) 95 | } 96 | }() 97 | 98 | // write body to file 99 | if _, err = io.Copy(out, res.Body); err != nil { 100 | return err 101 | } 102 | 103 | return nil 104 | } 105 | 106 | // getExtHeader attempts to infer a file extension from the Content-Type of a 107 | // given response header using mime types. 108 | func getExtHeader(r *http.Response) (string, error) { 109 | ct := r.Header["Content-Type"][0] 110 | mime, prs := mimes[ct] 111 | if !prs { 112 | return "", errors.New("could not detect mime-type from http response") 113 | } 114 | return mime, nil 115 | } 116 | -------------------------------------------------------------------------------- /pkg/download/download_test.go: -------------------------------------------------------------------------------- 1 | package download 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "testing" 9 | ) 10 | 11 | // TestBatch tests downloading a batch of images given a set of options. 12 | func TestBatch(t *testing.T) { 13 | testDir, err := ioutil.TempDir("", "creep") 14 | if err != nil { 15 | t.Fatalf("Failed to create temp directory: %s", err) 16 | } 17 | defer func() { 18 | if err := os.RemoveAll(testDir); err != nil { 19 | t.Fatalf("Failed to remove temp directory: %s", err) 20 | } 21 | }() 22 | 23 | fmt.Println(testDir) 24 | 25 | testCases := []struct { 26 | expectErr bool 27 | config Config 28 | }{ 29 | {expectErr: false, config: Config{ 30 | Count: 3, 31 | Name: "test", 32 | Out: testDir, 33 | Throttle: 0, 34 | URL: "https://thispersondoesnotexist.com/image", 35 | }}, 36 | {expectErr: true, config: Config{ 37 | Count: 1, 38 | Name: "test", 39 | Out: testDir, 40 | Throttle: 0, 41 | URL: "http://example.com", 42 | }}, 43 | } 44 | 45 | for _, tc := range testCases { 46 | errs := Batch(&tc.config) 47 | if tc.expectErr { 48 | if errs == nil { 49 | t.Fatalf("Expected error, got nil") 50 | } 51 | } else { 52 | if errs != nil { 53 | for _, err := range errs { 54 | t.Fatal(err.Error()) 55 | } 56 | } 57 | } 58 | } 59 | } 60 | 61 | // TestImageFile tests downloading an image. 62 | func TestImageFile(t *testing.T) { 63 | testDir, err := ioutil.TempDir("", "creep") 64 | if err != nil { 65 | t.Fatalf("Failed to create temp directory: %s", err) 66 | } 67 | defer func() { 68 | if err := os.RemoveAll(testDir); err != nil { 69 | t.Fatalf("Failed to remove temp directory: %s", err) 70 | } 71 | }() 72 | 73 | testCases := []struct { 74 | expectError bool 75 | URL string 76 | }{ 77 | {expectError: true, URL: ""}, 78 | {expectError: true, URL: "http://example.com/42"}, 79 | {expectError: true, URL: "http://example.com/"}, 80 | {expectError: false, URL: "https://source.unsplash.com/random"}, 81 | {expectError: false, URL: "https://thispersondoesnotexist.com/image"}, 82 | {expectError: false, URL: "https://picsum.photos/400"}, 83 | {expectError: false, URL: "http://lorempixel.com/400/200"}, 84 | {expectError: false, URL: "https://thiscatdoesnotexist.com/"}, 85 | {expectError: false, URL: "https://loremflickr.com/320/240"}, 86 | {expectError: false, URL: "https://placeimg.com/640/480/any"}, 87 | {expectError: false, URL: "http://placegoat.com/200"}, 88 | {expectError: false, URL: "https://thisartworkdoesnotexist.com"}, 89 | } 90 | for i, tc := range testCases { 91 | p := fmt.Sprintf("test-%d", i) 92 | f := path.Join(testDir, p) 93 | 94 | err = ImageFile(f, tc.URL) 95 | 96 | if tc.expectError { 97 | if err == nil { 98 | t.Fatalf("ImageFile download %s; expected error, got nil.", tc.URL) 99 | } 100 | } else { 101 | if err != nil { 102 | t.Fatalf("ImageFile returned unexpected error: %s: %v", tc.URL, err) 103 | } 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /pkg/flags/flags.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "os" 8 | ) 9 | 10 | // Version of the app. This is set by goreleaser during release builds using 11 | // the latest git tag. 12 | var Version = "Master" 13 | 14 | // Config represents the command-line configuration options. 15 | type Config struct { 16 | Count uint 17 | Name string 18 | Out string 19 | Throttle uint 20 | URL string 21 | } 22 | 23 | // HandleFlags parses command line arguments and returns a config. 24 | func HandleFlags() (c *Config, err error) { 25 | c = &Config{} 26 | v := false 27 | flag.StringVar(&c.Name, "name", "creep", "") 28 | flag.StringVar(&c.Name, "n", "creep", "") 29 | flag.UintVar(&c.Count, "count", 1, "") 30 | flag.UintVar(&c.Count, "c", 1, "") 31 | flag.StringVar(&c.Out, "out", "", "") 32 | flag.StringVar(&c.Out, "o", "", "") 33 | flag.UintVar(&c.Throttle, "throttle", 0, "") 34 | flag.UintVar(&c.Throttle, "t", 0, "") 35 | flag.BoolVar(&v, "version", false, "") 36 | flag.BoolVar(&v, "v", false, "") 37 | flag.Usage = generateUsage() 38 | flag.Parse() 39 | c.URL = flag.Arg(0) 40 | 41 | if v { 42 | fmt.Println(Version) 43 | os.Exit(0) 44 | } 45 | 46 | if c.URL == "" { 47 | err = errors.New("expected a URL, none given") 48 | } 49 | 50 | if c.Count <= 0 { 51 | err = fmt.Errorf("expected count to be an integer greater than 0, %d given", c.Count) 52 | } 53 | 54 | if c.Out != "" { 55 | err = parseOut(c.Out) 56 | } 57 | 58 | if err != nil { 59 | return &Config{}, err 60 | } 61 | 62 | return c, err 63 | } 64 | 65 | func generateUsage() func() { 66 | return func() { 67 | fmt.Fprintf(os.Stdout, "\ncreep %s", Version) 68 | fmt.Fprintf(os.Stdout, ` 69 | 70 | Downloads an image from the given URL a given number of times to the specified directory. 71 | 72 | Usage: 73 | creep [FLAGS] [OPTIONS] [URL] 74 | 75 | URL: 76 | The URL of the resource to access (required) 77 | 78 | Options: 79 | -c, --count int The number of times to access the resource (defaults to 1) 80 | -n, --name string The base filename to use as output (defaults to "creep") 81 | -o, --out string The output directory path (defaults to current directory) 82 | -t, --throttle int Number of seconds to wait between downloads (defaults to 0) 83 | 84 | Flags: 85 | -h, --help Prints help information 86 | -v, --version Prints version information 87 | 88 | Example usage: 89 | creep -c 32 https://thispersondoesnotexist.com/image 90 | creep --name=random --out=downloads --count=64 --throttle=3 https://source.unsplash.com/random`) 91 | fmt.Println() 92 | os.Exit(0) 93 | } 94 | } 95 | 96 | // parseOut validates the given directory path, creating the directory at the 97 | // given path if it does not exist. 98 | func parseOut(out string) error { 99 | if _, err := os.Stat(out); err != nil { 100 | if os.IsNotExist(err) { 101 | err := os.MkdirAll(out, os.ModePerm) 102 | if err != nil { 103 | return err 104 | } 105 | } else { 106 | return err 107 | } 108 | } 109 | return nil 110 | } 111 | --------------------------------------------------------------------------------