├── .github └── workflows │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── build └── docker │ ├── build │ └── Dockerfile │ └── gget │ └── Dockerfile ├── cmd └── gget │ ├── command.go │ ├── factory.go │ └── runtime.go ├── docs └── releases │ ├── v0.1.0.md │ ├── v0.1.1.md │ ├── v0.2.0.md │ ├── v0.3.0.md │ ├── v0.4.0.md │ ├── v0.5.0.md │ ├── v0.5.1.md │ ├── v0.5.2.md │ ├── v0.5.5.md │ ├── v0.6.0.md │ └── v0.6.1.md ├── go.mod ├── go.sum ├── internal └── tools │ └── tools.go ├── main.go ├── pkg ├── app │ ├── version.go │ └── writer.go ├── checksum │ ├── algorithm.go │ ├── all_checksum_selector.go │ ├── checksum.go │ ├── deferred_checksum.go │ ├── ginkgo_test.go │ ├── guesser.go │ ├── guesser_test.go │ ├── hash_checksum.go │ ├── hash_verification_error.go │ ├── hash_verifier.go │ ├── in_memory_alias_manager.go │ ├── in_memory_alias_manager_test.go │ ├── in_memory_manager.go │ ├── manager.go │ ├── multi_manager.go │ ├── parser │ │ ├── deferred_manager.go │ │ ├── ginkgo_test.go │ │ ├── import_lines.go │ │ ├── markdown.go │ │ ├── markdown_codefence.go │ │ ├── markdown_codeindent.go │ │ └── markdown_test.go │ ├── selector.go │ ├── strongest_checksum_selector.go │ └── verification_profile.go ├── cli │ └── opt │ │ ├── constraint.go │ │ ├── export.go │ │ ├── ref.go │ │ ├── resource_matcher.go │ │ ├── resource_transfer.go │ │ └── verify_checksum.go ├── export │ ├── data.go │ ├── exporter.go │ ├── go_template_exporter.go │ ├── json_exporter.go │ ├── jsonpath_exporter.go │ ├── marshal_data.go │ ├── plain_exporter.go │ └── yaml_exporter.go ├── ggetutil │ └── version_opt.go ├── gitutil │ └── commit.go ├── service │ ├── github │ │ ├── archive │ │ │ └── resource.go │ │ ├── asset │ │ │ └── resource.go │ │ ├── blob │ │ │ └── resource.go │ │ ├── client_factory.go │ │ ├── commit_ref.go │ │ ├── ref_resolver.go │ │ ├── release_checksum_manager.go │ │ ├── release_ref.go │ │ └── service.go │ ├── gitlab │ │ ├── archive │ │ │ └── resource.go │ │ ├── asset │ │ │ └── resource.go │ │ ├── blob │ │ │ └── resource.go │ │ ├── client_factory.go │ │ ├── commit_ref.go │ │ ├── gitlabutil │ │ │ └── util.go │ │ ├── release_ref.go │ │ └── service.go │ ├── lookup_ref.go │ ├── multi_ref_resolver.go │ ├── ref.go │ ├── ref_resolver.go │ ├── resource.go │ └── resource_resolver.go └── transfer │ ├── batch.go │ ├── state.go │ ├── step.go │ ├── step │ ├── executable.go │ ├── rename.go │ ├── temp_file_target.go │ ├── verify_checksum.go │ └── writer_target.go │ ├── transfer.go │ └── transferutil │ └── transfer_factory.go ├── scripts ├── build.dockerized.sh ├── build.local.sh ├── integration.test.sh └── website.build.sh └── website ├── .editorconfig ├── .gitignore ├── assets ├── asciinema │ └── hugo.cast └── css │ └── tailwind.css ├── jsconfig.json ├── layouts └── default.vue ├── nuxt.config.js ├── package-lock.json ├── package.json ├── pages └── index.vue ├── plugins └── axios.js ├── static └── img │ └── card~v1~1280x640.png ├── tailwind.config.js └── yarn.lock /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | jobs: 7 | release: 8 | name: Publish Release 9 | runs-on: ubuntu-18.04 10 | steps: 11 | - name: Checkout Tag 12 | uses: actions/checkout@v1 13 | - name: Build Assets 14 | id: build 15 | run: | 16 | version="$( echo "${{ github.ref }}" | sed 's#refs/tags/v##' )" 17 | echo ::set-output name=version::"$version" 18 | 19 | ./scripts/build.dockerized.sh "$version" 20 | 21 | if [[ -e "docs/releases/v${version}.md" ]]; then 22 | sed '1{/^---$/!q;};1,/^---$/d' "docs/releases/v${version}.md" > tmp/release-body 23 | echo "" >> tmp/release-body 24 | fi 25 | 26 | cd tmp/build 27 | 28 | ( 29 | echo "**Assets (sha256)**" 30 | echo "" 31 | ) >> ../release-body 32 | 33 | for os in darwin linux windows; do 34 | filename=$( echo *-$os-* ) 35 | echo ::set-output name=filename_$os::"$filename" 36 | 37 | shasum -a 256 $filename | tee /dev/stderr | sed 's/^/ /' >> ../release-body 38 | done 39 | 40 | echo ::set-output name=release_body::"$( jq -rRs 'gsub("\r";"%0D")|gsub("\n";"%0A")' < ../release-body )" 41 | - name: Create Release 42 | id: create_release 43 | uses: actions/create-release@master # v1 https://github.com/actions/create-release/issues/38 44 | env: 45 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 46 | with: 47 | tag_name: ${{ github.ref }} 48 | release_name: v${{ steps.build.outputs.version }} 49 | body: ${{ steps.build.outputs.release_body }} 50 | draft: false 51 | prerelease: false 52 | - name: Upload Asset (darwin) 53 | uses: actions/upload-release-asset@v1.0.1 54 | env: 55 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | with: 57 | upload_url: ${{ steps.create_release.outputs.upload_url }} 58 | asset_path: ./tmp/build/${{ steps.build.outputs.filename_darwin }} 59 | asset_name: ${{ steps.build.outputs.filename_darwin }} 60 | asset_content_type: application/octet-stream 61 | - name: Upload Asset (linux) 62 | uses: actions/upload-release-asset@v1.0.1 63 | env: 64 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 65 | with: 66 | upload_url: ${{ steps.create_release.outputs.upload_url }} 67 | asset_path: ./tmp/build/${{ steps.build.outputs.filename_linux }} 68 | asset_name: ${{ steps.build.outputs.filename_linux }} 69 | asset_content_type: application/octet-stream 70 | - name: Upload Asset (windows) 71 | uses: actions/upload-release-asset@v1.0.1 72 | env: 73 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 74 | with: 75 | upload_url: ${{ steps.create_release.outputs.upload_url }} 76 | asset_path: ./tmp/build/${{ steps.build.outputs.filename_windows }} 77 | asset_name: ${{ steps.build.outputs.filename_windows }} 78 | asset_content_type: application/octet-stream 79 | docker: 80 | name: Publish Docker 81 | runs-on: ubuntu-18.04 82 | needs: 83 | - release 84 | env: 85 | IMAGE_NAME: gget 86 | steps: 87 | - name: Prepare 88 | id: prepare 89 | run: | 90 | version="$( echo "${{ github.ref }}" | sed 's#refs/tags/v##' )" 91 | echo ::set-output name=version::"$version" 92 | 93 | image=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME 94 | echo ::set-output name=image::"$image" 95 | - uses: actions/checkout@v1 96 | - name: Build Image 97 | run: | 98 | docker build build/docker/$IMAGE_NAME \ 99 | --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ 100 | --build-arg GGET_REF=github.com/${{ github.repository }} \ 101 | --build-arg GGET_VERSION=v${{ steps.prepare.outputs.version }} \ 102 | --tag image 103 | - name: Registry Login 104 | run: | 105 | echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin 106 | - name: Publish Images 107 | run: | 108 | for tag in \ 109 | v${{ steps.prepare.outputs.version }} \ 110 | v$( echo "${{ steps.prepare.outputs.version }}" | cut -d. -f1-2 ) \ 111 | v$( echo "${{ steps.prepare.outputs.version }}" | cut -d. -f1 ) \ 112 | latest \ 113 | ; do 114 | fqimagetag="${{ steps.prepare.outputs.image }}:$tag" 115 | docker tag image $fqimagetag 116 | docker push $fqimagetag 117 | done 118 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.cache 2 | /tmp 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Danny Berger 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gget 2 | 3 | An easier way to find and automate file downloads from GitHub and GitLab repositories. Learn more from the examples and documentation at [gget.io](https://gget.io/). 4 | 5 | * Standalone CLI - no `git` or local clones required 6 | * Public & Private Servers - supporting both [GitHub](https://github.com/) and [GitLab](https://gitlab.com/) 7 | * Public & Private Repos - API-based access for private resources 8 | * Tags, Branches, and Commits - also semver-based constraint matching 9 | * Archives, Assets, and Blobs - download any type of resource from repos 10 | * Built-in File Operations - easy rename, binary, and stream options 11 | * Checksum Verification - verify downloads by SHA (if published) 12 | * Open-Source Project - contribute and discuss at [dpb587/gget](https://github.com/dpb587/gget) 13 | 14 | ## Command Line Usage 15 | 16 | Pass the repository and file globs to match and download. 17 | 18 | ``` 19 | $ gget github.com/gohugoio/hugo 'hugo_extended_*_Linux-64bit.deb' 20 | Found 1 file (13.9M) from github.com/gohugoio/hugo@v0.73.0 21 | √ hugo_extended_0.73.0_Linux-64bit.deb done (sha256 OK) 22 | ``` 23 | 24 | Use `--help` to see the full list of options for more advanced usage. 25 | 26 | ### Installation 27 | 28 | Binaries for Linux, macOS, and Windows can be downloaded from the [releases](https://github.com/dpb587/gget/releases) page. A [Homebrew](https://brew.sh/) recipe is also available for Linux and macOS. 29 | 30 | ``` 31 | brew install dpb587/tap/gget 32 | ``` 33 | 34 | ## Docker Usage 35 | 36 | The `gget` image can be used as a build stage to download assets for a later stage. 37 | 38 | ```dockerfile 39 | FROM ghcr.io/dpb587/gget/gget as gget 40 | RUN gget --executable github.com/cloudfoundry/bosh-cli --ref-version=5.x bosh=bosh-cli-*-linux-amd64 41 | RUN gget --executable github.com/cloudfoundry/bosh-bootloader bbl=bbl-*_linux_x86-64 42 | RUN gget --stdout github.com/pivotal-cf/om om-linux-*.tar.gz | tar -xzf- om 43 | 44 | FROM ubuntu 45 | COPY --from=gget /result/* /usr/local/bin/ 46 | # ...everything else for your image... 47 | ``` 48 | 49 | ## Technical Notes 50 | 51 | ### Checksum Verification 52 | 53 | When downloading files, `gget` attempts to validate checksums when they are found for files (erroring if they do not match). Since checksums are not an official feature for repository assets, the following common conventions are being used. 54 | 55 | * Algorithms: `sha512`, `sha256`, `sha1`, `md5` 56 | * File Format: `shasum`/`md5sum` output (`{checksum} {file}`) 57 | * Sources: 58 | * release notes - code block or code fence 59 | * sibling files with an algorithm suffix (case-insensitive) - `*.{algorithm}` 60 | * checksum list files (case-insensitive) - `checksum`, `checksums`, `*checksums.txt`, `{algorithm}sum.txt`, `{algorithm}sums.txt` 61 | 62 | ## Alternatives 63 | 64 | * `wget`/`curl` -- if you want to manually maintain version download URLs and private signing 65 | * [`hub release download ...`](https://github.com/github/hub) -- if you already have `git` configured and a cloned GitHub repository 66 | * [`fetch`](https://github.com/gruntwork-io/fetch) -- similar tool and capabilities for GitHub (discovered this after gget@0.5.2) 67 | 68 | ## License 69 | 70 | [MIT License](LICENSE) 71 | -------------------------------------------------------------------------------- /build/docker/build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.6 2 | RUN apt update && apt install -y xz-utils && rm -rf /var/lib/apt/lists/* 3 | RUN wget -qO- https://github.com/upx/upx/releases/download/v3.96/upx-3.96-amd64_linux.tar.xz | xz -cd | tar -xf- --strip-components=1 -C /usr/local/bin --wildcards '*/upx' 4 | -------------------------------------------------------------------------------- /build/docker/gget/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine as base 2 | RUN apk add --no-cache ca-certificates wget 3 | 4 | ARG GGET_VERSION 5 | ARG GGET_REF 6 | ARG GITHUB_TOKEN 7 | ENV gget /usr/local/bin/gget 8 | RUN true \ 9 | && wget -qO "$gget" https://github.com/dpb587/gget/releases/download/v0.1.0/gget-0.1.0-linux-amd64 \ 10 | && echo "418b68caeb302898322465abc31c316a1dbdb7872cdad0fe4a9661c5118860c4 $gget" | sha256sum -c \ 11 | && chmod +x "$gget" 12 | RUN gget --exec ${GGET_REF:-github.com/dpb587/gget}@${GGET_VERSION:-} "$gget=gget-*-linux-amd64" 13 | RUN gget --version -v 14 | 15 | FROM alpine 16 | RUN apk add --no-cache ca-certificates xz 17 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 18 | COPY --from=base /usr/local/bin/gget /bin/gget 19 | RUN mkdir /result 20 | WORKDIR /result 21 | ENTRYPOINT ["/bin/gget"] 22 | CMD ["--help"] 23 | -------------------------------------------------------------------------------- /cmd/gget/command.go: -------------------------------------------------------------------------------- 1 | package gget 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "sort" 10 | 11 | "code.cloudfoundry.org/bytefmt" 12 | "github.com/dpb587/gget/pkg/cli/opt" 13 | "github.com/dpb587/gget/pkg/export" 14 | "github.com/dpb587/gget/pkg/service" 15 | "github.com/dpb587/gget/pkg/service/github" 16 | "github.com/dpb587/gget/pkg/service/gitlab" 17 | "github.com/dpb587/gget/pkg/transfer" 18 | "github.com/dpb587/gget/pkg/transfer/transferutil" 19 | "github.com/pkg/errors" 20 | ) 21 | 22 | type RepositoryOptions struct { 23 | RefStability []string `long:"ref-stability" description:"acceptable stability level(s) for latest (values: stable, pre-release, any) (default: stable)" value-name:"STABILITY"` 24 | RefVersions opt.ConstraintList `long:"ref-version" description:"version constraint(s) to require of latest (e.g. 4.x)" value-name:"CONSTRAINT"` 25 | Service string `long:"service" description:"specific git service to use (values: github, gitlab) (default: auto-detect)" value-name:"NAME"` 26 | 27 | // TODO(1.x) remove 28 | ShowRef bool `long:"show-ref" description:"show resolved repository ref instead of downloading" hidden:"true"` 29 | } 30 | 31 | type ResourceOptions struct { 32 | Exclude opt.ResourceMatcherList `long:"exclude" description:"exclude resource(s) from download (multiple)" value-name:"RESOURCE-GLOB"` 33 | IgnoreMissing opt.ResourceMatcherList `long:"ignore-missing" description:"if a resource is not found, skip it rather than failing (multiple)" value-name:"[RESOURCE-GLOB]" optional:"true" optional-value:"*"` 34 | Type service.ResourceType `long:"type" description:"type of resource to get (values: asset, archive, blob)" default:"asset" value-name:"TYPE"` 35 | List bool `long:"list" description:"list matching resources and stop before downloading"` 36 | 37 | // TODO(1.x) remove 38 | ShowResources bool `long:"show-resources" description:"show matched resources instead of downloading" hidden:"true"` 39 | } 40 | 41 | type DownloadOptions struct { 42 | CD string `long:"cd" description:"change to directory before writing files" value-name:"DIR"` 43 | Executable opt.ResourceMatcherList `long:"executable" description:"apply executable permissions to downloads (multiple)" value-name:"[RESOURCE-GLOB]" optional:"true" optional-value:"*"` 44 | Export *opt.Export `long:"export" description:"export details about the download profile (values: json, jsonpath=TEMPLATE, plain, yaml)" value-name:"FORMAT"` 45 | FailFast bool `long:"fail-fast" description:"fail and exit immediately if a download fails"` 46 | NoDownload bool `long:"no-download" description:"do not perform any downloads"` 47 | NoProgress bool `long:"no-progress" description:"do not show live-updating progress during downloads"` 48 | Parallel int `long:"parallel" description:"maximum number of parallel downloads" default:"3" value-name:"NUM"` 49 | Stdout bool `long:"stdout" description:"write file contents to stdout rather than disk"` 50 | VerifyChecksum opt.VerifyChecksum `long:"verify-checksum" description:"strategy for verifying checksums (values: auto, required, none, {algo}, {algo}-min)" value-name:"[METHOD]" default:"auto" optional-value:"required"` 51 | } 52 | 53 | type Command struct { 54 | *Runtime `group:"Runtime Options"` 55 | *RepositoryOptions `group:"Repository Options"` 56 | *ResourceOptions `group:"Resource Options"` 57 | *DownloadOptions `group:"Download Options"` 58 | Args CommandArgs `positional-args:"true"` 59 | } 60 | 61 | type CommandArgs struct { 62 | Ref opt.Ref `positional-arg-name:"HOST/OWNER/REPOSITORY[@REF]" description:"repository reference"` 63 | Resources opt.ResourceTransferList `positional-arg-name:"[LOCAL-PATH=]RESOURCE-GLOB" description:"resource name(s) to download" optional:"true"` 64 | } 65 | 66 | func (c *Command) applySettings() { 67 | if v := c.Service; v != "" { 68 | c.Args.Ref.Service = v 69 | } 70 | 71 | if len(c.Args.Resources) == 0 { 72 | c.Args.Resources = opt.ResourceTransferList{ 73 | { 74 | RemoteMatch: opt.ResourceMatcher("*"), 75 | }, 76 | } 77 | } 78 | 79 | if c.Stdout { 80 | for resourceIdx, resource := range c.Args.Resources { 81 | if resource.LocalPath() != "" { 82 | continue 83 | } 84 | 85 | c.Args.Resources[resourceIdx].LocalName = "-" 86 | } 87 | } 88 | 89 | if c.CD != "" { 90 | for resourceIdx, resource := range c.Args.Resources { 91 | if resource.LocalPath() == "-" { 92 | continue 93 | } else if resource.LocalDir != "" { 94 | continue 95 | } 96 | 97 | c.Args.Resources[resourceIdx].LocalDir = c.CD 98 | } 99 | } 100 | 101 | { // TODO(1.x) remove 102 | if c.ShowRef { 103 | c.Runtime.Logger().Error("deprecated option: --show-ref was used, but --export is preferred") 104 | 105 | c.NoDownload = true 106 | } 107 | 108 | if c.ShowResources { 109 | c.Runtime.Logger().Error("deprecated option: --show-resources was used, but --list is preferred") 110 | 111 | c.List = true 112 | } 113 | } 114 | } 115 | 116 | func (c *Command) RefResolver(ref service.Ref) (service.RefResolver, error) { 117 | res := service.NewMultiRefResolver( 118 | c.Runtime.Logger(), 119 | github.NewService(c.Runtime.Logger(), github.NewClientFactory(c.Runtime.Logger(), c.Runtime.NewHTTPClient)), 120 | gitlab.NewService(c.Runtime.Logger(), gitlab.NewClientFactory(c.Runtime.Logger(), c.Runtime.NewHTTPClient)), 121 | ) 122 | 123 | return res, nil 124 | } 125 | 126 | func (c *Command) Execute(_ []string) error { 127 | c.applySettings() 128 | 129 | if c.Args.Ref.Repository == "" { 130 | return fmt.Errorf("missing argument: repository reference") 131 | } 132 | 133 | verifyChecksumProfile, err := c.VerifyChecksum.Profile() 134 | if err != nil { 135 | return errors.Wrap(err, "parsing --verify-checksum") // pseudo-parsing 136 | } 137 | 138 | refResolver, err := c.RefResolver(service.Ref(c.Args.Ref)) 139 | if err != nil { 140 | return errors.Wrap(err, "getting ref resolver") 141 | } 142 | 143 | ctx := context.Background() 144 | 145 | ref, err := refResolver.ResolveRef(ctx, service.LookupRef{ 146 | Ref: service.Ref(c.Args.Ref), 147 | RefVersions: c.RefVersions.Constraints(), 148 | RefStability: c.RefStability, 149 | }) 150 | if err != nil { 151 | return errors.Wrap(err, "resolving ref") 152 | } 153 | 154 | { // TODO(1.x) remove 155 | if c.ShowRef { 156 | metadata, err := ref.GetMetadata(ctx) 157 | if err != nil { 158 | return errors.Wrap(err, "getting metadata") 159 | } 160 | 161 | for _, metadatum := range metadata { 162 | fmt.Printf("%s\t%s\n", metadatum.Name, metadatum.Value) 163 | } 164 | } 165 | } 166 | 167 | resourceMap := map[string]service.ResolvedResource{} 168 | 169 | for _, userResource := range c.Args.Resources { 170 | candidateResources, err := ref.ResolveResource(ctx, c.Type, service.ResourceName(string(userResource.RemoteMatch))) 171 | if err != nil { 172 | return errors.Wrapf(err, "resolving resource %s", string(userResource.RemoteMatch)) 173 | } else if len(candidateResources) == 0 { 174 | if !c.IgnoreMissing.Match(string(userResource.RemoteMatch)).IsEmpty() { 175 | continue 176 | } 177 | 178 | return fmt.Errorf("no resource matched: %s", userResource.RemoteMatch) 179 | } 180 | 181 | for _, candidate := range candidateResources { 182 | if !c.Exclude.Match(candidate.GetName()).IsEmpty() { 183 | continue 184 | } 185 | 186 | resolved, matched := userResource.Resolve(candidate.GetName()) 187 | if !matched { 188 | panic("TODO should always match by now?") 189 | } 190 | 191 | localPath := resolved.LocalPath() 192 | 193 | if _, found := resourceMap[localPath]; found { 194 | return fmt.Errorf("target file already specified: %s", localPath) 195 | } 196 | 197 | resourceMap[localPath] = candidate 198 | } 199 | } 200 | 201 | // output = stderr since everything should be progress reports 202 | stdout := os.Stderr 203 | var finalStatus io.Writer 204 | 205 | if !c.Runtime.Quiet { 206 | l := len(resourceMap) 207 | ls := "" 208 | 209 | if l != 1 { 210 | ls = "s" 211 | } 212 | 213 | var downloadSizeMissing bool 214 | var downloadSize int64 215 | 216 | for _, resource := range resourceMap { 217 | size := resource.GetSize() 218 | if size == 0 { 219 | downloadSizeMissing = true 220 | 221 | break 222 | } 223 | 224 | downloadSize += size 225 | } 226 | 227 | var extra string 228 | 229 | if !downloadSizeMissing { 230 | extra = fmt.Sprintf(" (%s)", bytefmt.ByteSize(uint64(downloadSize))) 231 | } 232 | 233 | fmt.Fprintf(stdout, "Found %d file%s%s from %s\n", l, ls, extra, ref.CanonicalRef()) 234 | 235 | if c.NoProgress { 236 | finalStatus = stdout 237 | } 238 | } 239 | 240 | if c.List { 241 | var results []string 242 | 243 | for _, resource := range resourceMap { 244 | results = append(results, resource.GetName()) 245 | } 246 | 247 | sort.Strings(results) 248 | 249 | for _, result := range results { 250 | fmt.Println(result) 251 | } 252 | 253 | return nil 254 | } 255 | 256 | if c.Export != nil { 257 | var resourcesList []service.ResolvedResource 258 | 259 | for _, resource := range resourceMap { 260 | resourcesList = append(resourcesList, resource) 261 | } 262 | 263 | exportData := export.NewData(ref.CanonicalRef(), ref.GetMetadata, resourcesList, verifyChecksumProfile) 264 | 265 | err = c.Export.Export(ctx, os.Stdout, exportData) 266 | if err != nil { 267 | return errors.Wrap(err, "exporting") 268 | } 269 | } 270 | 271 | if c.NoDownload { 272 | return nil 273 | } 274 | 275 | var transfers []*transfer.Transfer 276 | 277 | for localPath, resource := range resourceMap { 278 | xfer, err := transferutil.BuildTransfer( 279 | ctx, 280 | resource, 281 | localPath, 282 | transferutil.TransferOptions{ 283 | Executable: !c.Executable.Match(resource.GetName()).IsEmpty(), 284 | ChecksumVerification: verifyChecksumProfile, 285 | FinalStatus: finalStatus, 286 | }, 287 | ) 288 | if err != nil { 289 | return errors.Wrapf(err, "preparing transfer of %s", resource.GetName()) 290 | } 291 | 292 | transfers = append(transfers, xfer) 293 | } 294 | 295 | sort.Slice(transfers, func(i, j int) bool { 296 | // TODO first order by user arg order 297 | return transfers[i].GetSubject() < transfers[j].GetSubject() 298 | }) 299 | 300 | var pbO io.Writer = stdout 301 | 302 | if c.Runtime.Quiet || c.NoProgress { 303 | pbO = ioutil.Discard 304 | } 305 | 306 | batch := transfer.NewBatch(c.Runtime.Logger(), transfers, c.Parallel, pbO) 307 | 308 | return batch.Transfer(ctx, c.FailFast) 309 | } 310 | -------------------------------------------------------------------------------- /cmd/gget/factory.go: -------------------------------------------------------------------------------- 1 | package gget 2 | 3 | import "github.com/dpb587/gget/pkg/app" 4 | 5 | func NewCommand(app app.Version) *Command { 6 | return &Command{ 7 | Runtime: NewRuntime(app), 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /cmd/gget/runtime.go: -------------------------------------------------------------------------------- 1 | package gget 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "os" 8 | "time" 9 | 10 | "github.com/dpb587/gget/pkg/app" 11 | "github.com/dpb587/gget/pkg/ggetutil" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type Runtime struct { 16 | Help bool `long:"help" short:"h" description:"show documentation of this command"` 17 | Quiet bool `long:"quiet" short:"q" description:"suppress runtime status messages"` 18 | Verbose []bool `long:"verbose" short:"v" description:"increase logging verbosity (multiple)"` 19 | Version *ggetutil.VersionOpt `long:"version" description:"show version of this command (with optional constraint to validate)" optional:"true" optional-value:"*" value-name:"[CONSTRAINT]"` 20 | 21 | app app.Version 22 | logger *logrus.Logger 23 | httpClient *http.Client 24 | } 25 | 26 | func NewRuntime(app app.Version) *Runtime { 27 | return &Runtime{ 28 | app: app, 29 | } 30 | } 31 | 32 | func (r *Runtime) Logger() *logrus.Logger { 33 | if r.logger == nil { 34 | var logLevel logrus.Level 35 | 36 | switch len(r.Verbose) { 37 | case 0: 38 | logLevel = logrus.ErrorLevel 39 | case 1: 40 | logLevel = logrus.WarnLevel 41 | case 2: 42 | logLevel = logrus.InfoLevel 43 | case 3: 44 | logLevel = logrus.DebugLevel 45 | default: 46 | logLevel = logrus.TraceLevel 47 | } 48 | 49 | r.logger = logrus.New() 50 | r.logger.Level = logLevel 51 | r.logger.Out = os.Stderr 52 | 53 | r.logger.Infof("starting %s", r.app.String()) 54 | } 55 | 56 | return r.logger 57 | } 58 | 59 | func (r *Runtime) NewHTTPClient() *http.Client { 60 | return &http.Client{ 61 | Timeout: 30 * time.Second, 62 | Transport: roundTripLogger{ 63 | l: r.Logger(), 64 | rt: &http.Transport{ 65 | Proxy: http.ProxyFromEnvironment, 66 | Dial: (&net.Dialer{ 67 | Timeout: 30 * time.Second, 68 | KeepAlive: 30 * time.Second, 69 | }).Dial, 70 | IdleConnTimeout: 15 * time.Second, 71 | TLSHandshakeTimeout: 15 * time.Second, 72 | ResponseHeaderTimeout: 15 * time.Second, 73 | ExpectContinueTimeout: 5 * time.Second, 74 | }, 75 | ua: r.app.Version(), 76 | }, 77 | } 78 | } 79 | 80 | type roundTripLogger struct { 81 | l *logrus.Logger 82 | rt http.RoundTripper 83 | ua string 84 | } 85 | 86 | func (rtl roundTripLogger) RoundTrip(req *http.Request) (resp *http.Response, err error) { 87 | rtl.l.Debugf("http: %s %s", req.Method, req.URL.String()) 88 | 89 | ua := rtl.ua 90 | if rua := req.Header.Get("user-agent"); rua != "" { 91 | ua = fmt.Sprintf("%s %s", rua, ua) 92 | } 93 | 94 | req.Header.Set("user-agent", ua) 95 | 96 | res, err := rtl.rt.RoundTrip(req) 97 | 98 | if res == nil { 99 | rtl.l.Infof("http: %s %s (response error)", req.Method, req.URL.String()) 100 | } else { 101 | rtl.l.Infof("http: %s %s (status: %s)", req.Method, req.URL.String(), res.Status) 102 | 103 | if v := res.Header.Get("ratelimit-remaining"); v != "" { 104 | rtl.l.Debugf("http: %s %s (ratelimit-remaining: %s)", req.Method, req.URL.String(), v) 105 | } else if v := res.Header.Get("x-ratelimit-remaining"); v != "" { 106 | rtl.l.Debugf("http: %s %s (x-ratelimit-remaining: %s)", req.Method, req.URL.String(), v) 107 | } 108 | } 109 | 110 | return res, err 111 | } 112 | -------------------------------------------------------------------------------- /docs/releases/v0.1.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.1.0 3 | weight: 1000 4 | --- 5 | 6 | Initial Release 7 | -------------------------------------------------------------------------------- /docs/releases/v0.1.1.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.1.1 3 | weight: 1001 4 | --- 5 | 6 | Fixes error from cross-link renames ([#1](https://github.com/dpb587/gget/issues/1)). 7 | -------------------------------------------------------------------------------- /docs/releases/v0.2.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.2.0 3 | weight: 2000 4 | --- 5 | 6 | * Renames `--exec` to `--executable` 7 | * Fixes panic when parsing some release notes for checksums. 8 | * Automate publishing of Docker images on GitHub Packages. 9 | -------------------------------------------------------------------------------- /docs/releases/v0.3.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.3.0 3 | weight: 3000 4 | --- 5 | 6 | * add alpha support for GitLab 7 | * reduce number of GitHub API requests 8 | * improve error reporting and argument parsing 9 | * support verifying checksums based on `*checksums.txt` files 10 | -------------------------------------------------------------------------------- /docs/releases/v0.4.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.4.0 3 | weight: 4000 4 | --- 5 | 6 | * add download summary line showing number of files and repository reference used 7 | * add `--ref-stability=` to support downloading pre-releases 8 | * add `--ref-version=` to support filtering releases for a particular version constraint 9 | * allow `--version` to receive a constraint for verifying compatibility 10 | * add proxy support (from `http_proxy`, `https_proxy`, and `no_proxy`) 11 | * add checksum lookups for additional file conventions 12 | -------------------------------------------------------------------------------- /docs/releases/v0.5.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.5.0 3 | weight: 5000 4 | --- 5 | 6 | * reduce number of GitHub API requests 7 | * add `--export` option for machine-parseable metadata about downloads 8 | * add `--verify-checksum` option to control checksum behavior 9 | * add `--list` flag for listing all matched resources instead of performing downloads 10 | * add `--no-download` flag to stop before performing downloads 11 | * add `--no-progress` flag to avoid live-updating progress output for basic TTYs 12 | * add `--fail-fast` flag to interrupt and immediately stop downloads when a failure is detected 13 | * fix error reporting on interrupted or failed downloads 14 | * deprecate `--show-ref` (replaced by `--export`) and `--show-resources` (replaced by `--list`) 15 | -------------------------------------------------------------------------------- /docs/releases/v0.5.1.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.5.1 3 | weight: 5001 4 | --- 5 | 6 | * fix hanging process after some completed downloads 7 | * start a [website about gget](https://gget.io/) 8 | -------------------------------------------------------------------------------- /docs/releases/v0.5.2.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.5.2 3 | weight: 5002 4 | --- 5 | 6 | * improve support for self-hosted GitHub/GitLab ([#11](https://github.com/dpb587/gget/issues/11)) 7 | -------------------------------------------------------------------------------- /docs/releases/v0.5.5.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.5.5 3 | weight: 5005 4 | --- 5 | 6 | * fix `--cd` behavior when using resource globs ([#19](https://github.com/dpb587/gget/issues/19)) 7 | -------------------------------------------------------------------------------- /docs/releases/v0.6.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.6.0 3 | weight: 6000 4 | --- 5 | 6 | * update dependencies 7 | -------------------------------------------------------------------------------- /docs/releases/v0.6.1.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v0.6.1 3 | weight: 6001 4 | --- 5 | 6 | * fix automated Docker build tagging 7 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dpb587/gget 2 | 3 | go 1.13 4 | 5 | require ( 6 | code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 7 | github.com/Masterminds/semver v1.5.0 8 | github.com/VividCortex/ewma v1.2.0 // indirect 9 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d 10 | github.com/fatih/color v1.13.0 // indirect 11 | github.com/fsnotify/fsnotify v1.5.1 // indirect 12 | github.com/google/go-cmp v0.5.7 // indirect 13 | github.com/google/go-github/v29 v29.0.3 14 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 15 | github.com/hashicorp/go-hclog v1.1.0 // indirect 16 | github.com/hashicorp/go-retryablehttp v0.7.0 // indirect 17 | github.com/jessevdk/go-flags v1.5.0 18 | github.com/kr/pretty v0.3.0 // indirect 19 | github.com/mattn/go-colorable v0.1.12 // indirect 20 | github.com/mitchellh/go-homedir v1.1.0 21 | github.com/onsi/ginkgo v1.16.5 22 | github.com/onsi/gomega v1.18.1 23 | github.com/pkg/errors v0.9.1 24 | github.com/rogpeppe/go-internal v1.8.1 // indirect 25 | github.com/sirupsen/logrus v1.8.1 26 | github.com/stretchr/testify v1.7.0 // indirect 27 | github.com/tidwall/limiter v0.0.0-20181220020158-fcddc63bb521 28 | github.com/vbauerster/mpb/v4 v4.12.2 29 | github.com/xanzy/go-gitlab v0.54.4 30 | golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf // indirect 31 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect 32 | golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 33 | golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect 34 | golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect 35 | google.golang.org/appengine v1.6.7 // indirect 36 | google.golang.org/protobuf v1.27.1 // indirect 37 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 38 | gopkg.in/yaml.v2 v2.4.0 39 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 40 | k8s.io/client-go v11.0.0+incompatible 41 | rsc.io/goversion v1.2.0 42 | ) 43 | -------------------------------------------------------------------------------- /internal/tools/tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package tools 4 | 5 | import ( 6 | // for testing 7 | _ "github.com/onsi/ginkgo" 8 | _ "github.com/onsi/gomega" 9 | ) 10 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/Masterminds/semver" 12 | "github.com/dpb587/gget/cmd/gget" 13 | "github.com/dpb587/gget/pkg/app" 14 | "github.com/dpb587/gget/pkg/service" 15 | "github.com/dpb587/gget/pkg/service/github" 16 | "github.com/jessevdk/go-flags" 17 | "github.com/pkg/errors" 18 | ) 19 | 20 | var ( 21 | appName = "gget" 22 | appOrigin = "github.com/dpb587/gget" 23 | appSemver, appCommit, appBuilt string 24 | ) 25 | 26 | func main() { 27 | v := app.MustVersion(appName, appSemver, appCommit, appBuilt) 28 | cmd := gget.NewCommand(v) 29 | 30 | parser := flags.NewParser(cmd, flags.PassDoubleDash) 31 | 32 | fatal := func(err error) { 33 | if debug, _ := strconv.ParseBool(os.Getenv("DEBUG")); debug { 34 | panic(err) 35 | } 36 | 37 | fmt.Fprintf(os.Stderr, "%s: error: %s\n", parser.Command.Name, err) 38 | 39 | os.Exit(1) 40 | } 41 | 42 | _, err := parser.Parse() 43 | if err != nil { 44 | fatal(err) 45 | } else if cmd.Runtime.Help { 46 | helpBuf := &bytes.Buffer{} 47 | parser.WriteHelp(helpBuf) 48 | help := helpBuf.String() 49 | 50 | // imply origin is required (optional to support --version, -h) 51 | help = strings.Replace(help, "[HOST/OWNER/REPOSITORY[@REF]]", "HOST/OWNER/REPOSITORY[@REF]", -1) 52 | 53 | // join conventional paren groups 54 | help = strings.Replace(help, ") (", "; ", -1) 55 | 56 | fmt.Print(help) 57 | fmt.Printf("\n") 58 | 59 | return 60 | } else if cmd.Runtime.Version != nil { 61 | if cmd.Runtime.Version.IsLatest { 62 | err := func() error { 63 | ref, err := service.ParseRefString(appOrigin) 64 | if err != nil { 65 | return errors.Wrap(err, "parsing app origin") 66 | } 67 | 68 | svc := github.NewService(cmd.Runtime.Logger(), github.NewClientFactory(cmd.Runtime.Logger(), cmd.Runtime.NewHTTPClient)) 69 | res, err := svc.ResolveRef(context.Background(), service.LookupRef{Ref: ref}) 70 | if err != nil { 71 | return errors.Wrap(err, "resolving app origin") 72 | } 73 | 74 | err = cmd.Runtime.Version.UnmarshalFlag(res.CanonicalRef().Ref) 75 | if err != nil { 76 | return errors.Wrap(err, "parsing version constraint") 77 | } 78 | 79 | return nil 80 | }() 81 | if err != nil { 82 | fatal(errors.Wrap(err, "checking latest app version")) 83 | } 84 | } 85 | 86 | ver, err := semver.NewVersion(v.Semver) 87 | if err != nil { 88 | fatal(errors.Wrap(err, "parsing application version")) 89 | } 90 | 91 | if !cmd.Runtime.Quiet { 92 | app.WriteVersion(os.Stdout, os.Args[0], v, len(cmd.Runtime.Verbose)) 93 | } 94 | 95 | if !cmd.Runtime.Version.Constraint.Check(ver) { 96 | if cmd.Runtime.Quiet { 97 | os.Exit(1) 98 | } 99 | 100 | fatal(errors.Wrapf(fmt.Errorf("constraint not met: %s", cmd.Runtime.Version.Constraint.RawValue), "verifying application version")) 101 | } 102 | 103 | return 104 | } 105 | 106 | if err = cmd.Execute(nil); err != nil { 107 | fatal(err) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /pkg/app/version.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type Version struct { 9 | Name string 10 | Semver string 11 | Commit string 12 | Built time.Time 13 | } 14 | 15 | func MustVersion(name string, semver string, commit string, built_ string) Version { 16 | if name == "" { 17 | name = "unknown" 18 | } 19 | 20 | if semver == "" { 21 | semver = "0.0.0+dev" 22 | } 23 | 24 | if commit == "" { 25 | commit = "unknown" 26 | } 27 | 28 | var builtval time.Time 29 | var err error 30 | 31 | if built_ == "" { 32 | builtval = time.Now() 33 | } else { 34 | builtval, err = time.Parse(time.RFC3339, built_) 35 | if err != nil { 36 | panic(fmt.Errorf("cannot parse version time: %s", built_)) 37 | } 38 | } 39 | 40 | return Version{name, semver, commit, builtval} 41 | } 42 | 43 | func (v Version) Version() string { 44 | return fmt.Sprintf("%s/%s", v.Name, v.Semver) 45 | } 46 | 47 | func (v Version) String() string { 48 | return fmt.Sprintf("%s/%s (commit %s; built %s)", v.Name, v.Semver, v.Commit, v.Built.Format(time.RFC3339)) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/app/writer.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "runtime" 8 | "strings" 9 | "time" 10 | 11 | "rsc.io/goversion/version" 12 | ) 13 | 14 | type versionRecord struct { 15 | Type string 16 | Component string 17 | Version string 18 | Metadata []string 19 | } 20 | 21 | func WriteVersion(w io.Writer, self string, v Version, verbosity int) { 22 | if verbosity == 0 { 23 | fmt.Fprintf(w, "%s\n", v.String()) 24 | 25 | return 26 | } 27 | 28 | records := []versionRecord{ 29 | { 30 | Type: "app", 31 | Component: v.Name, 32 | Version: v.Semver, 33 | Metadata: []string{ 34 | fmt.Sprintf("commit %s", v.Commit), 35 | fmt.Sprintf("built %s", v.Built.Format(time.RFC3339)), 36 | }, 37 | }, 38 | { 39 | Type: "runtime", 40 | Component: "go", 41 | Version: runtime.Version(), 42 | Metadata: []string{ 43 | fmt.Sprintf("arch %s", runtime.GOARCH), 44 | fmt.Sprintf("os %s", runtime.GOOS), 45 | }, 46 | }, 47 | } 48 | 49 | if vv, err := version.ReadExe(os.Args[0]); err == nil { 50 | for _, line := range strings.Split(strings.TrimSpace(vv.ModuleInfo), "\n") { 51 | row := strings.Split(line, "\t") 52 | if row[0] != "dep" { 53 | continue 54 | } 55 | 56 | records = append( 57 | records, 58 | versionRecord{ 59 | Type: "dep", 60 | Component: row[1], 61 | Version: row[2], 62 | Metadata: []string{ 63 | fmt.Sprintf("hash %s", row[3]), 64 | }, 65 | }, 66 | ) 67 | } 68 | } 69 | 70 | var f func(versionRecord) string = func(r versionRecord) string { 71 | return fmt.Sprintf("%s\t%s\t%s\n", r.Type, r.Component, r.Version) 72 | } 73 | 74 | if verbosity > 1 { 75 | f = func(r versionRecord) string { 76 | return fmt.Sprintf("%s\t%s\t%s\t%s\n", r.Type, r.Component, r.Version, strings.Join(r.Metadata, "; ")) 77 | } 78 | } 79 | 80 | for _, record := range records { 81 | fmt.Fprintf(w, f(record)) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /pkg/checksum/algorithm.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type Algorithm string 8 | 9 | const MD5 Algorithm = "md5" 10 | const SHA1 Algorithm = "sha1" 11 | const SHA256 Algorithm = "sha256" 12 | const SHA384 Algorithm = "sha384" 13 | const SHA512 Algorithm = "sha512" 14 | 15 | type AlgorithmList []Algorithm 16 | 17 | func (l AlgorithmList) FilterMin(min Algorithm) AlgorithmList { 18 | var found bool 19 | var res AlgorithmList 20 | 21 | for _, a := range l { 22 | res = append(res, a) 23 | 24 | if a == min { 25 | found = true 26 | 27 | break 28 | } 29 | } 30 | 31 | if !found { 32 | return AlgorithmList{} 33 | } 34 | 35 | return res 36 | } 37 | 38 | func (l AlgorithmList) Contains(in Algorithm) bool { 39 | for _, v := range l { 40 | if v == in { 41 | return true 42 | } 43 | } 44 | 45 | return false 46 | } 47 | 48 | func (l AlgorithmList) Intersection(in AlgorithmList) AlgorithmList { 49 | var res AlgorithmList 50 | 51 | for _, exp := range l { 52 | for _, des := range in { 53 | if exp == des { 54 | res = append(res, exp) 55 | 56 | break 57 | } 58 | } 59 | } 60 | 61 | return res 62 | } 63 | 64 | func (l AlgorithmList) Join(sep string) string { 65 | var res []string 66 | 67 | for _, v := range l { 68 | res = append(res, string(v)) 69 | } 70 | 71 | return strings.Join(res, sep) 72 | } 73 | 74 | var AlgorithmsByStrength = AlgorithmList{SHA512, SHA384, SHA256, SHA1, MD5} 75 | -------------------------------------------------------------------------------- /pkg/checksum/all_checksum_selector.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | type AllChecksumSelector struct{} 4 | 5 | var _ ChecksumSelector = AllChecksumSelector{} 6 | 7 | func (AllChecksumSelector) SelectChecksums(in ChecksumList) ChecksumList { 8 | return in 9 | } 10 | -------------------------------------------------------------------------------- /pkg/checksum/checksum.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type Checksum interface { 8 | Algorithm() Algorithm 9 | NewVerifier(context.Context) (*HashVerifier, error) 10 | } 11 | 12 | type ChecksumList []Checksum 13 | 14 | func (l ChecksumList) FilterAlgorithms(algorithms AlgorithmList) ChecksumList { 15 | var res ChecksumList 16 | 17 | for _, c := range l { 18 | for _, a := range algorithms { 19 | if c.Algorithm() == a { 20 | res = append(res, c) 21 | } 22 | } 23 | } 24 | 25 | return res 26 | } 27 | -------------------------------------------------------------------------------- /pkg/checksum/deferred_checksum.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | type deferredChecksum struct { 11 | manager Manager 12 | resource string 13 | algorithm Algorithm 14 | } 15 | 16 | func NewDeferredChecksum(manager Manager, resource string, algorithm Algorithm) Checksum { 17 | return &deferredChecksum{ 18 | manager: manager, 19 | resource: resource, 20 | algorithm: algorithm, 21 | } 22 | } 23 | 24 | func (c deferredChecksum) Algorithm() Algorithm { 25 | return c.algorithm 26 | } 27 | 28 | func (c deferredChecksum) NewVerifier(ctx context.Context) (*HashVerifier, error) { 29 | checksum, err := c.requireChecksum(ctx) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return checksum.NewVerifier(ctx) 35 | } 36 | 37 | func (c *deferredChecksum) requireChecksum(ctx context.Context) (Checksum, error) { 38 | checksums, err := c.manager.GetChecksums(ctx, c.resource, AlgorithmList{c.algorithm}) 39 | if err != nil { 40 | return nil, errors.Wrap(err, "getting deferred checksum") 41 | } else if len(checksums) != 1 { 42 | return nil, fmt.Errorf("expected deferred checksum: %s", c.algorithm) 43 | } 44 | 45 | return checksums[0], nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/checksum/ginkgo_test.go: -------------------------------------------------------------------------------- 1 | package checksum_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestChecksum(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "github.com/dpb587/gget/pkg/checksum") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/checksum/guesser.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha1" 6 | "crypto/sha256" 7 | "crypto/sha512" 8 | "encoding/hex" 9 | "fmt" 10 | "hash" 11 | 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | func MustGuessChecksumHex(expected string) Checksum { 16 | hashHex, err := hex.DecodeString("3b9cd0cd920e355805a6a243c62628dce2bb62fc4c2e0269a824f8589d905d50") 17 | if err != nil { 18 | panic(errors.Wrap(err, "decoding checksum hex")) 19 | } 20 | 21 | cs, err := GuessChecksum(hashHex) 22 | if err != nil { 23 | panic(errors.Wrap(err, "guessing checksum")) 24 | } 25 | 26 | return cs 27 | } 28 | 29 | func GuessChecksum(expected []byte) (Checksum, error) { 30 | var hasher func() hash.Hash 31 | var algorithm Algorithm 32 | 33 | switch len(expected) { 34 | case 16: 35 | algorithm = MD5 36 | hasher = md5.New 37 | case 20: 38 | algorithm = SHA1 39 | hasher = sha1.New 40 | case 32: 41 | algorithm = SHA256 42 | hasher = sha256.New 43 | case 48: 44 | algorithm = SHA384 45 | hasher = sha512.New384 46 | case 64: 47 | algorithm = SHA512 48 | hasher = sha512.New 49 | default: 50 | return nil, fmt.Errorf("unrecognized hash: %s", expected) 51 | } 52 | 53 | return NewHashChecksum(algorithm, expected, hasher), nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/checksum/guesser_test.go: -------------------------------------------------------------------------------- 1 | package checksum_test 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | 10 | "github.com/dpb587/gget/pkg/checksum" 11 | . "github.com/dpb587/gget/pkg/checksum" 12 | ) 13 | 14 | var _ = Describe("Guesser", func() { 15 | It("guesses md5", func() { 16 | hashHex, _ := hex.DecodeString("dd16fe9f76604a7400d5e1fcf88afaca") 17 | 18 | cs, err := GuessChecksum(hashHex) 19 | Expect(err).ToNot(HaveOccurred()) 20 | Expect(cs.Algorithm()).To(Equal(checksum.MD5)) 21 | 22 | h, err := cs.NewVerifier(context.Background()) 23 | Expect(err).ToNot(HaveOccurred()) 24 | h.Write([]byte("hashable")) 25 | Expect(h.Verify()).ToNot(HaveOccurred()) 26 | }) 27 | 28 | It("guesses sha1", func() { 29 | hashHex, _ := hex.DecodeString("705d0123108e62b9a94842986e4f12c7ef0a9239") 30 | 31 | cs, err := GuessChecksum(hashHex) 32 | Expect(err).ToNot(HaveOccurred()) 33 | Expect(cs.Algorithm()).To(Equal(checksum.SHA1)) 34 | 35 | h, err := cs.NewVerifier(context.Background()) 36 | Expect(err).ToNot(HaveOccurred()) 37 | h.Write([]byte("hashable")) 38 | Expect(h.Verify()).ToNot(HaveOccurred()) 39 | }) 40 | 41 | It("guesses sha256", func() { 42 | hashHex, _ := hex.DecodeString("3b9cd0cd920e355805a6a243c62628dce2bb62fc4c2e0269a824f8589d905d50") 43 | 44 | cs, err := GuessChecksum(hashHex) 45 | Expect(err).ToNot(HaveOccurred()) 46 | Expect(cs.Algorithm()).To(Equal(checksum.SHA256)) 47 | 48 | h, err := cs.NewVerifier(context.Background()) 49 | Expect(err).ToNot(HaveOccurred()) 50 | h.Write([]byte("hashable")) 51 | Expect(h.Verify()).ToNot(HaveOccurred()) 52 | }) 53 | 54 | It("guesses sha512", func() { 55 | hashHex, _ := hex.DecodeString("768dd75ae44b3d5537d047ef454c15833326602e568c1dbc31e5198c0e9a76380b8e392df6625adcb1f1411ad520c15f514008f4306196059dbe726a9e64c4da") 56 | 57 | cs, err := GuessChecksum(hashHex) 58 | Expect(err).ToNot(HaveOccurred()) 59 | Expect(cs.Algorithm()).To(Equal(checksum.SHA512)) 60 | 61 | h, err := cs.NewVerifier(context.Background()) 62 | Expect(err).ToNot(HaveOccurred()) 63 | h.Write([]byte("hashable")) 64 | Expect(h.Verify()).ToNot(HaveOccurred()) 65 | }) 66 | 67 | It("errors for unknown", func() { 68 | hashHex, _ := hex.DecodeString("dead") 69 | 70 | _, err := GuessChecksum(hashHex) 71 | Expect(err).To(HaveOccurred()) 72 | Expect(err.Error()).To(ContainSubstring("unrecognized hash")) 73 | }) 74 | }) 75 | -------------------------------------------------------------------------------- /pkg/checksum/hash_checksum.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "context" 5 | "hash" 6 | ) 7 | 8 | type hashChecksum struct { 9 | algorithm Algorithm 10 | expected []byte 11 | hasher func() hash.Hash 12 | } 13 | 14 | var _ Checksum = hashChecksum{} 15 | 16 | func NewHashChecksum(algorithm Algorithm, expected []byte, hasher func() hash.Hash) Checksum { 17 | return hashChecksum{ 18 | algorithm: algorithm, 19 | expected: expected, 20 | hasher: hasher, 21 | } 22 | } 23 | 24 | func (c hashChecksum) Algorithm() Algorithm { 25 | return c.algorithm 26 | } 27 | 28 | func (c hashChecksum) NewVerifier(ctx context.Context) (*HashVerifier, error) { 29 | return NewHashVerifier(c.algorithm, c.expected, c.hasher()), nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/checksum/hash_verification_error.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import "fmt" 4 | 5 | type HashVerificationError struct { 6 | algorithm Algorithm 7 | expected []byte 8 | actual []byte 9 | } 10 | 11 | var _ error = HashVerificationError{} 12 | 13 | func (err HashVerificationError) Error() string { 14 | return fmt.Sprintf("expected %s checksum %x, but found %x", err.algorithm, err.expected, err.actual) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/checksum/hash_verifier.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "bytes" 5 | "hash" 6 | "io" 7 | ) 8 | 9 | type HashVerifier struct { 10 | algorithm Algorithm 11 | expected []byte 12 | hasher hash.Hash 13 | } 14 | 15 | var _ io.Writer = &HashVerifier{} 16 | 17 | func NewHashVerifier(algorithm Algorithm, expected []byte, hasher hash.Hash) *HashVerifier { 18 | return &HashVerifier{ 19 | algorithm: algorithm, 20 | expected: expected, 21 | hasher: hasher, 22 | } 23 | } 24 | 25 | func (hv *HashVerifier) Algorithm() Algorithm { 26 | return hv.algorithm 27 | } 28 | 29 | func (hv *HashVerifier) Expected() []byte { 30 | return hv.expected[0:] 31 | } 32 | 33 | func (hv *HashVerifier) Write(p []byte) (int, error) { 34 | return hv.hasher.Write(p) 35 | } 36 | 37 | func (hv *HashVerifier) Verify() error { 38 | actual := hv.hasher.Sum(nil) 39 | 40 | if bytes.Compare(hv.expected, actual) == 0 { 41 | return nil 42 | } 43 | 44 | return &HashVerificationError{ 45 | algorithm: hv.algorithm, 46 | expected: hv.expected, 47 | actual: actual, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/checksum/in_memory_alias_manager.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import "context" 4 | 5 | // InMemoryAliasManager enforces a specific file name is used for any added checksums. 6 | // 7 | // Namely useful for name-based, *.sha256-type files are used and the contents may have been generated using a different 8 | // file name, but there is high certainty about the subject. The caller should always be using the expected name. 9 | type InMemoryAliasManager struct { 10 | manager WriteableManager 11 | resource string 12 | } 13 | 14 | var _ WriteableManager = &InMemoryAliasManager{} 15 | 16 | func NewInMemoryAliasManager(resource string) WriteableManager { 17 | return &InMemoryAliasManager{ 18 | manager: NewInMemoryManager(), 19 | resource: resource, 20 | } 21 | } 22 | 23 | func (m *InMemoryAliasManager) GetChecksums(ctx context.Context, resource string, algos AlgorithmList) (ChecksumList, error) { 24 | if m.resource != resource { 25 | return nil, nil 26 | } 27 | 28 | return m.manager.GetChecksums(ctx, m.resource, algos) 29 | } 30 | 31 | func (m *InMemoryAliasManager) AddChecksum(_ string, checksum Checksum) { 32 | m.manager.AddChecksum(m.resource, checksum) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/checksum/in_memory_alias_manager_test.go: -------------------------------------------------------------------------------- 1 | package checksum_test 2 | 3 | import ( 4 | "encoding/hex" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | . "github.com/dpb587/gget/pkg/checksum" 10 | ) 11 | 12 | var _ = Describe("InMemoryAliasManager", func() { 13 | It("enforces a name", func() { 14 | subject := NewInMemoryAliasManager("my-test-file.zip") 15 | 16 | hashHex, _ := hex.DecodeString("3b9cd0cd920e355805a6a243c62628dce2bb62fc4c2e0269a824f8589d905d50") 17 | cs, _ := GuessChecksum(hashHex) 18 | 19 | subject.AddChecksum("alternative-name", cs) 20 | 21 | checksums, err := subject.GetChecksums(nil, "my-test-file.zip", nil) 22 | Expect(err).NotTo(HaveOccurred()) 23 | Expect(checksums).To(HaveLen(1)) 24 | Expect(checksums[0].Algorithm()).To(Equal(SHA256)) 25 | 26 | checksums, err = subject.GetChecksums(nil, "alternative-name", nil) 27 | Expect(err).NotTo(HaveOccurred()) 28 | Expect(checksums).To(HaveLen(0)) 29 | }) 30 | }) 31 | -------------------------------------------------------------------------------- /pkg/checksum/in_memory_manager.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type InMemoryManager struct { 8 | resourceChecksums map[string]ChecksumList 9 | } 10 | 11 | var _ Manager = &InMemoryManager{} 12 | 13 | func NewInMemoryManager() WriteableManager { 14 | return &InMemoryManager{ 15 | resourceChecksums: map[string]ChecksumList{}, 16 | } 17 | } 18 | 19 | func (m *InMemoryManager) GetChecksums(ctx context.Context, resource string, algos AlgorithmList) (ChecksumList, error) { 20 | res, found := m.resourceChecksums[resource] 21 | if !found { 22 | return nil, nil 23 | } 24 | 25 | if algos != nil { 26 | res = res.FilterAlgorithms(algos) 27 | } 28 | 29 | return res, nil 30 | } 31 | 32 | func (m *InMemoryManager) Resources() []string { 33 | var res []string 34 | 35 | for resource := range m.resourceChecksums { 36 | res = append(res, resource) 37 | } 38 | 39 | return res 40 | } 41 | 42 | func (m *InMemoryManager) AddChecksum(resource string, checksum Checksum) { 43 | m.resourceChecksums[resource] = append(m.resourceChecksums[resource], checksum) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/checksum/manager.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import "context" 4 | 5 | type Manager interface { 6 | GetChecksums(ctx context.Context, resource string, algos AlgorithmList) (ChecksumList, error) 7 | } 8 | 9 | type WriteableManager interface { 10 | Manager 11 | AddChecksum(string, Checksum) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/checksum/multi_manager.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type MultiManager struct { 10 | managers []Manager 11 | } 12 | 13 | var _ Manager = MultiManager{} 14 | 15 | func NewMultiManager(managers ...Manager) Manager { 16 | return MultiManager{ 17 | managers: managers, 18 | } 19 | } 20 | 21 | func (m MultiManager) GetChecksums(ctx context.Context, resource string, algos AlgorithmList) (ChecksumList, error) { 22 | var res ChecksumList 23 | 24 | for managerIdx, manager := range m.managers { 25 | checksums, err := manager.GetChecksums(ctx, resource, algos) 26 | if err != nil { 27 | return nil, errors.Wrapf(err, "manager %d", managerIdx) 28 | } 29 | 30 | res = append(res, checksums...) 31 | } 32 | 33 | return res.FilterAlgorithms(algos), nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/checksum/parser/deferred_manager.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/ioutil" 7 | "sync" 8 | 9 | "github.com/dpb587/gget/pkg/checksum" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | type DeferredManager struct { 14 | manager checksum.WriteableManager 15 | expectedAlgorithms checksum.AlgorithmList 16 | opener func(context.Context) (io.ReadCloser, error) 17 | 18 | loaded bool 19 | mutex sync.RWMutex 20 | } 21 | 22 | var _ checksum.Manager = &DeferredManager{} 23 | 24 | func NewDeferredManager(manager checksum.WriteableManager, expectedAlgorithms checksum.AlgorithmList, opener func(context.Context) (io.ReadCloser, error)) checksum.Manager { 25 | return &DeferredManager{ 26 | manager: manager, 27 | expectedAlgorithms: expectedAlgorithms, 28 | opener: opener, 29 | } 30 | } 31 | 32 | func (m *DeferredManager) GetChecksums(ctx context.Context, resource string, algos checksum.AlgorithmList) (checksum.ChecksumList, error) { 33 | if len(m.expectedAlgorithms) > 0 { 34 | if len(m.expectedAlgorithms.Intersection(algos)) == 0 { 35 | // avoid loading if we don't expect it to be found 36 | return nil, nil 37 | } 38 | } 39 | 40 | err := m.requireLoad(ctx) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return m.manager.GetChecksums(ctx, resource, algos) 46 | } 47 | 48 | func (m *DeferredManager) requireLoad(ctx context.Context) error { 49 | m.mutex.Lock() 50 | defer m.mutex.Unlock() 51 | 52 | if m.loaded { 53 | return nil 54 | } 55 | 56 | fh, err := m.opener(ctx) 57 | if err != nil { 58 | return errors.Wrap(err, "opening") 59 | } 60 | 61 | defer fh.Close() 62 | 63 | buf, err := ioutil.ReadAll(fh) 64 | if err != nil { 65 | return errors.Wrap(err, "reading") 66 | } 67 | 68 | ImportLines(m.manager, buf) 69 | 70 | m.loaded = true 71 | 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/checksum/parser/ginkgo_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestParser(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "github.com/dpb587/gget/pkg/checksum/parser") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/checksum/parser/import_lines.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/hex" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/dpb587/gget/pkg/checksum" 9 | ) 10 | 11 | var lines = regexp.MustCompile(`^([a-f0-9]{40,64})\s+([^\s]+)$`) 12 | 13 | func ImportLines(m checksum.WriteableManager, content []byte) { 14 | checksums := strings.Split(strings.TrimSpace(string(content)), "\n") 15 | 16 | for _, checksumLine := range checksums { 17 | checksumSplit := strings.Fields(strings.TrimSpace(checksumLine)) 18 | if len(checksumSplit) != 2 { 19 | continue 20 | } 21 | 22 | if len(checksumSplit[0]) < 16 { 23 | continue 24 | } 25 | 26 | hashBytes, err := hex.DecodeString(checksumSplit[0]) 27 | if err != nil { 28 | // TODO log? 29 | continue 30 | } 31 | 32 | checksum, err := checksum.GuessChecksum(hashBytes) 33 | if err != nil { 34 | // TODO log? 35 | continue 36 | } 37 | 38 | m.AddChecksum(checksumSplit[1], checksum) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pkg/checksum/parser/markdown.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "github.com/dpb587/gget/pkg/checksum" 4 | 5 | func ImportMarkdown(m checksum.WriteableManager, content []byte) { 6 | ImportMarkdownCodefence(m, content) 7 | ImportMarkdownCodeindent(m, content) 8 | } 9 | -------------------------------------------------------------------------------- /pkg/checksum/parser/markdown_codefence.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/dpb587/gget/pkg/checksum" 7 | ) 8 | 9 | var markdownCodefence = regexp.MustCompile("(?mU)```([^`]+)```") 10 | 11 | func ImportMarkdownCodefence(m checksum.WriteableManager, content []byte) { 12 | contentSubmatches := markdownCodefence.FindAllSubmatch(content, -1) 13 | 14 | if len(contentSubmatches) == 0 { 15 | return 16 | } 17 | 18 | for _, contentSubmatch := range contentSubmatches { 19 | ImportLines(m, contentSubmatch[1]) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pkg/checksum/parser/markdown_codeindent.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/hex" 5 | "regexp" 6 | 7 | "github.com/dpb587/gget/pkg/checksum" 8 | ) 9 | 10 | var codeindent = regexp.MustCompile(" ([a-f0-9]{40,64})\\s+([^\\s]+)") 11 | 12 | func ImportMarkdownCodeindent(m checksum.WriteableManager, content []byte) { 13 | contentSubmatch := codeindent.FindAllStringSubmatch(string(content), -1) 14 | 15 | if len(contentSubmatch) == 0 { 16 | return 17 | } 18 | 19 | for _, match := range contentSubmatch { 20 | hashBytes, err := hex.DecodeString(match[1]) 21 | if err != nil { 22 | // TODO log? 23 | continue 24 | } 25 | 26 | checksum, err := checksum.GuessChecksum(hashBytes) 27 | if err != nil { 28 | // TODO log? 29 | continue 30 | } 31 | 32 | m.AddChecksum(match[2], checksum) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pkg/checksum/parser/markdown_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | 10 | "github.com/dpb587/gget/pkg/checksum" 11 | . "github.com/dpb587/gget/pkg/checksum/parser" 12 | ) 13 | 14 | var _ = Describe("Markdown", func() { 15 | var csm checksum.WriteableManager 16 | var ctx context.Context 17 | 18 | BeforeEach(func() { 19 | csm = checksum.NewInMemoryManager() 20 | }) 21 | 22 | Context("code fences", func() { 23 | It("parses", func() { 24 | ImportMarkdownCodefence(csm, []byte(strings.Join([]string{ 25 | "dear release note readers. here are your checksums", 26 | "```", 27 | "9534cebfed045f466d446f45ff1d76e38aa94941ccdbbcd8a8b82e51657a579e gget-0.1.1-darwin-amd64", 28 | "9d61c2edcdb8ed71d58d94970d7ef4aeacbe1ac4bce4aecb06e2f3d804caee4b gget-0.1.1-linux-amd64", 29 | "```", 30 | }, "\n"))) 31 | 32 | cs, err := csm.GetChecksums(ctx, "gget-0.1.1-darwin-amd64", checksum.AlgorithmsByStrength) 33 | Expect(err).ToNot(HaveOccurred()) 34 | Expect(cs).To(HaveLen(1)) 35 | 36 | csv, err := cs[0].NewVerifier(ctx) 37 | csv.Write([]byte("gget-0.1.1-darwin-amd64")) 38 | Expect(csv.Verify()).ToNot(HaveOccurred()) 39 | 40 | cs, err = csm.GetChecksums(ctx, "gget-0.1.1-linux-amd64", checksum.AlgorithmsByStrength) 41 | Expect(err).ToNot(HaveOccurred()) 42 | Expect(cs).To(HaveLen(1)) 43 | 44 | csv, err = cs[0].NewVerifier(ctx) 45 | csv.Write([]byte("gget-0.1.1-linux-amd64")) 46 | Expect(csv.Verify()).ToNot(HaveOccurred()) 47 | }) 48 | 49 | It("ignores unexpected data", func() { 50 | ImportMarkdownCodefence(csm, []byte(strings.Join([]string{ 51 | "dear release note readers. here are your checksums", 52 | "", 53 | "```", 54 | "other", 55 | "```", 56 | "", 57 | "some other note", 58 | "", 59 | "```", 60 | "9534cebfed045f466d446f45ff1d76e38aa94941ccdbbcd8a8b82e51657a579e gget-0.1.1-darwin-amd64", 61 | "```", 62 | }, "\n"))) 63 | 64 | cs, err := csm.GetChecksums(ctx, "gget-0.1.1-darwin-amd64", checksum.AlgorithmsByStrength) 65 | Expect(err).ToNot(HaveOccurred()) 66 | Expect(cs).To(HaveLen(1)) 67 | 68 | csv, err := cs[0].NewVerifier(ctx) 69 | csv.Write([]byte("gget-0.1.1-darwin-amd64")) 70 | Expect(csv.Verify()).ToNot(HaveOccurred()) 71 | }) 72 | }) 73 | 74 | Context("code indent", func() { 75 | It("parses", func() { 76 | ImportMarkdownCodeindent(csm, []byte(strings.Join([]string{ 77 | "dear release note readers. here are your checksums", 78 | "", 79 | " 9534cebfed045f466d446f45ff1d76e38aa94941ccdbbcd8a8b82e51657a579e gget-0.1.1-darwin-amd64", 80 | " 9d61c2edcdb8ed71d58d94970d7ef4aeacbe1ac4bce4aecb06e2f3d804caee4b gget-0.1.1-linux-amd64", 81 | }, "\n"))) 82 | 83 | cs, err := csm.GetChecksums(ctx, "gget-0.1.1-darwin-amd64", checksum.AlgorithmsByStrength) 84 | Expect(err).ToNot(HaveOccurred()) 85 | Expect(cs).To(HaveLen(1)) 86 | 87 | csv, err := cs[0].NewVerifier(ctx) 88 | csv.Write([]byte("gget-0.1.1-darwin-amd64")) 89 | Expect(csv.Verify()).ToNot(HaveOccurred()) 90 | 91 | cs, err = csm.GetChecksums(ctx, "gget-0.1.1-linux-amd64", checksum.AlgorithmsByStrength) 92 | Expect(err).ToNot(HaveOccurred()) 93 | Expect(cs).To(HaveLen(1)) 94 | 95 | csv, err = cs[0].NewVerifier(ctx) 96 | csv.Write([]byte("gget-0.1.1-linux-amd64")) 97 | Expect(csv.Verify()).ToNot(HaveOccurred()) 98 | }) 99 | 100 | It("ignores unexpected data", func() { 101 | ImportMarkdownCodeindent(csm, []byte(strings.Join([]string{ 102 | "dear release note readers. here are your checksums", 103 | "", 104 | " other", 105 | "", 106 | "some other note", 107 | "", 108 | " 9534cebfed045f466d446f45ff1d76e38aa94941ccdbbcd8a8b82e51657a579e gget-0.1.1-darwin-amd64", 109 | "```", 110 | }, "\n"))) 111 | 112 | cs, err := csm.GetChecksums(ctx, "gget-0.1.1-darwin-amd64", checksum.AlgorithmsByStrength) 113 | Expect(err).ToNot(HaveOccurred()) 114 | Expect(cs).To(HaveLen(1)) 115 | 116 | csv, err := cs[0].NewVerifier(ctx) 117 | csv.Write([]byte("gget-0.1.1-darwin-amd64")) 118 | Expect(csv.Verify()).ToNot(HaveOccurred()) 119 | }) 120 | }) 121 | }) 122 | -------------------------------------------------------------------------------- /pkg/checksum/selector.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | type ChecksumSelector interface { 4 | SelectChecksums(ChecksumList) ChecksumList 5 | } 6 | -------------------------------------------------------------------------------- /pkg/checksum/strongest_checksum_selector.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | type StrongestChecksumSelector struct{} 4 | 5 | var _ ChecksumSelector = StrongestChecksumSelector{} 6 | 7 | func (StrongestChecksumSelector) SelectChecksums(in ChecksumList) ChecksumList { 8 | for _, algo := range AlgorithmsByStrength { 9 | for _, cs := range in { 10 | if cs.Algorithm() == algo { 11 | return ChecksumList{cs} 12 | } 13 | } 14 | } 15 | 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /pkg/checksum/verification_profile.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | type VerificationProfile struct { 4 | Required bool 5 | Acceptable AlgorithmList 6 | Selector ChecksumSelector 7 | } 8 | -------------------------------------------------------------------------------- /pkg/cli/opt/constraint.go: -------------------------------------------------------------------------------- 1 | package opt 2 | 3 | import ( 4 | "github.com/Masterminds/semver" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | type Constraint struct { 9 | *semver.Constraints 10 | RawValue string 11 | } 12 | 13 | func (o *Constraint) UnmarshalFlag(data string) error { 14 | con, err := semver.NewConstraint(data) 15 | if err != nil { 16 | return errors.Wrap(err, "parsing version constraint") 17 | } 18 | 19 | o.Constraints = con 20 | o.RawValue = data 21 | 22 | return nil 23 | } 24 | 25 | type ConstraintList []*Constraint 26 | 27 | func (cl ConstraintList) Constraints() []*semver.Constraints { 28 | var res []*semver.Constraints 29 | 30 | for _, v := range cl { 31 | res = append(res, v.Constraints) 32 | } 33 | 34 | return res 35 | } 36 | -------------------------------------------------------------------------------- /pkg/cli/opt/export.go: -------------------------------------------------------------------------------- 1 | package opt 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dpb587/gget/pkg/export" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type Export struct { 12 | export.Exporter 13 | } 14 | 15 | func (e *Export) UnmarshalFlag(data string) error { 16 | var exporter export.Exporter 17 | var template string 18 | 19 | if data == "json" { 20 | exporter = export.JSONExporter{} 21 | } else if strings.HasPrefix(data, "jsonpath=") { 22 | exporter = &export.JSONPathExporter{} 23 | template = strings.TrimPrefix(data, "jsonpath=") 24 | } else if data == "jsonpath" { 25 | return fmt.Errorf("jsonpath must include a template (e.g. `jsonpath='{.origin.ref}'`)") 26 | } else if data == "yaml" { 27 | exporter = export.YAMLExporter{} 28 | } else if data == "plain" { 29 | exporter = export.PlainExporter{} 30 | // } else if strings.HasPrefix(data, "go-template=") { 31 | // exporter = &export.GoTemplateExporter{} 32 | // template = strings.TrimPrefix(data, "go-template=") 33 | // } else if data == "go-template" { 34 | // return fmt.Errorf("go-template must include a template (e.g. `go-template='{{.Origin.Ref}}'`") 35 | } else { 36 | return fmt.Errorf("unsupported export type: %s", data) 37 | } 38 | 39 | if template != "" { 40 | tex, ok := exporter.(export.TemplatedExporter) 41 | if !ok { 42 | panic(fmt.Errorf("exporter does not support templating: %T", exporter)) 43 | } 44 | 45 | err := tex.ParseTemplate(template) 46 | if err != nil { 47 | return errors.Wrap(err, "parsing export template") 48 | } 49 | } 50 | 51 | *e = Export{Exporter: exporter} 52 | 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/cli/opt/ref.go: -------------------------------------------------------------------------------- 1 | package opt 2 | 3 | import ( 4 | "github.com/dpb587/gget/pkg/service" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | type Ref service.Ref 9 | 10 | func (o *Ref) UnmarshalFlag(data string) error { 11 | parsed, err := service.ParseRefString(data) 12 | if err != nil { 13 | return errors.Wrap(err, "parsing ref option") 14 | } 15 | 16 | *o = Ref(parsed) 17 | 18 | return nil 19 | } 20 | -------------------------------------------------------------------------------- /pkg/cli/opt/resource_matcher.go: -------------------------------------------------------------------------------- 1 | package opt 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type ResourceMatcher string 10 | 11 | func (o *ResourceMatcher) Match(remote string) bool { 12 | match, _ := filepath.Match(string(*o), remote) 13 | 14 | return match 15 | } 16 | 17 | func (o *ResourceMatcher) Validate() error { 18 | _, err := filepath.Match(string(*o), "test") 19 | if err != nil { 20 | return errors.Wrap(err, "expected valid Resource matcher") 21 | } 22 | 23 | return nil 24 | } 25 | 26 | type ResourceMatcherList []ResourceMatcher 27 | 28 | func (o ResourceMatcherList) Match(remote string) ResourceMatcherList { 29 | var res ResourceMatcherList 30 | 31 | for _, m := range o { 32 | if !m.Match(remote) { 33 | continue 34 | } 35 | 36 | res = append(res, m) 37 | } 38 | 39 | return res 40 | } 41 | 42 | func (o ResourceMatcherList) IsEmpty() bool { 43 | return len(o) == 0 44 | } 45 | -------------------------------------------------------------------------------- /pkg/cli/opt/resource_transfer.go: -------------------------------------------------------------------------------- 1 | package opt 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | ) 7 | 8 | type ResourceTransfer struct { 9 | RemoteMatch ResourceMatcher 10 | LocalDir string 11 | LocalName string 12 | } 13 | 14 | type ResourceTransferList []ResourceTransfer 15 | 16 | func (o *ResourceTransfer) LocalPath() string { 17 | return filepath.Join(o.LocalDir, o.LocalName) 18 | } 19 | 20 | func (o *ResourceTransfer) Resolve(remote string) (ResourceTransfer, bool) { 21 | match := o.RemoteMatch.Match(remote) 22 | if !match { 23 | return ResourceTransfer{}, false 24 | } 25 | 26 | res := ResourceTransfer{ 27 | RemoteMatch: ResourceMatcher(remote), 28 | LocalDir: o.LocalDir, 29 | LocalName: o.LocalName, 30 | } 31 | 32 | if res.LocalName == "" { 33 | res.LocalName = remote 34 | } 35 | 36 | return res, true 37 | } 38 | 39 | func (o *ResourceTransfer) UnmarshalFlag(data string) error { 40 | dataSplit := strings.SplitN(data, "=", 2) 41 | localPath := "" 42 | 43 | if len(dataSplit) == 2 { 44 | o.RemoteMatch = ResourceMatcher(dataSplit[1]) 45 | localPath = dataSplit[0] 46 | } else { 47 | o.RemoteMatch = ResourceMatcher(dataSplit[0]) 48 | } 49 | 50 | o.LocalDir, o.LocalName = filepath.Split(localPath) 51 | 52 | if err := o.RemoteMatch.Validate(); err != nil { 53 | return err 54 | } 55 | 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /pkg/cli/opt/verify_checksum.go: -------------------------------------------------------------------------------- 1 | package opt 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/dpb587/gget/pkg/checksum" 8 | ) 9 | 10 | type VerifyChecksum []string 11 | 12 | func (vR VerifyChecksum) Profile() (checksum.VerificationProfile, error) { 13 | var standaloneValues []string 14 | var usedCustomAlgos bool 15 | 16 | res := checksum.VerificationProfile{ 17 | Selector: checksum.StrongestChecksumSelector{}, 18 | } 19 | 20 | v := vR 21 | 22 | for _, data := range v { 23 | if data == "required" { // modifier 24 | res.Required = true 25 | } else if data == "all" { // modifier 26 | res.Selector = checksum.AllChecksumSelector{} 27 | } else if data == "auto" { // standalone 28 | // explicit default 29 | standaloneValues = append(standaloneValues, data) 30 | } else if data == "none" { 31 | res.Acceptable = checksum.AlgorithmList{} 32 | standaloneValues = append(standaloneValues, data) 33 | usedCustomAlgos = true 34 | } else if strings.HasSuffix(data, "-min") { 35 | algo := checksum.Algorithm(strings.TrimSuffix(data, "-min")) 36 | 37 | add := checksum.AlgorithmsByStrength.FilterMin(algo) 38 | if len(add) == 0 { 39 | return checksum.VerificationProfile{}, fmt.Errorf("unsupported algorithm: %s", algo) 40 | } 41 | 42 | res.Acceptable = append(res.Acceptable, add...) 43 | res.Required = true 44 | 45 | usedCustomAlgos = true 46 | } else { 47 | algo := checksum.Algorithm(data) 48 | 49 | if !checksum.AlgorithmsByStrength.Contains(algo) { 50 | return checksum.VerificationProfile{}, fmt.Errorf("unsupported algorithm: %s", data) 51 | } 52 | 53 | res.Acceptable = append(res.Acceptable, algo) 54 | res.Required = true 55 | 56 | usedCustomAlgos = true 57 | } 58 | } 59 | 60 | if len(standaloneValues) > 1 { 61 | return checksum.VerificationProfile{}, fmt.Errorf("standalone value combined with others: %s", strings.Join(standaloneValues, ", ")) 62 | } 63 | 64 | if !usedCustomAlgos { 65 | res.Acceptable = checksum.AlgorithmsByStrength 66 | } 67 | 68 | return res, nil 69 | } 70 | -------------------------------------------------------------------------------- /pkg/export/data.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/dpb587/gget/pkg/checksum" 9 | "github.com/dpb587/gget/pkg/service" 10 | ) 11 | 12 | type metadataGetterFunc func(ctx context.Context) (service.RefMetadata, error) 13 | 14 | type Data struct { 15 | origin service.Ref 16 | metadataGetter metadataGetterFunc 17 | resources []service.ResolvedResource 18 | checksumVerification checksum.VerificationProfile 19 | 20 | metadata service.RefMetadata 21 | } 22 | 23 | func NewData(origin service.Ref, metadataGetter metadataGetterFunc, resources []service.ResolvedResource, checksumVerification checksum.VerificationProfile) *Data { 24 | return &Data{ 25 | origin: origin, 26 | metadataGetter: metadataGetter, 27 | resources: resources, 28 | checksumVerification: checksumVerification, 29 | } 30 | } 31 | 32 | func (d *Data) Origin() service.Ref { 33 | return d.origin 34 | } 35 | 36 | func (d *Data) Metadata(ctx context.Context) (service.RefMetadata, error) { 37 | if d.metadata == nil { 38 | metadata, err := d.metadataGetter(ctx) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | if metadata == nil { 44 | // avoid empty re-lookups 45 | metadata = service.RefMetadata{} 46 | } 47 | 48 | // deterministic 49 | sort.Slice(metadata, func(i, j int) bool { 50 | return strings.Compare(metadata[i].Name, metadata[j].Name) < 0 51 | }) 52 | 53 | d.metadata = metadata 54 | } 55 | 56 | return d.metadata, nil 57 | } 58 | 59 | func (d *Data) Resources() []service.ResolvedResource { 60 | return d.resources 61 | } 62 | -------------------------------------------------------------------------------- /pkg/export/exporter.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | type Exporter interface { 9 | Export(ctx context.Context, w io.Writer, data *Data) error 10 | } 11 | 12 | type TemplatedExporter interface { 13 | ParseTemplate(text string) error 14 | } 15 | -------------------------------------------------------------------------------- /pkg/export/go_template_exporter.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "text/template" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type GoTemplateExporter struct { 12 | template *template.Template 13 | } 14 | 15 | var _ Exporter = &GoTemplateExporter{} 16 | var _ TemplatedExporter = &GoTemplateExporter{} 17 | 18 | func (e *GoTemplateExporter) ParseTemplate(text string) error { 19 | tmpl, err := template.New("export").Parse(text) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | e.template = tmpl 25 | 26 | return nil 27 | } 28 | 29 | func (e *GoTemplateExporter) Export(ctx context.Context, w io.Writer, data *Data) error { 30 | // TODO lazy load and helper methods 31 | res, err := newMarshalData(ctx, data) 32 | if err != nil { 33 | return errors.Wrap(err, "preparing export") 34 | } 35 | 36 | err = e.template.Execute(w, res) 37 | if err != nil { 38 | return errors.Wrap(err, "executing template") 39 | } 40 | 41 | w.Write([]byte("\n")) 42 | 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/export/json_exporter.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type JSONExporter struct{} 12 | 13 | var _ Exporter = JSONExporter{} 14 | 15 | func (e JSONExporter) Export(ctx context.Context, w io.Writer, data *Data) error { 16 | res, err := newMarshalData(ctx, data) 17 | if err != nil { 18 | return errors.Wrap(err, "preparing export") 19 | } 20 | 21 | buf, err := json.MarshalIndent(res, "", " ") 22 | if err != nil { 23 | return errors.Wrap(err, "marshalling") 24 | } 25 | 26 | w.Write(buf) 27 | w.Write([]byte("\n")) 28 | 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/export/jsonpath_exporter.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/pkg/errors" 8 | "k8s.io/client-go/util/jsonpath" 9 | ) 10 | 11 | type JSONPathExporter struct { 12 | template *jsonpath.JSONPath 13 | } 14 | 15 | var _ Exporter = &JSONPathExporter{} 16 | var _ TemplatedExporter = &JSONPathExporter{} 17 | 18 | func (e *JSONPathExporter) ParseTemplate(text string) error { 19 | e.template = jsonpath.New("export") 20 | 21 | return e.template.Parse(text) 22 | } 23 | 24 | func (e *JSONPathExporter) Export(ctx context.Context, w io.Writer, data *Data) error { 25 | // TODO lazy load and helper methods 26 | res, err := newMarshalData(ctx, data) 27 | if err != nil { 28 | return errors.Wrap(err, "preparing export") 29 | } 30 | 31 | err = e.template.Execute(w, res) 32 | if err != nil { 33 | return errors.Wrap(err, "executing template") 34 | } 35 | 36 | w.Write([]byte("\n")) 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/export/marshal_data.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "strings" 8 | 9 | "github.com/dpb587/gget/pkg/service" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | func newMarshalData(ctx context.Context, data *Data) (marshalData, error) { 14 | origin := data.Origin() 15 | 16 | res := marshalData{ 17 | Origin: marshalDataOrigin{ 18 | String: origin.String(), 19 | Service: origin.Service, 20 | Server: origin.Server, 21 | Owner: origin.Owner, 22 | Repository: origin.Repository, 23 | Ref: origin.Ref, 24 | }, 25 | } 26 | 27 | metadata, err := data.Metadata(ctx) 28 | if err != nil { 29 | return marshalData{}, errors.Wrap(err, "getting metadata") 30 | } 31 | 32 | for _, metadatum := range metadata { 33 | res.Metadata = append( 34 | res.Metadata, 35 | marshalDataMetadatum{ 36 | Key: metadatum.Name, 37 | Value: metadatum.Value, 38 | }, 39 | ) 40 | } 41 | 42 | for _, resource := range data.Resources() { 43 | var size *int64 44 | 45 | rawSize := resource.GetSize() 46 | if rawSize > 0 { 47 | // TODO differentiate between 0 size and missing 48 | size = &rawSize 49 | } 50 | 51 | var jsonChecksums []marshalDataResourceChecksum 52 | 53 | if len(data.checksumVerification.Acceptable) > 0 { 54 | // TODO probably ought to refactor to respect .Required here as well 55 | if csr, ok := resource.(service.ChecksumSupportedResolvedResource); ok { 56 | checksums, err := csr.GetChecksums(ctx, data.checksumVerification.Acceptable) 57 | if err != nil { 58 | return marshalData{}, errors.Wrapf(err, "getting checksums of %s", resource.GetName()) 59 | } 60 | 61 | for _, checksum := range data.checksumVerification.Selector.SelectChecksums(checksums) { 62 | v, err := checksum.NewVerifier(ctx) 63 | if err != nil { 64 | return marshalData{}, errors.Wrapf(err, "getting checksum %s of %s", checksum.Algorithm(), resource.GetName()) 65 | } 66 | 67 | jsonChecksums = append( 68 | jsonChecksums, 69 | marshalDataResourceChecksum{ 70 | Algo: string(v.Algorithm()), 71 | Data: fmt.Sprintf("%x", v.Expected()), 72 | }, 73 | ) 74 | } 75 | } 76 | } 77 | 78 | res.Resources = append( 79 | res.Resources, 80 | marshalDataResource{ 81 | Name: resource.GetName(), 82 | Size: size, 83 | Checksums: jsonChecksums, 84 | }, 85 | ) 86 | } 87 | 88 | sort.Slice(res.Resources, func(i, j int) bool { 89 | return strings.Compare(res.Resources[i].Name, res.Resources[j].Name) < 0 90 | }) 91 | 92 | return res, nil 93 | } 94 | 95 | type marshalData struct { 96 | Origin marshalDataOrigin `json:"origin"` 97 | Metadata []marshalDataMetadatum `json:"metadata"` 98 | Resources []marshalDataResource `json:"resources"` 99 | } 100 | 101 | type marshalDataOrigin struct { 102 | String string `json:"string"` 103 | Service string `json:"service"` 104 | Server string `json:"server"` 105 | Owner string `json:"owner"` 106 | Repository string `json:"repository"` 107 | Ref string `json:"ref"` 108 | } 109 | 110 | type marshalDataMetadatum struct { 111 | Key string `json:"key"` 112 | Value string `json:"value"` 113 | } 114 | 115 | type marshalDataResource struct { 116 | Name string `json:"name"` 117 | Size *int64 `json:"size,omitempty"` 118 | Checksums []marshalDataResourceChecksum `json:"checksums,omitempty"` 119 | } 120 | 121 | type marshalDataResourceChecksum struct { 122 | Algo string `json:"algo"` 123 | Data string `json:"data"` 124 | } 125 | -------------------------------------------------------------------------------- /pkg/export/plain_exporter.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type PlainExporter struct{} 12 | 13 | var _ Exporter = PlainExporter{} 14 | 15 | func (e PlainExporter) Export(ctx context.Context, w io.Writer, data *Data) error { 16 | res, err := newMarshalData(ctx, data) 17 | if err != nil { 18 | return errors.Wrap(err, "preparing export") 19 | } 20 | 21 | { // origin 22 | fmt.Fprintf(w, "origin\tresolved\t%s\n", res.Origin.String) 23 | fmt.Fprintf(w, "origin\tservice\t%s\n", res.Origin.Service) 24 | fmt.Fprintf(w, "origin\tserver\t%s\n", res.Origin.Server) 25 | fmt.Fprintf(w, "origin\towner\t%s\n", res.Origin.Owner) 26 | fmt.Fprintf(w, "origin\trepository\t%s\n", res.Origin.Repository) 27 | fmt.Fprintf(w, "origin\tref\t%s\n", res.Origin.Ref) 28 | } 29 | 30 | { // metadata 31 | for _, metadatum := range res.Metadata { 32 | fmt.Fprintf(w, "metadata\t%s\t%s\n", metadatum.Key, metadatum.Value) 33 | } 34 | } 35 | 36 | { // resources 37 | for _, resource := range res.Resources { 38 | fmt.Fprintf(w, "resource-name\t%s\n", resource.Name) 39 | 40 | if resource.Size != nil { 41 | fmt.Fprintf(w, "resource-size\t%s\t%d\n", resource.Name, resource.Size) 42 | } 43 | 44 | for _, checksum := range resource.Checksums { 45 | fmt.Fprintf(w, "resource-checksum\t%s\t%s\t%s\n", resource.Name, checksum.Algo, checksum.Data) 46 | } 47 | } 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/export/yaml_exporter.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/pkg/errors" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | type YAMLExporter struct{} 12 | 13 | var _ Exporter = YAMLExporter{} 14 | 15 | func (e YAMLExporter) Export(ctx context.Context, w io.Writer, data *Data) error { 16 | res, err := newMarshalData(ctx, data) 17 | if err != nil { 18 | return errors.Wrap(err, "preparing export") 19 | } 20 | 21 | buf, err := yaml.Marshal(res) 22 | if err != nil { 23 | return errors.Wrap(err, "marshalling") 24 | } 25 | 26 | w.Write(buf) 27 | w.Write([]byte("\n")) 28 | 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/ggetutil/version_opt.go: -------------------------------------------------------------------------------- 1 | package ggetutil 2 | 3 | import "github.com/dpb587/gget/pkg/cli/opt" 4 | 5 | type VersionOpt struct { 6 | Constraint opt.Constraint 7 | IsLatest bool 8 | } 9 | 10 | func (v *VersionOpt) UnmarshalFlag(data string) error { 11 | if data == "latest" { 12 | v.IsLatest = true 13 | 14 | return nil 15 | } 16 | 17 | return v.Constraint.UnmarshalFlag(data) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/gitutil/commit.go: -------------------------------------------------------------------------------- 1 | package gitutil 2 | 3 | import "regexp" 4 | 5 | var CommitRE = regexp.MustCompile(`^[0-9a-f]{40}$`) 6 | var PotentialCommitRE = regexp.MustCompile(`^[0-9a-f]{1,40}$`) 7 | -------------------------------------------------------------------------------- /pkg/service/github/archive/resource.go: -------------------------------------------------------------------------------- 1 | package archive 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/dpb587/gget/pkg/service" 13 | "github.com/google/go-github/v29/github" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | type Resource struct { 18 | client *github.Client 19 | ref service.Ref 20 | target string 21 | filename string 22 | } 23 | 24 | var _ service.ResolvedResource = &Resource{} 25 | 26 | func NewResource(client *github.Client, ref service.Ref, target, filename string) *Resource { 27 | return &Resource{ 28 | client: client, 29 | ref: ref, 30 | target: target, 31 | filename: filename, 32 | } 33 | } 34 | 35 | func (r *Resource) GetName() string { 36 | return r.filename 37 | } 38 | 39 | func (r *Resource) GetSize() int64 { 40 | return 0 41 | } 42 | 43 | func (r *Resource) Open(ctx context.Context) (io.ReadCloser, error) { 44 | var archiveLink *url.URL 45 | var err error 46 | 47 | ext := filepath.Ext(r.filename) 48 | if ext == ".gz" { 49 | ext = fmt.Sprintf("%s%s", filepath.Ext(strings.TrimSuffix(r.filename, ext)), ext) 50 | } 51 | 52 | switch ext { 53 | case ".tar.gz", ".tgz": 54 | archiveLink, _, err = r.client.Repositories.GetArchiveLink(ctx, r.ref.Owner, r.ref.Repository, github.Tarball, &github.RepositoryContentGetOptions{ 55 | Ref: r.target, 56 | }, false) 57 | case ".zip": 58 | archiveLink, _, err = r.client.Repositories.GetArchiveLink(ctx, r.ref.Owner, r.ref.Repository, github.Zipball, &github.RepositoryContentGetOptions{ 59 | Ref: r.target, 60 | }, false) 61 | default: 62 | return nil, fmt.Errorf("unrecognized extension: %s", ext) 63 | } 64 | 65 | if err != nil { 66 | return nil, errors.Wrap(err, "getting archive url") 67 | } 68 | 69 | res, err := http.DefaultClient.Get(archiveLink.String()) 70 | if err != nil { 71 | return nil, errors.Wrap(err, "getting download url") 72 | } 73 | 74 | if res.StatusCode != 200 { 75 | return nil, errors.Wrapf(fmt.Errorf("expected status 200: got %d", res.StatusCode), "getting download url %s", archiveLink) 76 | } 77 | 78 | return res.Body, nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/service/github/asset/resource.go: -------------------------------------------------------------------------------- 1 | package asset 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/dpb587/gget/pkg/checksum" 10 | "github.com/dpb587/gget/pkg/service" 11 | "github.com/google/go-github/v29/github" 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | type Resource struct { 16 | client *github.Client 17 | releaseOwner string 18 | releaseRepository string 19 | checksumManager checksum.Manager 20 | asset github.ReleaseAsset 21 | } 22 | 23 | var _ service.ResolvedResource = &Resource{} 24 | var _ service.ChecksumSupportedResolvedResource = &Resource{} 25 | 26 | func NewResource(client *github.Client, releaseOwner, releaseRepository string, asset github.ReleaseAsset, checksumManager checksum.Manager) *Resource { 27 | return &Resource{ 28 | client: client, 29 | releaseOwner: releaseOwner, 30 | releaseRepository: releaseRepository, 31 | asset: asset, 32 | checksumManager: checksumManager, 33 | } 34 | } 35 | 36 | func (r *Resource) GetName() string { 37 | return r.asset.GetName() 38 | } 39 | 40 | func (r *Resource) GetSize() int64 { 41 | return int64(r.asset.GetSize()) 42 | } 43 | 44 | func (r *Resource) GetChecksums(ctx context.Context, algos checksum.AlgorithmList) (checksum.ChecksumList, error) { 45 | if r.checksumManager == nil { 46 | return nil, nil 47 | } 48 | 49 | cs, err := r.checksumManager.GetChecksums(ctx, r.asset.GetName(), algos) 50 | if err != nil { 51 | return nil, errors.Wrapf(err, "getting checksum of %s", r.asset.GetName()) 52 | } else if len(cs) == 0 { 53 | return nil, nil 54 | } 55 | 56 | return cs, nil 57 | } 58 | 59 | func (r *Resource) Open(ctx context.Context) (io.ReadCloser, error) { 60 | remoteHandle, redirectURL, err := r.client.Repositories.DownloadReleaseAsset(ctx, r.releaseOwner, r.releaseRepository, r.asset.GetID(), nil) 61 | if err != nil { 62 | return nil, errors.Wrap(err, "requesting asset") 63 | } 64 | 65 | if redirectURL != "" { 66 | res, err := http.DefaultClient.Get(redirectURL) 67 | if err != nil { 68 | return nil, errors.Wrapf(err, "getting download url %s", redirectURL) 69 | } 70 | 71 | if res.StatusCode != 200 { 72 | return nil, errors.Wrapf(fmt.Errorf("expected status 200: got %d", res.StatusCode), "getting download url %s", redirectURL) 73 | } 74 | 75 | remoteHandle = res.Body 76 | } 77 | 78 | return remoteHandle, nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/service/github/blob/resource.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "io/ioutil" 8 | 9 | "github.com/dpb587/gget/pkg/service" 10 | "github.com/google/go-github/v29/github" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type Resource struct { 15 | client *github.Client 16 | releaseOwner string 17 | releaseRepository string 18 | asset github.TreeEntry 19 | } 20 | 21 | var _ service.ResolvedResource = &Resource{} 22 | 23 | func NewResource(client *github.Client, releaseOwner, releaseRepository string, asset github.TreeEntry) *Resource { 24 | return &Resource{ 25 | client: client, 26 | releaseOwner: releaseOwner, 27 | releaseRepository: releaseRepository, 28 | asset: asset, 29 | } 30 | } 31 | 32 | func (r *Resource) GetName() string { 33 | return r.asset.GetPath() 34 | } 35 | 36 | func (r *Resource) GetSize() int64 { 37 | return int64(r.asset.GetSize()) 38 | } 39 | 40 | func (r *Resource) Open(ctx context.Context) (io.ReadCloser, error) { 41 | // TODO switch to stream? 42 | buf, _, err := r.client.Git.GetBlobRaw(ctx, r.releaseOwner, r.releaseRepository, r.asset.GetSHA()) 43 | if err != nil { 44 | return nil, errors.Wrap(err, "getting blob") 45 | } 46 | 47 | return ioutil.NopCloser(bytes.NewReader(buf)), nil 48 | } 49 | -------------------------------------------------------------------------------- /pkg/service/github/client_factory.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/bgentry/go-netrc/netrc" 11 | "github.com/dpb587/gget/pkg/service" 12 | "github.com/google/go-github/v29/github" 13 | "github.com/mitchellh/go-homedir" 14 | "github.com/pkg/errors" 15 | "github.com/sirupsen/logrus" 16 | "golang.org/x/oauth2" 17 | ) 18 | 19 | type roundTripTransformer func(http.RoundTripper) http.RoundTripper 20 | 21 | type ClientFactory struct { 22 | log *logrus.Logger 23 | httpClientFactory func() *http.Client 24 | } 25 | 26 | func NewClientFactory(log *logrus.Logger, httpClientFactory func() *http.Client) *ClientFactory { 27 | return &ClientFactory{ 28 | log: log, 29 | httpClientFactory: httpClientFactory, 30 | } 31 | } 32 | 33 | func (cf ClientFactory) Get(ctx context.Context, lookupRef service.LookupRef) (*github.Client, error) { 34 | var tokenSource oauth2.TokenSource 35 | 36 | if v := os.Getenv("GITHUB_TOKEN"); v != "" { 37 | cf.log.Infof("found authentication for %s: env GITHUB_TOKEN", lookupRef.Ref.Server) 38 | 39 | tokenSource = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: v}) 40 | } else { 41 | var err error 42 | 43 | tokenSource, err = cf.loadNetrc(ctx, lookupRef) 44 | if err != nil { 45 | return nil, errors.Wrap(err, "loading auth from netrc") 46 | } 47 | } 48 | 49 | httpClient := cf.httpClientFactory() 50 | 51 | if tokenSource != nil { 52 | httpClient.Transport = &oauth2.Transport{ 53 | Base: httpClient.Transport, 54 | Source: oauth2.ReuseTokenSource(nil, tokenSource), 55 | } 56 | } 57 | 58 | if lookupRef.Ref.Server == "github.com" { 59 | return github.NewClient(httpClient), nil 60 | } 61 | 62 | // TODO figure out https configurability 63 | baseURL := fmt.Sprintf("https://%s", lookupRef.Ref.Server) 64 | 65 | c, err := github.NewEnterpriseClient(baseURL, baseURL, httpClient) 66 | if err != nil { 67 | return nil, errors.Wrap(err, "building enterprise client") 68 | } 69 | 70 | return c, nil 71 | } 72 | 73 | func (cf ClientFactory) loadNetrc(ctx context.Context, lookupRef service.LookupRef) (oauth2.TokenSource, error) { 74 | netrcPath := os.Getenv("NETRC") 75 | if netrcPath == "" { 76 | var err error 77 | 78 | netrcPath, err = homedir.Expand(filepath.Join("~", ".netrc")) 79 | if err != nil { 80 | return nil, errors.Wrap(err, "expanding $HOME") 81 | } 82 | } 83 | 84 | fi, err := os.Stat(netrcPath) 85 | if err != nil { 86 | if os.IsNotExist(err) { 87 | return nil, nil 88 | } 89 | 90 | return nil, errors.Wrap(err, "checking file") 91 | } else if fi.IsDir() { 92 | // weird 93 | return nil, nil 94 | } 95 | 96 | rc, err := netrc.ParseFile(netrcPath) 97 | if err != nil { 98 | return nil, errors.Wrap(err, "parsing netrc") 99 | } 100 | 101 | machine := rc.FindMachine(lookupRef.Ref.Server) 102 | if machine == nil { 103 | return nil, nil 104 | } 105 | 106 | cf.log.Infof("found authentication for %s: netrc %s", lookupRef.Ref.Server, netrcPath) 107 | 108 | return oauth2.StaticTokenSource(&oauth2.Token{AccessToken: machine.Password}), nil 109 | } 110 | -------------------------------------------------------------------------------- /pkg/service/github/commit_ref.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | 8 | "github.com/dpb587/gget/pkg/service" 9 | "github.com/dpb587/gget/pkg/service/github/archive" 10 | "github.com/dpb587/gget/pkg/service/github/blob" 11 | "github.com/google/go-github/v29/github" 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | type CommitRef struct { 16 | client *github.Client 17 | ref service.Ref 18 | commit string 19 | metadata service.RefMetadata 20 | 21 | archiveFileBase string 22 | } 23 | 24 | var _ service.ResolvedRef = &ReleaseRef{} 25 | var _ service.ResourceResolver = &CommitRef{} 26 | 27 | func (r *CommitRef) CanonicalRef() service.Ref { 28 | return r.ref 29 | } 30 | 31 | func (r *CommitRef) GetMetadata(_ context.Context) (service.RefMetadata, error) { 32 | return r.metadata, nil 33 | } 34 | 35 | func (r *CommitRef) ResolveResource(ctx context.Context, resourceType service.ResourceType, resource service.ResourceName) ([]service.ResolvedResource, error) { 36 | switch resourceType { 37 | case service.ArchiveResourceType: 38 | return r.resolveArchiveResource(ctx, resource) 39 | case service.BlobResourceType: 40 | return r.resolveBlobResource(ctx, resource) 41 | } 42 | 43 | return nil, fmt.Errorf("unsupported resource type for commit ref: %s", resourceType) 44 | } 45 | 46 | func (r *CommitRef) resolveArchiveResource(ctx context.Context, resource service.ResourceName) ([]service.ResolvedResource, error) { 47 | candidates := []string{ 48 | fmt.Sprintf("%s.tar.gz", r.archiveFileBase), 49 | fmt.Sprintf("%s.zip", r.archiveFileBase), 50 | } 51 | 52 | var res []service.ResolvedResource 53 | 54 | for _, candidate := range candidates { 55 | if match, _ := filepath.Match(string(resource), candidate); !match { 56 | continue 57 | } 58 | 59 | res = append( 60 | res, 61 | archive.NewResource( 62 | r.client, 63 | r.ref, 64 | r.commit, 65 | candidate, 66 | ), 67 | ) 68 | } 69 | 70 | return res, nil 71 | } 72 | 73 | func (r *CommitRef) resolveBlobResource(ctx context.Context, resource service.ResourceName) ([]service.ResolvedResource, error) { 74 | var res []service.ResolvedResource 75 | 76 | // get the full tree 77 | tree, _, err := r.client.Git.GetTree(ctx, r.ref.Owner, r.ref.Repository, r.commit, true) 78 | if err != nil { 79 | return nil, errors.Wrap(err, "getting commit tree") 80 | } 81 | 82 | for _, candidate := range tree.Entries { 83 | if match, _ := filepath.Match(string(resource), candidate.GetPath()); !match { 84 | continue 85 | } 86 | 87 | res = append(res, blob.NewResource(r.client, r.ref.Owner, r.ref.Repository, candidate)) 88 | } 89 | 90 | return res, nil 91 | } 92 | -------------------------------------------------------------------------------- /pkg/service/github/ref_resolver.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "path" 8 | "strings" 9 | 10 | "github.com/dpb587/gget/pkg/service" 11 | "github.com/google/go-github/v29/github" 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | type refResolver struct { 16 | client *github.Client 17 | lookupRef service.LookupRef 18 | canonicalRef service.Ref 19 | } 20 | 21 | func (rr *refResolver) resolveTagWithRelease(ctx context.Context, release *github.RepositoryRelease) (service.ResolvedRef, error) { 22 | return rr.resolveTag( 23 | ctx, 24 | &github.Reference{ 25 | Ref: release.TagName, 26 | Object: &github.GitObject{ 27 | // Type: "commit", 28 | SHA: release.TargetCommitish, 29 | }, 30 | }, 31 | false, 32 | ) 33 | } 34 | 35 | func (rr *refResolver) resolveCommit(ctx context.Context, commitSHA string) (service.ResolvedRef, error) { 36 | res := &CommitRef{ 37 | client: rr.client, 38 | ref: rr.canonicalRef, 39 | commit: commitSHA, 40 | archiveFileBase: fmt.Sprintf("%s-%s", rr.canonicalRef.Repository, commitSHA[0:9]), 41 | metadata: service.RefMetadata{ 42 | { 43 | Name: "commit", 44 | Value: commitSHA, 45 | }, 46 | }, 47 | } 48 | 49 | return res, nil 50 | } 51 | 52 | func (rr *refResolver) resolveHead(ctx context.Context, headRef *github.Reference) (service.ResolvedRef, error) { 53 | branchName := strings.TrimPrefix(headRef.GetRef(), "refs/heads/") 54 | commitSHA := headRef.Object.GetSHA() 55 | 56 | res := &CommitRef{ 57 | client: rr.client, 58 | ref: rr.canonicalRef, 59 | commit: commitSHA, 60 | archiveFileBase: fmt.Sprintf("%s-%s", rr.canonicalRef.Repository, path.Base(branchName)), 61 | metadata: service.RefMetadata{ 62 | { 63 | Name: "branch", 64 | Value: branchName, 65 | }, 66 | { 67 | Name: "commit", 68 | Value: commitSHA, 69 | }, 70 | }, 71 | } 72 | 73 | return res, nil 74 | } 75 | 76 | func (rr *refResolver) resolveTag(ctx context.Context, tagRef *github.Reference, attemptRelease bool) (service.ResolvedRef, error) { 77 | var tagObj *github.Tag 78 | 79 | if tagRef.Object.GetType() == "tag" { // annotated tag 80 | var err error 81 | 82 | tagObj, _, err = rr.client.Git.GetTag(ctx, rr.canonicalRef.Owner, rr.canonicalRef.Repository, tagRef.Object.GetSHA()) 83 | if err != nil { 84 | return nil, errors.Wrap(err, "getting tag of annotated tag") 85 | } 86 | } else { // lightweight tag 87 | // mock to save an API call 88 | tagObj = &github.Tag{ 89 | Tag: tagRef.Ref, 90 | Object: &github.GitObject{ 91 | SHA: tagRef.Object.SHA, 92 | }, 93 | } 94 | } 95 | 96 | tagName := strings.TrimPrefix(tagObj.GetTag(), "refs/tags/") 97 | commitSHA := tagObj.Object.GetSHA() 98 | 99 | var res service.ResolvedRef = &CommitRef{ 100 | client: rr.client, 101 | ref: rr.canonicalRef, 102 | commit: commitSHA, 103 | archiveFileBase: fmt.Sprintf("%s-%s", rr.canonicalRef.Repository, tagName), 104 | metadata: service.RefMetadata{ 105 | { 106 | Name: "tag", 107 | Value: tagName, 108 | }, 109 | { 110 | Name: "commit", 111 | Value: commitSHA, 112 | }, 113 | }, 114 | } 115 | 116 | if !attemptRelease { 117 | return res, nil 118 | } 119 | 120 | release, resp, err := rr.client.Repositories.GetReleaseByTag(ctx, rr.canonicalRef.Owner, rr.canonicalRef.Repository, tagName) 121 | if resp.StatusCode == http.StatusNotFound { 122 | // oh well 123 | } else if err != nil { 124 | return nil, errors.Wrap(err, "getting release by tag") 125 | } else if release != nil { 126 | res = &ReleaseRef{ 127 | refResolver: rr, 128 | release: release, 129 | targetRef: res, 130 | } 131 | } 132 | 133 | return res, nil 134 | } 135 | -------------------------------------------------------------------------------- /pkg/service/github/release_checksum_manager.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/dpb587/gget/pkg/checksum" 10 | "github.com/dpb587/gget/pkg/checksum/parser" 11 | "github.com/dpb587/gget/pkg/service/github/asset" 12 | "github.com/google/go-github/v29/github" 13 | ) 14 | 15 | func NewReleaseChecksumManager(client *github.Client, releaseOwner, releaseRepository string, release *github.RepositoryRelease) checksum.Manager { 16 | literalManager := checksum.NewInMemoryManager() 17 | var deferredManagers []checksum.Manager 18 | 19 | // parse from release notes 20 | parser.ImportMarkdown(literalManager, []byte(release.GetBody())) 21 | 22 | // checksums from convention-based file names 23 | for _, releaseAsset := range release.Assets { 24 | algorithm, resource, useful := checkReleaseAssetChecksumBehavior(releaseAsset) 25 | if !useful { 26 | continue 27 | } 28 | 29 | opener := newReleaseAssetChecksumOpener(client, releaseOwner, releaseRepository, releaseAsset) 30 | 31 | var expectedAlgos checksum.AlgorithmList 32 | 33 | if algorithm != "" && algorithm != "unknown" { 34 | expectedAlgos = append(expectedAlgos, algorithm) 35 | } 36 | 37 | if resource != "" { 38 | literalManager.AddChecksum( 39 | resource, 40 | checksum.NewDeferredChecksum( 41 | parser.NewDeferredManager(checksum.NewInMemoryAliasManager(resource), expectedAlgos, opener), 42 | resource, 43 | algorithm, 44 | ), 45 | ) 46 | } else if algorithm != "" { 47 | deferredManagers = append(deferredManagers, parser.NewDeferredManager(checksum.NewInMemoryManager(), expectedAlgos, opener)) 48 | } 49 | } 50 | 51 | return checksum.NewMultiManager(append([]checksum.Manager{literalManager}, deferredManagers...)...) 52 | } 53 | 54 | func checkReleaseAssetChecksumBehavior(releaseAsset github.ReleaseAsset) (checksum.Algorithm, string, bool) { 55 | name := releaseAsset.GetName() 56 | nameLower := strings.ToLower(name) 57 | ext := filepath.Ext(releaseAsset.GetName()) 58 | extLower := strings.ToLower(strings.TrimPrefix(ext, ".")) 59 | 60 | if extLower == "md5" || extLower == "sha1" || extLower == "sha256" || extLower == "sha384" || extLower == "sha512" { 61 | return checksum.Algorithm(extLower), strings.TrimSuffix(name, ext), true 62 | } else if nameLower == "md5sum" || nameLower == "md5sums" || nameLower == "md5sum.txt" || nameLower == "md5sums.txt" { 63 | return checksum.MD5, "", true 64 | } else if nameLower == "sha1sum" || nameLower == "sha1sums" || nameLower == "sha1sum.txt" || nameLower == "sha1sums.txt" { 65 | return checksum.SHA1, "", true 66 | } else if nameLower == "sha384sum" || nameLower == "sha384sums" || nameLower == "sha384sum.txt" || nameLower == "sha384sums.txt" { 67 | return checksum.SHA384, "", true 68 | } else if nameLower == "sha256sum" || nameLower == "sha256sums" || nameLower == "sha256sum.txt" || nameLower == "sha256sums.txt" { 69 | return checksum.SHA256, "", true 70 | } else if nameLower == "sha512sum" || nameLower == "sha512sums" || nameLower == "sha512sum.txt" || nameLower == "sha512sums.txt" { 71 | return checksum.SHA512, "", true 72 | } else if nameLower == "checksum" || nameLower == "checksums" || strings.HasSuffix(nameLower, "checksum.txt") || strings.HasSuffix(nameLower, "checksums.txt") { 73 | return checksum.Algorithm("unknown"), "", true 74 | } 75 | 76 | return "", "", false 77 | } 78 | 79 | func newReleaseAssetChecksumOpener(client *github.Client, releaseOwner, releaseRepository string, releaseAsset github.ReleaseAsset) func(context.Context) (io.ReadCloser, error) { 80 | return func(ctx context.Context) (io.ReadCloser, error) { 81 | resource := asset.NewResource(client, releaseOwner, releaseRepository, releaseAsset, nil) // TODO pass shared checksum manager 82 | 83 | return resource.Open(ctx) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/service/github/release_ref.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "time" 8 | 9 | "github.com/dpb587/gget/pkg/checksum" 10 | "github.com/dpb587/gget/pkg/service" 11 | "github.com/dpb587/gget/pkg/service/github/asset" 12 | "github.com/google/go-github/v29/github" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | type ReleaseRef struct { 17 | refResolver *refResolver 18 | release *github.RepositoryRelease 19 | 20 | targetRef service.ResolvedRef 21 | checksumManager checksum.Manager 22 | } 23 | 24 | var _ service.ResolvedRef = &ReleaseRef{} 25 | var _ service.ResourceResolver = &ReleaseRef{} 26 | 27 | func (r *ReleaseRef) CanonicalRef() service.Ref { 28 | return r.refResolver.canonicalRef 29 | } 30 | 31 | func (r *ReleaseRef) GetMetadata(ctx context.Context) (service.RefMetadata, error) { 32 | targetRef, err := r.requireTargetRef(ctx) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | tagMetadata, err := targetRef.GetMetadata(ctx) 38 | if err != nil { 39 | return nil, errors.Wrap(err, "getting commit metadata") 40 | } 41 | 42 | res := append( 43 | service.RefMetadata{ 44 | { 45 | Name: "github-release-id", 46 | Value: fmt.Sprintf("%d", r.release.GetID()), 47 | }, 48 | { 49 | Name: "github-release-published-at", 50 | Value: r.release.GetPublishedAt().Format(time.RFC3339), 51 | }, 52 | { 53 | Name: "github-release-body", 54 | Value: r.release.GetBody(), 55 | }, 56 | }, 57 | tagMetadata..., 58 | ) 59 | 60 | return res, nil 61 | } 62 | 63 | func (r *ReleaseRef) ResolveResource(ctx context.Context, resourceType service.ResourceType, resource service.ResourceName) ([]service.ResolvedResource, error) { 64 | if resourceType == service.AssetResourceType { 65 | return r.resolveAssetResource(ctx, resource) 66 | } 67 | 68 | targetRef, err := r.requireTargetRef(ctx) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | return targetRef.ResolveResource(ctx, resourceType, resource) 74 | } 75 | 76 | func (r *ReleaseRef) requireTargetRef(ctx context.Context) (service.ResolvedRef, error) { 77 | if r.targetRef == nil { 78 | ref, err := r.refResolver.resolveTagWithRelease(ctx, r.release) 79 | if err != nil { 80 | return nil, errors.Wrap(err, "resolving commit") 81 | } 82 | 83 | r.targetRef = ref 84 | } 85 | 86 | return r.targetRef, nil 87 | } 88 | 89 | func (r *ReleaseRef) resolveAssetResource(ctx context.Context, resource service.ResourceName) ([]service.ResolvedResource, error) { 90 | var res []service.ResolvedResource 91 | 92 | for _, candidate := range r.release.Assets { 93 | if match, _ := filepath.Match(string(resource), candidate.GetName()); !match { 94 | continue 95 | } 96 | 97 | res = append( 98 | res, 99 | asset.NewResource(r.refResolver.client, r.refResolver.canonicalRef.Owner, r.refResolver.canonicalRef.Repository, candidate, r.requireChecksumManager()), 100 | ) 101 | } 102 | 103 | return res, nil 104 | } 105 | 106 | func (r *ReleaseRef) requireChecksumManager() checksum.Manager { 107 | if r.checksumManager == nil { 108 | r.checksumManager = NewReleaseChecksumManager(r.refResolver.client, r.refResolver.canonicalRef.Owner, r.refResolver.canonicalRef.Repository, r.release) 109 | } 110 | 111 | return r.checksumManager 112 | } 113 | -------------------------------------------------------------------------------- /pkg/service/github/service.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "path" 9 | "strings" 10 | 11 | "github.com/dpb587/gget/pkg/gitutil" 12 | "github.com/dpb587/gget/pkg/service" 13 | "github.com/google/go-github/v29/github" 14 | "github.com/pkg/errors" 15 | "github.com/sirupsen/logrus" 16 | ) 17 | 18 | type Service struct { 19 | log *logrus.Logger 20 | clientFactory *ClientFactory 21 | } 22 | 23 | func NewService(log *logrus.Logger, clientFactory *ClientFactory) *Service { 24 | return &Service{ 25 | log: log, 26 | clientFactory: clientFactory, 27 | } 28 | } 29 | 30 | var _ service.RefResolver = &Service{} 31 | var _ service.ConditionalRefResolver = &Service{} 32 | 33 | func (s Service) ServiceName() string { 34 | return "github" 35 | } 36 | 37 | func (s Service) IsKnownServer(_ context.Context, lookupRef service.LookupRef) bool { 38 | return lookupRef.Ref.Server == "github.com" 39 | } 40 | 41 | func (s Service) IsDetectedServer(_ context.Context, lookupRef service.LookupRef) bool { 42 | res, err := s.clientFactory.httpClientFactory().Get(fmt.Sprintf("https://%s/api/v3/", lookupRef.Ref.Server)) 43 | if err != nil { 44 | s.log.Debugf("github detection attempt error: %s", errors.Wrap(err, "requesting GET /api/v3/")) 45 | 46 | return false 47 | } else if res.StatusCode != http.StatusOK { 48 | return false 49 | } 50 | 51 | buf, err := ioutil.ReadAll(res.Body) 52 | if err != nil { 53 | s.log.Debugf("github detection attempt error: %s", errors.Wrap(err, "reading body")) 54 | 55 | return false 56 | } 57 | 58 | return strings.Contains(string(buf), `"organization_repositories_url"`) 59 | } 60 | 61 | func (s Service) ResolveRef(ctx context.Context, lookupRef service.LookupRef) (service.ResolvedRef, error) { 62 | client, err := s.clientFactory.Get(ctx, lookupRef) 63 | if err != nil { 64 | return nil, errors.Wrap(err, "building client") 65 | } 66 | 67 | ref := lookupRef.Ref 68 | ref.Service = s.ServiceName() 69 | 70 | rr := &refResolver{ 71 | client: client, 72 | lookupRef: lookupRef, 73 | canonicalRef: ref, 74 | } 75 | 76 | if ref.Ref == "" { 77 | release, err := s.resolveLatest(ctx, client, lookupRef) 78 | if err != nil { 79 | return nil, errors.Wrap(err, "resolving latest") 80 | } 81 | 82 | rr.canonicalRef.Ref = release.GetTagName() 83 | 84 | return &ReleaseRef{ 85 | refResolver: rr, 86 | release: release, 87 | }, nil 88 | } 89 | 90 | { // tag 91 | gitref, resp, err := client.Git.GetRefs(ctx, rr.canonicalRef.Owner, rr.canonicalRef.Repository, path.Join("tags", rr.canonicalRef.Ref)) 92 | if resp.StatusCode == http.StatusNotFound { 93 | // oh well 94 | } else if err != nil { 95 | return nil, errors.Wrap(err, "attempting tag resolution") 96 | } else if len(gitref) == 1 { 97 | return rr.resolveTag(ctx, gitref[0], true) 98 | } 99 | } 100 | 101 | { // head 102 | gitref, resp, err := client.Git.GetRefs(ctx, rr.canonicalRef.Owner, rr.canonicalRef.Repository, path.Join("heads", rr.canonicalRef.Ref)) 103 | if resp.StatusCode == http.StatusNotFound { 104 | // oh well 105 | } else if err != nil { 106 | return nil, errors.Wrap(err, "attempting branch resolution") 107 | } else if len(gitref) == 1 { 108 | return rr.resolveHead(ctx, gitref[0]) 109 | } 110 | } 111 | 112 | if gitutil.PotentialCommitRE.MatchString(rr.canonicalRef.Ref) { // commit 113 | // client.Git.GetCommit does not resolve partial commits 114 | commitref, resp, err := client.Repositories.GetCommit(ctx, rr.canonicalRef.Owner, rr.canonicalRef.Repository, rr.canonicalRef.Ref) 115 | if resp.StatusCode == http.StatusNotFound { 116 | // oh well 117 | } else if err != nil { 118 | return nil, errors.Wrap(err, "attempting commit resolution") 119 | } else { 120 | rr.canonicalRef.Ref = commitref.GetSHA() 121 | 122 | return rr.resolveCommit(ctx, commitref.GetSHA()) 123 | } 124 | } 125 | 126 | return nil, fmt.Errorf("unable to resolve as tag, branch, nor commit: %s", rr.canonicalRef.Ref) 127 | } 128 | 129 | func (s Service) resolveLatest(ctx context.Context, client *github.Client, lookupRef service.LookupRef) (*github.RepositoryRelease, error) { 130 | if lookupRef.IsComplexRef() { 131 | opts := github.ListOptions{ 132 | PerPage: 25, 133 | } 134 | 135 | for { 136 | releases, resp, err := client.Repositories.ListReleases(ctx, lookupRef.Ref.Owner, lookupRef.Ref.Repository, &opts) 137 | if err != nil { 138 | return nil, errors.Wrap(err, "iterating releases") 139 | } else if resp.StatusCode == http.StatusNotFound { 140 | return nil, errors.New("repository not found") 141 | } 142 | 143 | for _, release := range releases { 144 | { 145 | var stability = "stable" 146 | 147 | if release.GetPrerelease() { 148 | stability = "pre-release" 149 | } 150 | 151 | if !lookupRef.SatisfiesStability(stability) { 152 | continue 153 | } 154 | } 155 | 156 | tagName := release.GetTagName() 157 | match, err := lookupRef.SatisfiesVersion(tagName) 158 | if err != nil { 159 | s.log.Debugf("skipping invalid semver tag: %s", tagName) 160 | 161 | continue 162 | } else if !match { 163 | continue 164 | } 165 | 166 | return release, nil 167 | } 168 | 169 | opts.Page = resp.NextPage 170 | 171 | if opts.Page == 0 { 172 | break 173 | } 174 | } 175 | 176 | return nil, fmt.Errorf("failed to find release matching constraints: %s", strings.Join(lookupRef.ComplexRefModes(), ", ")) 177 | } 178 | 179 | release, resp, err := client.Repositories.GetLatestRelease(ctx, lookupRef.Ref.Owner, lookupRef.Ref.Repository) 180 | if err != nil { 181 | return nil, errors.Wrap(err, "getting latest release") 182 | } else if resp.StatusCode == http.StatusNotFound { 183 | return nil, errors.New("repository not found") 184 | } 185 | 186 | return release, nil 187 | } 188 | -------------------------------------------------------------------------------- /pkg/service/gitlab/archive/resource.go: -------------------------------------------------------------------------------- 1 | package archive 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "io/ioutil" 8 | 9 | "github.com/dpb587/gget/pkg/service" 10 | "github.com/dpb587/gget/pkg/service/gitlab/gitlabutil" 11 | "github.com/pkg/errors" 12 | "github.com/xanzy/go-gitlab" 13 | ) 14 | 15 | type Resource struct { 16 | client *gitlab.Client 17 | ref service.Ref 18 | target string 19 | filename string 20 | format string 21 | } 22 | 23 | var _ service.ResolvedResource = &Resource{} 24 | 25 | func NewResource(client *gitlab.Client, ref service.Ref, target, filename, format string) *Resource { 26 | return &Resource{ 27 | client: client, 28 | ref: ref, 29 | target: target, 30 | filename: filename, 31 | format: format, 32 | } 33 | } 34 | 35 | func (r *Resource) GetName() string { 36 | return r.filename 37 | } 38 | 39 | func (r *Resource) GetSize() int64 { 40 | return 0 41 | } 42 | 43 | func (r *Resource) Open(ctx context.Context) (io.ReadCloser, error) { 44 | buf, _, err := r.client.Repositories.Archive(gitlabutil.GetRepositoryID(r.ref), &gitlab.ArchiveOptions{ 45 | Format: &r.format, 46 | SHA: &r.target, 47 | }) 48 | 49 | if err != nil { 50 | return nil, errors.Wrap(err, "getting archive url") 51 | } 52 | 53 | return ioutil.NopCloser(bytes.NewReader(buf)), nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/service/gitlab/asset/resource.go: -------------------------------------------------------------------------------- 1 | package asset 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "path" 9 | 10 | "github.com/dpb587/gget/pkg/service" 11 | "github.com/pkg/errors" 12 | "github.com/xanzy/go-gitlab" 13 | ) 14 | 15 | type Resource struct { 16 | client *gitlab.Client 17 | releaseOwner string 18 | releaseRepository string 19 | asset *gitlab.ReleaseLink 20 | } 21 | 22 | var _ service.ResolvedResource = &Resource{} 23 | 24 | func NewResource(client *gitlab.Client, releaseOwner, releaseRepository string, asset *gitlab.ReleaseLink) *Resource { 25 | return &Resource{ 26 | client: client, 27 | releaseOwner: releaseOwner, 28 | releaseRepository: releaseRepository, 29 | asset: asset, 30 | } 31 | } 32 | 33 | func (r *Resource) GetName() string { 34 | return path.Base(r.asset.URL) 35 | } 36 | 37 | func (r *Resource) GetSize() int64 { 38 | return 0 39 | } 40 | 41 | func (r *Resource) Open(ctx context.Context) (io.ReadCloser, error) { 42 | res, err := http.DefaultClient.Get(r.asset.URL) 43 | if err != nil { 44 | return nil, errors.Wrapf(err, "getting %s", r.asset.URL) 45 | } 46 | 47 | if res.StatusCode != 200 { 48 | return nil, errors.Wrapf(fmt.Errorf("expected status 200: got %d", res.StatusCode), "getting %s", r.asset.URL) 49 | } 50 | 51 | return res.Body, nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/service/gitlab/blob/resource.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/base64" 7 | "encoding/json" 8 | "io" 9 | "io/ioutil" 10 | 11 | "github.com/dpb587/gget/pkg/service" 12 | "github.com/dpb587/gget/pkg/service/gitlab/gitlabutil" 13 | "github.com/pkg/errors" 14 | "github.com/xanzy/go-gitlab" 15 | ) 16 | 17 | type Resource struct { 18 | client *gitlab.Client 19 | ref service.Ref 20 | target string 21 | node *gitlab.TreeNode 22 | } 23 | 24 | var _ service.ResolvedResource = &Resource{} 25 | 26 | func NewResource(client *gitlab.Client, ref service.Ref, target string, node *gitlab.TreeNode) *Resource { 27 | return &Resource{ 28 | client: client, 29 | ref: ref, 30 | target: target, 31 | node: node, 32 | } 33 | } 34 | 35 | func (r *Resource) GetName() string { 36 | return r.node.Name 37 | } 38 | 39 | func (r *Resource) GetSize() int64 { 40 | return 0 41 | } 42 | 43 | func (r *Resource) Open(ctx context.Context) (io.ReadCloser, error) { 44 | // TODO switch to stream? 45 | bufRes, _, err := r.client.Repositories.Blob(gitlabutil.GetRepositoryID(r.ref), r.node.ID, nil) 46 | if err != nil { 47 | return nil, errors.Wrap(err, "getting blob") 48 | } 49 | 50 | var apiRes apiResponse 51 | 52 | err = json.Unmarshal(bufRes, &apiRes) 53 | if err != nil { 54 | return nil, errors.Wrap(err, "parsing blob api") 55 | } 56 | 57 | var buf []byte 58 | 59 | if apiRes.Encoding == "base64" { 60 | buf, err = base64.StdEncoding.DecodeString(apiRes.Content) 61 | if err != nil { 62 | return nil, errors.Wrap(err, "decoding blob") 63 | } 64 | } else { 65 | return nil, errors.Wrap(err, "unsupported content encoding") 66 | } 67 | 68 | return ioutil.NopCloser(bytes.NewReader(buf)), nil 69 | } 70 | 71 | type apiResponse struct { 72 | Size int `json:"size"` 73 | Encoding string `json:"encoding"` 74 | Content string `json:"content"` 75 | SHA string `json:"sha"` 76 | } 77 | -------------------------------------------------------------------------------- /pkg/service/gitlab/client_factory.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/bgentry/go-netrc/netrc" 11 | "github.com/dpb587/gget/pkg/service" 12 | "github.com/mitchellh/go-homedir" 13 | "github.com/pkg/errors" 14 | "github.com/sirupsen/logrus" 15 | "github.com/xanzy/go-gitlab" 16 | ) 17 | 18 | type roundTripTransformer func(http.RoundTripper) http.RoundTripper 19 | 20 | type ClientFactory struct { 21 | log *logrus.Logger 22 | httpClientFactory func() *http.Client 23 | } 24 | 25 | func NewClientFactory(log *logrus.Logger, httpClientFactory func() *http.Client) *ClientFactory { 26 | return &ClientFactory{ 27 | log: log, 28 | httpClientFactory: httpClientFactory, 29 | } 30 | } 31 | 32 | func (cf ClientFactory) Get(ctx context.Context, lookupRef service.LookupRef) (*gitlab.Client, error) { 33 | var token string 34 | 35 | if v := os.Getenv("GITLAB_TOKEN"); v != "" { 36 | cf.log.Infof("found authentication for %s: env GITLAB_TOKEN", lookupRef.Ref.Server) 37 | 38 | token = v 39 | } else { 40 | var err error 41 | 42 | token, err = cf.loadNetrc(ctx, lookupRef) 43 | if err != nil { 44 | return nil, errors.Wrap(err, "loading auth from netrc") 45 | } 46 | } 47 | 48 | var clientOpts = []gitlab.ClientOptionFunc{ 49 | gitlab.WithHTTPClient(cf.httpClientFactory()), 50 | // TODO figure out https configurability 51 | gitlab.WithBaseURL(fmt.Sprintf("https://%s/", lookupRef.Ref.Server)), 52 | } 53 | 54 | res, err := gitlab.NewClient(token, clientOpts...) 55 | if err != nil { 56 | return nil, errors.Wrap(err, "creating client") 57 | } 58 | 59 | return res, nil 60 | } 61 | 62 | func (cf ClientFactory) loadNetrc(ctx context.Context, lookupRef service.LookupRef) (string, error) { 63 | netrcPath := os.Getenv("NETRC") 64 | if netrcPath == "" { 65 | var err error 66 | 67 | netrcPath, err = homedir.Expand(filepath.Join("~", ".netrc")) 68 | if err != nil { 69 | return "", errors.Wrap(err, "expanding $HOME") 70 | } 71 | } 72 | 73 | fi, err := os.Stat(netrcPath) 74 | if err != nil { 75 | if os.IsNotExist(err) { 76 | return "", nil 77 | } 78 | 79 | return "", errors.Wrap(err, "checking file") 80 | } else if fi.IsDir() { 81 | // weird 82 | return "", nil 83 | } 84 | 85 | rc, err := netrc.ParseFile(netrcPath) 86 | if err != nil { 87 | return "", errors.Wrap(err, "parsing netrc") 88 | } 89 | 90 | machine := rc.FindMachine(lookupRef.Ref.Server) 91 | if machine == nil { 92 | return "", nil 93 | } 94 | 95 | cf.log.Infof("found authentication for %s: netrc %s", lookupRef.Ref.Server, netrcPath) 96 | 97 | return machine.Password, nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/service/gitlab/commit_ref.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/dpb587/gget/pkg/service" 10 | "github.com/dpb587/gget/pkg/service/gitlab/archive" 11 | "github.com/dpb587/gget/pkg/service/gitlab/blob" 12 | "github.com/dpb587/gget/pkg/service/gitlab/gitlabutil" 13 | "github.com/pkg/errors" 14 | "github.com/xanzy/go-gitlab" 15 | ) 16 | 17 | type CommitRef struct { 18 | client *gitlab.Client 19 | ref service.Ref 20 | commit string 21 | metadata service.RefMetadata 22 | 23 | archiveFileBase string 24 | } 25 | 26 | var _ service.ResolvedRef = &ReleaseRef{} 27 | var _ service.ResourceResolver = &CommitRef{} 28 | 29 | func (r *CommitRef) CanonicalRef() service.Ref { 30 | return r.ref 31 | } 32 | 33 | func (r *CommitRef) GetMetadata(ctx context.Context) (service.RefMetadata, error) { 34 | return r.metadata, nil 35 | } 36 | 37 | func (r *CommitRef) ResolveResource(ctx context.Context, resourceType service.ResourceType, resource service.ResourceName) ([]service.ResolvedResource, error) { 38 | switch resourceType { 39 | case service.ArchiveResourceType: 40 | return r.resolveArchiveResource(ctx, resource) 41 | case service.BlobResourceType: 42 | return r.resolveBlobResource(ctx, resource) 43 | } 44 | 45 | return nil, fmt.Errorf("unsupported resource type for commit ref: %s", resourceType) 46 | } 47 | 48 | func (r *CommitRef) resolveArchiveResource(ctx context.Context, resource service.ResourceName) ([]service.ResolvedResource, error) { 49 | // https://docs.gitlab.com/ce/api/repositories.html#get-file-archive 50 | candidates := []string{ 51 | fmt.Sprintf("%s.bz2", r.archiveFileBase), 52 | fmt.Sprintf("%s.tar", r.archiveFileBase), 53 | fmt.Sprintf("%s.tar.bz2", r.archiveFileBase), 54 | fmt.Sprintf("%s.tar.gz", r.archiveFileBase), 55 | fmt.Sprintf("%s.tb2", r.archiveFileBase), 56 | fmt.Sprintf("%s.tbz", r.archiveFileBase), 57 | fmt.Sprintf("%s.tbz2", r.archiveFileBase), 58 | fmt.Sprintf("%s.zip", r.archiveFileBase), 59 | } 60 | 61 | var res []service.ResolvedResource 62 | 63 | for _, candidate := range candidates { 64 | if match, _ := filepath.Match(string(resource), candidate); !match { 65 | continue 66 | } 67 | 68 | res = append( 69 | res, 70 | archive.NewResource( 71 | r.client, 72 | r.ref, 73 | r.commit, 74 | candidate, 75 | strings.TrimPrefix(candidate, fmt.Sprintf("%s.", r.archiveFileBase)), 76 | ), 77 | ) 78 | } 79 | 80 | return res, nil 81 | } 82 | 83 | func (r *CommitRef) resolveBlobResource(ctx context.Context, resource service.ResourceName) ([]service.ResolvedResource, error) { 84 | var res []service.ResolvedResource 85 | 86 | // get the full tree 87 | pt := true 88 | opts := &gitlab.ListTreeOptions{ 89 | ListOptions: gitlab.ListOptions{ 90 | PerPage: 100, 91 | }, 92 | Ref: &r.ref.Ref, 93 | Recursive: &pt, 94 | } 95 | 96 | for { 97 | nodes, resp, err := r.client.Repositories.ListTree(gitlabutil.GetRepositoryID(r.ref), opts) 98 | if err != nil { 99 | return nil, errors.Wrap(err, "getting commit tree") 100 | } 101 | 102 | for _, candidate := range nodes { 103 | if candidate.Type != "blob" { 104 | continue 105 | } else if match, _ := filepath.Match(string(resource), candidate.Path); !match { 106 | continue 107 | } 108 | 109 | res = append(res, blob.NewResource(r.client, r.ref, r.commit, candidate)) 110 | } 111 | 112 | if resp.NextPage == 0 { 113 | break 114 | } 115 | 116 | opts.ListOptions.Page = resp.NextPage 117 | } 118 | 119 | return res, nil 120 | } 121 | -------------------------------------------------------------------------------- /pkg/service/gitlab/gitlabutil/util.go: -------------------------------------------------------------------------------- 1 | package gitlabutil 2 | 3 | import ( 4 | "path" 5 | 6 | "github.com/dpb587/gget/pkg/service" 7 | ) 8 | 9 | func GetRepositoryID(ref service.Ref) string { 10 | return path.Join(ref.Owner, ref.Repository) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/service/gitlab/release_ref.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "path" 6 | "path/filepath" 7 | 8 | "github.com/dpb587/gget/pkg/checksum" 9 | "github.com/dpb587/gget/pkg/service" 10 | "github.com/dpb587/gget/pkg/service/gitlab/asset" 11 | "github.com/xanzy/go-gitlab" 12 | ) 13 | 14 | type ReleaseRef struct { 15 | client *gitlab.Client 16 | ref service.Ref 17 | release *gitlab.Release 18 | targetRef service.ResolvedRef 19 | 20 | checksumManager checksum.Manager 21 | } 22 | 23 | var _ service.ResolvedRef = &ReleaseRef{} 24 | var _ service.ResourceResolver = &ReleaseRef{} 25 | 26 | func (r *ReleaseRef) CanonicalRef() service.Ref { 27 | return r.ref 28 | } 29 | 30 | func (r *ReleaseRef) GetMetadata(ctx context.Context) (service.RefMetadata, error) { 31 | return r.targetRef.GetMetadata(ctx) 32 | } 33 | 34 | func (r *ReleaseRef) ResolveResource(ctx context.Context, resourceType service.ResourceType, resource service.ResourceName) ([]service.ResolvedResource, error) { 35 | if resourceType == service.AssetResourceType { 36 | return r.resolveAssetResource(ctx, resource) 37 | } 38 | 39 | return r.targetRef.ResolveResource(ctx, resourceType, resource) 40 | } 41 | 42 | func (r *ReleaseRef) resolveAssetResource(ctx context.Context, resource service.ResourceName) ([]service.ResolvedResource, error) { 43 | var res []service.ResolvedResource 44 | 45 | for _, candidate := range r.release.Assets.Links { 46 | // TODO kind of weird to extract from remote url "file" name, but 47 | // Name field is more traditionally a label. So currently using 48 | // the file name that a browser would typically produce. Doesn't 49 | // cover more complex URLs though. 50 | if match, _ := filepath.Match(string(resource), path.Base(candidate.URL)); !match { 51 | continue 52 | } 53 | 54 | res = append( 55 | res, 56 | asset.NewResource(r.client, r.ref.Owner, r.ref.Repository, candidate), 57 | ) 58 | } 59 | 60 | return res, nil 61 | } 62 | -------------------------------------------------------------------------------- /pkg/service/gitlab/service.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "path" 8 | "strings" 9 | 10 | "github.com/dpb587/gget/pkg/gitutil" 11 | "github.com/dpb587/gget/pkg/service" 12 | "github.com/dpb587/gget/pkg/service/gitlab/gitlabutil" 13 | "github.com/pkg/errors" 14 | "github.com/sirupsen/logrus" 15 | "github.com/xanzy/go-gitlab" 16 | ) 17 | 18 | type Service struct { 19 | log *logrus.Logger 20 | clientFactory *ClientFactory 21 | } 22 | 23 | func NewService(log *logrus.Logger, clientFactory *ClientFactory) *Service { 24 | return &Service{ 25 | log: log, 26 | clientFactory: clientFactory, 27 | } 28 | } 29 | 30 | var _ service.RefResolver = &Service{} 31 | var _ service.ConditionalRefResolver = &Service{} 32 | 33 | func (s Service) ServiceName() string { 34 | return "gitlab" 35 | } 36 | 37 | func (s Service) IsKnownServer(_ context.Context, lookupRef service.LookupRef) bool { 38 | return lookupRef.Ref.Server == "gitlab.com" 39 | } 40 | 41 | func (s Service) IsDetectedServer(_ context.Context, lookupRef service.LookupRef) bool { 42 | res, err := s.clientFactory.httpClientFactory().Head(fmt.Sprintf("https://%s/users/sign_in", lookupRef.Ref.Server)) 43 | if err != nil { 44 | s.log.Debugf("gitlab detection attempt error: %s", errors.Wrap(err, "requesting HEAD /users/sign_in")) 45 | 46 | return false 47 | } else if res.StatusCode != http.StatusOK { 48 | return false 49 | } 50 | 51 | for _, cookie := range res.Cookies() { 52 | if cookie.Name == "_gitlab_session" { 53 | return true 54 | } 55 | } 56 | 57 | return false 58 | } 59 | 60 | func (s Service) ResolveRef(ctx context.Context, lookupRef service.LookupRef) (service.ResolvedRef, error) { 61 | client, err := s.clientFactory.Get(ctx, lookupRef) 62 | if err != nil { 63 | return nil, errors.Wrap(err, "building client") 64 | } 65 | 66 | canonicalRef := lookupRef.Ref 67 | canonicalRef.Service = s.ServiceName() 68 | 69 | var cachedRelease *gitlab.Release 70 | 71 | if canonicalRef.Ref == "" { 72 | release, err := s.resolveLatest(ctx, client, lookupRef) 73 | if err != nil { 74 | return nil, errors.Wrap(err, "resolving latest") 75 | } 76 | 77 | canonicalRef.Ref = release.TagName 78 | cachedRelease = release 79 | } 80 | 81 | idPath := gitlabutil.GetRepositoryID(canonicalRef) 82 | 83 | { // tag 84 | tag, resp, err := client.Tags.GetTag(idPath, canonicalRef.Ref) 85 | if resp.StatusCode == http.StatusNotFound { 86 | // oh well 87 | } else if err != nil { 88 | return nil, errors.Wrap(err, "attempting tag resolution") 89 | } else if tag != nil { 90 | return s.resolveTagReference(ctx, client, canonicalRef, tag, cachedRelease) 91 | } 92 | } 93 | 94 | { // head 95 | branch, resp, err := client.Branches.GetBranch(idPath, canonicalRef.Ref) 96 | if resp.StatusCode == http.StatusNotFound { 97 | // oh well 98 | } else if err != nil { 99 | return nil, errors.Wrap(err, "attempting branch resolution") 100 | } else if branch != nil { 101 | return s.resolveHeadReference(ctx, client, canonicalRef, branch) 102 | } 103 | } 104 | 105 | if gitutil.PotentialCommitRE.MatchString(canonicalRef.Ref) { // commit 106 | commit, resp, err := client.Commits.GetCommit(idPath, canonicalRef.Ref) 107 | if resp.StatusCode == http.StatusNotFound { 108 | // oh well 109 | } else if err != nil { 110 | return nil, errors.Wrap(err, "attempting commit resolution") 111 | } else { 112 | canonicalRef.Ref = commit.ID 113 | 114 | return s.resolveCommitReference(ctx, client, canonicalRef, commit.ID) 115 | } 116 | } 117 | 118 | return nil, fmt.Errorf("unable to resolve as tag, branch, nor commit: %s", canonicalRef.Ref) 119 | } 120 | 121 | func (s Service) resolveLatest(ctx context.Context, client *gitlab.Client, lookupRef service.LookupRef) (*gitlab.Release, error) { 122 | idPath := gitlabutil.GetRepositoryID(lookupRef.Ref) 123 | 124 | opts := gitlab.ListReleasesOptions{ 125 | PerPage: 25, 126 | } 127 | 128 | for { 129 | releases, resp, err := client.Releases.ListReleases(idPath, &opts) 130 | if err != nil { 131 | return nil, errors.Wrap(err, "getting releases") 132 | } else if resp.StatusCode == http.StatusNotFound { 133 | return nil, errors.New("repository not found") 134 | } 135 | 136 | for _, release := range releases { 137 | if !lookupRef.SatisfiesStability("stable") { 138 | continue 139 | } 140 | 141 | tagName := release.TagName 142 | match, err := lookupRef.SatisfiesVersion(tagName) 143 | if err != nil { 144 | s.log.Debugf("skipping invalid semver tag: %s", tagName) 145 | 146 | continue 147 | } else if !match { 148 | continue 149 | } 150 | 151 | return release, nil 152 | } 153 | 154 | opts.Page = resp.NextPage 155 | 156 | if opts.Page == 0 { 157 | break 158 | } 159 | } 160 | 161 | if lookupRef.IsComplexRef() { 162 | return nil, fmt.Errorf("failed to find release matching constraints: %s", strings.Join(lookupRef.ComplexRefModes(), ", ")) 163 | } 164 | 165 | return nil, errors.New("no latest release found") 166 | } 167 | 168 | func (s Service) resolveCommitReference(ctx context.Context, client *gitlab.Client, ref service.Ref, commitSHA string) (service.ResolvedRef, error) { 169 | res := &CommitRef{ 170 | client: client, 171 | ref: ref, 172 | commit: commitSHA, 173 | archiveFileBase: fmt.Sprintf("%s-%s", ref.Repository, commitSHA[0:9]), 174 | metadata: service.RefMetadata{ 175 | { 176 | Name: "commit", 177 | Value: commitSHA, 178 | }, 179 | }, 180 | } 181 | 182 | return res, nil 183 | } 184 | 185 | func (s Service) resolveHeadReference(ctx context.Context, client *gitlab.Client, ref service.Ref, headRef *gitlab.Branch) (service.ResolvedRef, error) { 186 | branchName := headRef.Name 187 | commitSHA := headRef.Commit.ID 188 | 189 | res := &CommitRef{ 190 | client: client, 191 | ref: ref, 192 | commit: commitSHA, 193 | archiveFileBase: fmt.Sprintf("%s-%s", ref.Repository, path.Base(branchName)), 194 | metadata: service.RefMetadata{ 195 | { 196 | Name: "branch", 197 | Value: branchName, 198 | }, 199 | { 200 | Name: "commit", 201 | Value: commitSHA, 202 | }, 203 | }, 204 | } 205 | 206 | return res, nil 207 | } 208 | 209 | func (s Service) resolveTagReference(ctx context.Context, client *gitlab.Client, ref service.Ref, tagRef *gitlab.Tag, cachedRelease *gitlab.Release) (service.ResolvedRef, error) { 210 | tagName := tagRef.Name 211 | commitSHA := tagRef.Commit.ID 212 | 213 | var res service.ResolvedRef = &CommitRef{ 214 | client: client, 215 | ref: ref, 216 | commit: commitSHA, 217 | archiveFileBase: fmt.Sprintf("%s-%s", ref.Repository, tagName), 218 | metadata: service.RefMetadata{ 219 | { 220 | Name: "tag", 221 | Value: tagName, 222 | }, 223 | { 224 | Name: "commit", 225 | Value: commitSHA, 226 | }, 227 | }, 228 | } 229 | 230 | var release *gitlab.Release 231 | 232 | if cachedRelease != nil && cachedRelease.TagName == tagName { 233 | release = cachedRelease 234 | } else { 235 | var resp *gitlab.Response 236 | var err error 237 | 238 | release, resp, err = client.Releases.GetRelease(gitlabutil.GetRepositoryID(ref), tagName) 239 | if resp.StatusCode == http.StatusNotFound { 240 | // oh well 241 | } else if err != nil { 242 | return nil, errors.Wrap(err, "getting release by tag") 243 | } 244 | } 245 | 246 | if release != nil { 247 | res = &ReleaseRef{ 248 | client: client, 249 | ref: ref, 250 | release: release, 251 | targetRef: res, 252 | // checksumManager: NewReleaseChecksumManager(client, ref.Owner, ref.Repository, release), // TODO 253 | } 254 | } 255 | 256 | return res, nil 257 | } 258 | -------------------------------------------------------------------------------- /pkg/service/lookup_ref.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/Masterminds/semver" 7 | ) 8 | 9 | type LookupRef struct { 10 | Ref 11 | RefVersions []*semver.Constraints 12 | RefStability []string 13 | } 14 | 15 | func (lr LookupRef) SatisfiesStability(actual string) bool { 16 | if len(lr.RefStability) == 0 { 17 | return true 18 | } 19 | 20 | for _, desired := range lr.RefStability { 21 | if desired == "any" { 22 | return true 23 | } else if desired == actual { 24 | return true 25 | } 26 | } 27 | 28 | return false 29 | } 30 | 31 | func (lr LookupRef) SatisfiesVersion(actual string) (bool, error) { 32 | if len(lr.RefVersions) == 0 { 33 | return true, nil 34 | } 35 | 36 | ver, err := semver.NewVersion(strings.TrimPrefix(actual, "v")) 37 | if err != nil { 38 | return false, err 39 | } 40 | 41 | for _, desired := range lr.RefVersions { 42 | if !desired.Check(ver) { 43 | return false, nil 44 | } 45 | } 46 | 47 | return true, nil 48 | } 49 | 50 | func (lr LookupRef) ComplexRefModes() []string { 51 | var res []string 52 | 53 | lv := len(lr.RefVersions) 54 | if lv > 0 { 55 | res = append(res, "version") 56 | } 57 | 58 | ls := len(lr.RefStability) 59 | if ls > 0 { 60 | if lv == 0 && ls == 1 && lr.RefStability[0] == "stable" { 61 | // explicit default; shortcut this to allow services to use cheaper APIs 62 | return nil 63 | } 64 | 65 | res = append(res, "stability") 66 | } 67 | 68 | return res 69 | } 70 | 71 | func (lr LookupRef) IsComplexRef() bool { 72 | return len(lr.ComplexRefModes()) > 0 73 | } 74 | -------------------------------------------------------------------------------- /pkg/service/multi_ref_resolver.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | type MultiRefResolver struct { 11 | log *logrus.Logger 12 | resolvers []ConditionalRefResolver 13 | } 14 | 15 | var _ RefResolver = MultiRefResolver{} 16 | 17 | func NewMultiRefResolver(log *logrus.Logger, resolvers ...ConditionalRefResolver) RefResolver { 18 | return MultiRefResolver{ 19 | log: log, 20 | resolvers: resolvers, 21 | } 22 | } 23 | 24 | type ConditionalRefResolver interface { 25 | RefResolver 26 | 27 | ServiceName() string 28 | IsKnownServer(context.Context, LookupRef) bool 29 | IsDetectedServer(context.Context, LookupRef) bool 30 | } 31 | 32 | func (rr MultiRefResolver) ResolveRef(ctx context.Context, lookupRef LookupRef) (ResolvedRef, error) { 33 | if serviceName := lookupRef.Service; serviceName != "" { 34 | for _, resolver := range rr.resolvers { 35 | if resolver.ServiceName() != serviceName { 36 | continue 37 | } 38 | 39 | rr.log.Infof("using service based on ref: %s", resolver.ServiceName()) 40 | 41 | return resolver.ResolveRef(ctx, lookupRef) 42 | } 43 | 44 | return nil, fmt.Errorf("service not recognized: %s", serviceName) 45 | } 46 | 47 | for _, resolver := range rr.resolvers { 48 | if !resolver.IsKnownServer(ctx, lookupRef) { 49 | continue 50 | } 51 | 52 | rr.log.Infof("using service based on known servers: %s", resolver.ServiceName()) 53 | 54 | return resolver.ResolveRef(ctx, lookupRef) 55 | } 56 | 57 | rr.log.Debugf("attempting ref server detection (ref server not known)") 58 | 59 | for _, resolver := range rr.resolvers { 60 | if !resolver.IsDetectedServer(ctx, lookupRef) { 61 | continue 62 | } 63 | 64 | rr.log.Infof("using service based on server detection: %s", resolver.ServiceName()) 65 | 66 | return resolver.ResolveRef(ctx, lookupRef) 67 | } 68 | 69 | return nil, fmt.Errorf("failed to find service for ref: %s", lookupRef) 70 | } 71 | -------------------------------------------------------------------------------- /pkg/service/ref.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strings" 7 | ) 8 | 9 | type Ref struct { 10 | Service string 11 | Server string 12 | Owner string 13 | Repository string 14 | Ref string 15 | } 16 | 17 | func (r Ref) String() string { 18 | str := r.Server 19 | 20 | if str == "" { 21 | str = "{:server-missing}" 22 | } 23 | 24 | str = path.Join(str, r.Owner, r.Repository) 25 | 26 | if r.Ref != "" { 27 | str = fmt.Sprintf("%s@%s", str, r.Ref) 28 | } 29 | 30 | return str 31 | } 32 | 33 | func ParseRefString(in string) (Ref, error) { 34 | slugVersion := strings.SplitN(in, "@", 2) 35 | ownerRepo := strings.SplitN(slugVersion[0], "/", 3) 36 | 37 | res := Ref{} 38 | 39 | if len(slugVersion) == 2 { 40 | res.Ref = slugVersion[1] 41 | } else { 42 | res.Ref = "" 43 | } 44 | 45 | if len(ownerRepo) == 3 { 46 | res.Server = ownerRepo[0] 47 | res.Owner = ownerRepo[1] 48 | res.Repository = ownerRepo[2] 49 | } else if len(ownerRepo) == 2 { 50 | res.Server = "" 51 | res.Owner = ownerRepo[0] 52 | res.Repository = ownerRepo[1] 53 | } else { 54 | return Ref{}, fmt.Errorf("input does not match expected format: [server/]owner/repository[@version]; received %s", in) 55 | } 56 | 57 | return res, nil 58 | } 59 | 60 | type RefMetadatum struct { 61 | Name string 62 | Value string 63 | } 64 | 65 | type RefMetadata []RefMetadatum 66 | -------------------------------------------------------------------------------- /pkg/service/ref_resolver.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type RefResolver interface { 8 | ResolveRef(ctx context.Context, ref LookupRef) (ResolvedRef, error) 9 | } 10 | 11 | type ResolvedRef interface { 12 | ResourceResolver 13 | 14 | CanonicalRef() Ref 15 | GetMetadata(ctx context.Context) (RefMetadata, error) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/service/resource.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | type ResourceType string 4 | 5 | // ArchiveResourceType is a tar/zip export of the repository from the ref. 6 | const ArchiveResourceType ResourceType = "archive" 7 | 8 | // AssetResourceType is a user-provided file associated with the ref. 9 | const AssetResourceType ResourceType = "asset" 10 | 11 | // BlobResourceType is a blob of the repository at the ref. 12 | const BlobResourceType ResourceType = "blob" 13 | 14 | type ResourceName string 15 | -------------------------------------------------------------------------------- /pkg/service/resource_resolver.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/dpb587/gget/pkg/checksum" 8 | ) 9 | 10 | type ResourceResolver interface { 11 | ResolveResource(ctx context.Context, resourceType ResourceType, resource ResourceName) ([]ResolvedResource, error) 12 | } 13 | 14 | type ResolvedResource interface { 15 | GetName() string 16 | GetSize() int64 17 | Open(ctx context.Context) (io.ReadCloser, error) 18 | } 19 | 20 | type ChecksumSupportedResolvedResource interface { 21 | GetChecksums(ctx context.Context, algos checksum.AlgorithmList) (checksum.ChecksumList, error) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/transfer/batch.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/pkg/errors" 11 | "github.com/sirupsen/logrus" 12 | "github.com/tidwall/limiter" 13 | "github.com/vbauerster/mpb/v4" 14 | ) 15 | 16 | type Batch struct { 17 | log logrus.FieldLogger 18 | transfers []*Transfer 19 | limiter *limiter.Limiter 20 | output io.Writer 21 | 22 | errs []string 23 | errsM sync.Mutex 24 | } 25 | 26 | func NewBatch(log logrus.FieldLogger, transfers []*Transfer, parallel int, output io.Writer) *Batch { 27 | return &Batch{ 28 | log: log, 29 | transfers: transfers, 30 | limiter: limiter.New(parallel), 31 | output: output, 32 | } 33 | } 34 | 35 | func (b *Batch) Transfer(ctx context.Context, failFast bool) error { 36 | pb := mpb.New(mpb.WithWidth(1), mpb.WithOutput(b.output)) 37 | 38 | for _, d := range b.transfers { 39 | d.Prepare(pb) 40 | } 41 | 42 | var cancel context.CancelFunc 43 | if failFast { 44 | ctx, cancel = context.WithCancel(ctx) 45 | } 46 | 47 | for idx := range b.transfers { 48 | go b.transfer(idx, ctx, cancel) 49 | } 50 | 51 | pb.Wait() 52 | 53 | if len(b.errs) > 0 { 54 | return fmt.Errorf("transfers failed: %s", strings.Join(b.errs, ", ")) 55 | } 56 | 57 | return nil 58 | } 59 | 60 | func (b *Batch) transfer(idx int, ctx context.Context, cancel context.CancelFunc) { 61 | b.limiter.Begin() 62 | defer b.limiter.End() 63 | 64 | xfer := b.transfers[idx] 65 | 66 | b.errsM.Lock() 67 | errsLen := len(b.errs) 68 | b.errsM.Unlock() 69 | 70 | if errsLen > 0 && cancel != nil { 71 | xfer.finalize("!", "skipped (due to previous error)") 72 | 73 | return 74 | } 75 | 76 | if err := xfer.Execute(ctx); err != nil { 77 | // only warning since it should be handled/printed elsewhere 78 | b.log.Warn(errors.Wrapf(err, "downloading %s", xfer.GetSubject())) 79 | 80 | b.errsM.Lock() 81 | if cancel != nil && len(b.errs) > 0 { 82 | // assume context was canceled and ignore as root cause 83 | xfer.finalize("!", "aborted (due to previous error)") 84 | } else { 85 | b.errs = append(b.errs, xfer.GetSubject()) 86 | xfer.finalize("X", fmt.Sprintf("errored (%s)", err)) 87 | } 88 | b.errsM.Unlock() 89 | 90 | if cancel != nil { 91 | cancel() 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/transfer/state.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/vbauerster/mpb/v4" 8 | ) 9 | 10 | type State struct { 11 | Bar *mpb.Bar 12 | LocalFilePath string 13 | Results []string 14 | } 15 | 16 | type DownloadAsset interface { 17 | GetName() string 18 | GetSize() int64 19 | Open(ctx context.Context) (io.ReadCloser, error) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/transfer/step.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/vbauerster/mpb/v4/decor" 7 | ) 8 | 9 | type Step interface { 10 | GetProgressParams() (int64, decor.Decorator) 11 | Execute(ctx context.Context, s *State) error 12 | } 13 | -------------------------------------------------------------------------------- /pkg/transfer/step/executable.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/dpb587/gget/pkg/transfer" 8 | "github.com/pkg/errors" 9 | "github.com/vbauerster/mpb/v4/decor" 10 | ) 11 | 12 | type Executable struct{} 13 | 14 | var _ transfer.Step = &Executable{} 15 | 16 | func (dpi Executable) GetProgressParams() (int64, decor.Decorator) { 17 | return 0, nil 18 | } 19 | 20 | func (dpi Executable) Execute(_ context.Context, state *transfer.State) error { 21 | err := os.Chmod(state.LocalFilePath, 0755) 22 | if err != nil { 23 | return errors.Wrap(err, "chmod'ing") 24 | } 25 | 26 | state.Results = append(state.Results, "executable") 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/transfer/step/rename.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/dpb587/gget/pkg/transfer" 8 | "github.com/pkg/errors" 9 | "github.com/vbauerster/mpb/v4/decor" 10 | ) 11 | 12 | type Rename struct { 13 | Target string 14 | } 15 | 16 | var _ transfer.Step = &Rename{} 17 | 18 | func (dpi Rename) GetProgressParams() (int64, decor.Decorator) { 19 | return 0, nil 20 | } 21 | 22 | func (dpi Rename) Execute(_ context.Context, state *transfer.State) error { 23 | err := os.Rename(state.LocalFilePath, dpi.Target) 24 | if err != nil { 25 | return errors.Wrap(err, "renaming") 26 | } 27 | 28 | state.LocalFilePath = dpi.Target 29 | 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/transfer/step/temp_file_target.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/dpb587/gget/pkg/transfer" 10 | "github.com/pkg/errors" 11 | "github.com/vbauerster/mpb/v4/decor" 12 | ) 13 | 14 | type TempFileTarget struct { 15 | Tmpdir string 16 | 17 | tmpfile *os.File 18 | } 19 | 20 | var _ transfer.Step = &TempFileTarget{} 21 | var _ io.Writer = &TempFileTarget{} 22 | 23 | func (dpi *TempFileTarget) GetProgressParams() (int64, decor.Decorator) { 24 | return 0, nil 25 | } 26 | 27 | func (dpi *TempFileTarget) Write(p []byte) (int, error) { 28 | if dpi.tmpfile == nil { 29 | p, err := ioutil.TempFile(dpi.Tmpdir, ".gget-*") 30 | if err != nil { 31 | return 0, errors.Wrap(err, "creating tempfile") 32 | } 33 | 34 | dpi.tmpfile = p 35 | } 36 | 37 | return dpi.tmpfile.Write(p) 38 | } 39 | 40 | func (dpi *TempFileTarget) Execute(_ context.Context, s *transfer.State) error { 41 | err := dpi.tmpfile.Close() 42 | if err != nil { 43 | return errors.Wrap(err, "closing file") 44 | } 45 | 46 | s.LocalFilePath = dpi.tmpfile.Name() 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /pkg/transfer/step/verify_checksum.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/dpb587/gget/pkg/checksum" 9 | "github.com/dpb587/gget/pkg/transfer" 10 | "github.com/vbauerster/mpb/v4/decor" 11 | ) 12 | 13 | type VerifyChecksum struct { 14 | Verifier *checksum.HashVerifier 15 | } 16 | 17 | var _ transfer.Step = &VerifyChecksum{} 18 | var _ io.Writer = &VerifyChecksum{} 19 | 20 | func (dhv *VerifyChecksum) GetProgressParams() (int64, decor.Decorator) { 21 | name := fmt.Sprintf("verifying (%s)", dhv.Verifier.Algorithm()) 22 | 23 | return 1, decor.Name(name, decor.WC{W: len(name), C: decor.DidentRight}) 24 | } 25 | 26 | func (dhv *VerifyChecksum) Write(in []byte) (n int, err error) { 27 | return dhv.Verifier.Write(in) 28 | } 29 | 30 | func (dhv *VerifyChecksum) Execute(_ context.Context, s *transfer.State) error { 31 | err := dhv.Verifier.Verify() 32 | if err != nil { 33 | return err 34 | } 35 | 36 | s.Results = append(s.Results, fmt.Sprintf("%s OK", dhv.Verifier.Algorithm())) 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/transfer/step/writer_target.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/dpb587/gget/pkg/transfer" 8 | "github.com/vbauerster/mpb/v4/decor" 9 | ) 10 | 11 | type WriterTarget struct { 12 | FilePath string 13 | Writer io.Writer 14 | } 15 | 16 | var _ transfer.Step = &WriterTarget{} 17 | var _ io.Writer = &WriterTarget{} 18 | 19 | func (dpi *WriterTarget) GetProgressParams() (int64, decor.Decorator) { 20 | return 0, nil 21 | } 22 | 23 | func (dpi *WriterTarget) Write(p []byte) (int, error) { 24 | return dpi.Writer.Write(p) 25 | } 26 | 27 | func (dpi *WriterTarget) Execute(_ context.Context, s *transfer.State) error { 28 | if dpi.FilePath != "" { 29 | s.LocalFilePath = dpi.FilePath 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /pkg/transfer/transfer.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/pkg/errors" 10 | "github.com/vbauerster/mpb/v4" 11 | "github.com/vbauerster/mpb/v4/decor" 12 | ) 13 | 14 | type Transfer struct { 15 | origin DownloadAsset 16 | steps []Step 17 | finalStatus io.Writer 18 | 19 | pb *mpb.Progress 20 | bars []*mpb.Bar 21 | } 22 | 23 | func NewTransfer(origin DownloadAsset, steps []Step, finalStatus io.Writer) *Transfer { 24 | return &Transfer{ 25 | origin: origin, 26 | steps: steps, 27 | finalStatus: finalStatus, 28 | } 29 | } 30 | 31 | func (w *Transfer) GetSubject() string { 32 | return w.origin.GetName() 33 | } 34 | 35 | func (w *Transfer) Prepare(pb *mpb.Progress) { 36 | w.pb = pb 37 | w.bars = make([]*mpb.Bar, len(w.steps)+5) 38 | 39 | w.bars[0] = w.newBar(pb, nil, " ", 1, decor.Name( 40 | "waiting", 41 | decor.WC{W: 7, C: decor.DidentRight}, 42 | )) 43 | 44 | w.bars[1] = w.newBar(pb, w.bars[0], "", 1, decor.Name( 45 | "connecting", 46 | decor.WC{W: 10, C: decor.DidentRight}, 47 | )) 48 | 49 | downloadSize := w.origin.GetSize() 50 | downloadDecor := decor.NewPercentage("downloading (%d)") 51 | if downloadSize == 0 { 52 | downloadDecor = decor.EwmaSpeed(decor.UnitKB, "downloading (%d)", 40) 53 | } 54 | 55 | w.bars[2] = w.newBar(pb, w.bars[1], "", downloadSize, decor.OnComplete( 56 | downloadDecor, 57 | "downloaded", 58 | )) 59 | 60 | lastBar := w.bars[2] 61 | 62 | for stepIdx, step := range w.steps { 63 | count, msg := step.GetProgressParams() 64 | if count == 0 { 65 | continue 66 | } 67 | 68 | w.bars[stepIdx+3] = w.newBar(pb, lastBar, "", count, msg) 69 | lastBar = w.bars[stepIdx+3] 70 | } 71 | 72 | w.bars[len(w.steps)+3] = w.newBar(pb, lastBar, "", 1, decor.Name( 73 | "finishing", 74 | decor.WC{W: 9, C: decor.DidentRight}, 75 | )) 76 | } 77 | 78 | func (w Transfer) Execute(ctx context.Context) error { 79 | { // waiting 80 | w.bars[0].SetTotal(1, true) 81 | } 82 | 83 | var assetHandle io.ReadCloser 84 | 85 | { // connecting 86 | var err error 87 | 88 | assetHandle, err = w.origin.Open(ctx) 89 | if err != nil { 90 | return errors.Wrap(err, "connecting") 91 | } 92 | 93 | w.bars[1].SetTotal(1, true) 94 | } 95 | 96 | defer assetHandle.Close() 97 | 98 | { // downloading 99 | r := w.bars[2].ProxyReader(assetHandle) 100 | defer r.Close() 101 | 102 | var dw io.Writer 103 | 104 | { // enumerate writers 105 | var writers []io.Writer 106 | 107 | for _, step := range w.steps { 108 | writer, ok := step.(io.Writer) 109 | if !ok { 110 | continue 111 | } 112 | 113 | writers = append(writers, writer) 114 | } 115 | 116 | if len(writers) == 0 { 117 | return fmt.Errorf("no download target found") 118 | } 119 | 120 | dw = io.MultiWriter(writers...) 121 | } 122 | 123 | _, err := io.Copy(dw, r) 124 | if err != nil { 125 | return errors.Wrap(err, "downloading") 126 | } 127 | 128 | w.bars[2].SetTotal(int64(w.origin.GetSize()), true) 129 | } 130 | 131 | var results []string 132 | 133 | { // stepwise 134 | state := State{} 135 | 136 | for stepIdx, step := range w.steps { 137 | state.Bar = w.bars[stepIdx+3] 138 | 139 | err := step.Execute(ctx, &state) 140 | if err != nil { 141 | return errors.Wrapf(err, "processing step %d", stepIdx) 142 | } 143 | 144 | if state.Bar != nil { 145 | count, _ := step.GetProgressParams() 146 | state.Bar.SetTotal(count, true) 147 | } 148 | } 149 | 150 | results = state.Results 151 | } 152 | 153 | { // done 154 | summary := fmt.Sprintf("done") 155 | if len(results) > 0 { 156 | summary = fmt.Sprintf("%s (%s)", summary, strings.Join(results, "; ")) 157 | } 158 | 159 | w.finalize("√", summary) 160 | } 161 | 162 | return nil 163 | } 164 | 165 | func (w Transfer) finalize(status, description string) { 166 | w.bars[len(w.steps)+4] = w.newBar(w.pb, w.bars[len(w.steps)+3], status, 1, decor.Name( 167 | description, 168 | decor.WC{W: len(description), C: decor.DidentRight}, 169 | )) 170 | 171 | // make sure everything is closed out 172 | for _, bar := range w.bars { 173 | if bar == nil || bar.Completed() { 174 | continue 175 | } 176 | 177 | bar.SetTotal(1, true) 178 | } 179 | 180 | if w.finalStatus != nil { 181 | fmt.Fprintf(w.finalStatus, "%s %s\n", w.GetSubject(), description) 182 | } 183 | } 184 | 185 | func (w Transfer) newBar(pb *mpb.Progress, pbp *mpb.Bar, spinner string, count int64, msg decor.Decorator) *mpb.Bar { 186 | var spinnerd decor.Decorator 187 | 188 | switch spinner { 189 | case "": 190 | spinnerd = decor.Spinner( 191 | mpb.DefaultSpinnerStyle, 192 | decor.WC{W: 1, C: decor.DSyncSpaceR}, 193 | ) 194 | default: 195 | spinnerd = decor.Name( 196 | spinner, 197 | decor.WC{W: 1, C: decor.DSyncSpaceR}, 198 | ) 199 | } 200 | 201 | subject := w.origin.GetName() 202 | 203 | return pb.AddBar( 204 | count, 205 | mpb.BarParkTo(pbp), 206 | mpb.PrependDecorators( 207 | spinnerd, 208 | decor.Name( 209 | subject, 210 | decor.WC{W: len(subject), C: decor.DSyncSpaceR}, 211 | ), 212 | msg, 213 | ), 214 | ) 215 | } 216 | -------------------------------------------------------------------------------- /pkg/transfer/transferutil/transfer_factory.go: -------------------------------------------------------------------------------- 1 | package transferutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/dpb587/gget/pkg/checksum" 11 | "github.com/dpb587/gget/pkg/service" 12 | "github.com/dpb587/gget/pkg/transfer" 13 | "github.com/dpb587/gget/pkg/transfer/step" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | func BuildTransfer(ctx context.Context, origin transfer.DownloadAsset, targetPath string, opts TransferOptions) (*transfer.Transfer, error) { 18 | var steps []transfer.Step 19 | 20 | if targetPath == "-" { 21 | steps = append( 22 | steps, 23 | &step.WriterTarget{ 24 | Writer: os.Stdout, 25 | }, 26 | ) 27 | } else { 28 | steps = append( 29 | steps, 30 | &step.TempFileTarget{ 31 | Tmpdir: filepath.Dir(targetPath), 32 | }, 33 | ) 34 | } 35 | 36 | if len(opts.ChecksumVerification.Acceptable) > 0 { 37 | var csl checksum.ChecksumList 38 | 39 | if csr, ok := origin.(service.ChecksumSupportedResolvedResource); ok { 40 | avail, err := csr.GetChecksums(ctx, opts.ChecksumVerification.Acceptable) 41 | if err != nil { 42 | return nil, errors.Wrap(err, "getting checksum") 43 | } 44 | 45 | csl = opts.ChecksumVerification.Selector.SelectChecksums(avail) 46 | } 47 | 48 | if opts.ChecksumVerification.Required && len(csl) == 0 { 49 | return nil, fmt.Errorf("checksum required but not found: %s", opts.ChecksumVerification.Acceptable.Join(", ")) 50 | } 51 | 52 | for _, cs := range csl { 53 | verifier, err := cs.NewVerifier(ctx) 54 | if err != nil { 55 | return nil, errors.Wrapf(err, "getting %s verifier", cs.Algorithm()) 56 | } 57 | 58 | steps = append( 59 | steps, 60 | &step.VerifyChecksum{ 61 | Verifier: verifier, 62 | }, 63 | ) 64 | } 65 | } 66 | 67 | if targetPath != "-" { 68 | if opts.Executable { 69 | steps = append( 70 | steps, 71 | &step.Executable{}, 72 | ) 73 | } 74 | 75 | steps = append( 76 | steps, 77 | &step.Rename{ 78 | Target: targetPath, 79 | }, 80 | ) 81 | } 82 | 83 | return transfer.NewTransfer(origin, steps, opts.FinalStatus), nil 84 | } 85 | 86 | type TransferOptions struct { 87 | Executable bool 88 | ChecksumVerification checksum.VerificationProfile 89 | FinalStatus io.Writer 90 | } 91 | -------------------------------------------------------------------------------- /scripts/build.dockerized.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." 6 | 7 | docker build -t dpb587/gget:build build/docker/build 8 | docker run --rm \ 9 | --volume="${PWD}":/root \ 10 | --workdir=/root \ 11 | --user=$( id -u "${USER}" ):$( id -g "${USER}" ) \ 12 | --env=GOCACHE=/tmp/.cache/go-build \ 13 | dpb587/gget:build \ 14 | ./scripts/build.local.sh "$@" 15 | -------------------------------------------------------------------------------- /scripts/build.local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # args: [version] 3 | 4 | set -eu 5 | 6 | cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." 7 | 8 | version="${1:-0.0.0}" 9 | 10 | if [ -z "${os_list:-}" ]; then 11 | os_list="darwin linux windows" 12 | fi 13 | 14 | if [ -z "${arch_list:-}" ]; then 15 | arch_list="amd64" 16 | fi 17 | 18 | built=$( date -u +%Y-%m-%dT%H:%M:%S+00:00 ) 19 | commit=$( git rev-parse HEAD | cut -c-10 ) 20 | 21 | if [[ $( git clean -dnx | wc -l ) -gt 0 ]] ; then 22 | commit="${commit}+dirty" 23 | 24 | if [[ "${version}" != "0.0.0" ]]; then 25 | echo "ERROR: building an official version requires a clean repository" 26 | echo "WARN: refusing to clean repository" 27 | git clean -dnx 28 | 29 | exit 1 30 | fi 31 | fi 32 | 33 | mkdir -p tmp/build 34 | 35 | export CGO_ENABLED=0 36 | 37 | cli=gget 38 | 39 | for os in $os_list ; do 40 | for arch in $arch_list ; do 41 | name=$cli-$version-$os-$arch 42 | 43 | if [ "$os" == "windows" ]; then 44 | name=$name.exe 45 | fi 46 | 47 | echo "$name" 48 | GOOS=$os GOARCH=$arch go build \ 49 | -ldflags " 50 | -s -w 51 | -X main.appSemver=$version 52 | -X main.appCommit=$commit 53 | -X main.appBuilt=$built 54 | " \ 55 | -o tmp/build/$name \ 56 | . 57 | 58 | # TODO 59 | # if which upx > /dev/null ; then 60 | # upx --ultra-brute tmp/build/$name 61 | # fi 62 | done 63 | done 64 | -------------------------------------------------------------------------------- /scripts/integration.test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | mkdir -p tmp/integrationtest 6 | 7 | go build -o tmp/integrationtest/gget . 8 | 9 | rm -fr tmp/integrationtest/workdir 10 | mkdir tmp/integrationtest/workdir 11 | cd tmp/integrationtest/workdir 12 | 13 | # dump-info, ref-version constraints, match constraint, cd 14 | 15 | ../gget github.com/dpb587/gget --ref-version=0.2.x --export=json --cd="${PWD}" '*linux*' > export.json 16 | 17 | diff <( shasum * ) - < /dev/null 102 | 103 | # version validation 104 | 105 | ../gget --version=0.0.0 > /dev/null 106 | 107 | # help 108 | 109 | ../gget --help > /dev/null 110 | 111 | # done 112 | 113 | cd ../../ 114 | 115 | rm -fr tmp/integrationtest 116 | 117 | echo Tests Successful 118 | -------------------------------------------------------------------------------- /scripts/website.build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | export BASE_URL=https://gget.io/ 6 | export GA_TRACKING_ID=UA-37464314-3 7 | 8 | suffix="$( go run . --version -vv | grep ^runtime | tr -d ';' | awk '{ print $7 "-" $5 }' )" 9 | go run . github.com/dpb587/gget --no-download --export=json > website/static/latest.json 10 | go run . github.com/dpb587/gget --executable website/static/gget="*-${suffix}" 11 | ./website/static/gget --help > website/static/latest-help.txt 12 | rm website/static/gget 13 | 14 | cd website/ 15 | 16 | npm ci 17 | yarn build 18 | yarn start & 19 | nuxt=$! 20 | 21 | sleep 5 22 | rm -fr dist 23 | cp -rp static dist 24 | cp -rp .nuxt/dist/client dist/_nuxt 25 | 26 | curl http://localhost:3000/ > dist/index.html 27 | 28 | kill "${nuxt}" 29 | 30 | cd dist 31 | 32 | touch .nojekyll 33 | echo -n gget.io > CNAME 34 | 35 | git init . 36 | git add . 37 | git commit -m 'regenerate' 38 | git branch -m gh-pages 39 | git remote add origin git@github.com:dpb587/gget.git 40 | git config branch.gh-pages.remote origin 41 | git config branch.gh-pages.merge refs/heads/gh-pages 42 | -------------------------------------------------------------------------------- /website/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | /static/latest.json 2 | /static/latest-help.txt 3 | 4 | # Created by .ignore support plugin (hsz.mobi) 5 | ### Node template 6 | # Logs 7 | /logs 8 | *.log 9 | npm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # Optional npm cache directory 48 | .npm 49 | 50 | # Optional eslint cache 51 | .eslintcache 52 | 53 | # Optional REPL history 54 | .node_repl_history 55 | 56 | # Output of 'npm pack' 57 | *.tgz 58 | 59 | # Yarn Integrity file 60 | .yarn-integrity 61 | 62 | # dotenv environment variables file 63 | .env 64 | 65 | # parcel-bundler cache (https://parceljs.org/) 66 | .cache 67 | 68 | # next.js build output 69 | .next 70 | 71 | # nuxt.js build output 72 | .nuxt 73 | 74 | # Nuxt generate 75 | dist 76 | 77 | # vuepress build output 78 | .vuepress/dist 79 | 80 | # Serverless directories 81 | .serverless 82 | 83 | # IDE / Editor 84 | .idea 85 | 86 | # Service worker 87 | sw.* 88 | 89 | # macOS 90 | .DS_Store 91 | 92 | # Vim swap files 93 | *.swp 94 | -------------------------------------------------------------------------------- /website/assets/asciinema/hugo.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 80, "height": 4, "timestamp": 1593360624, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} 2 | [0.00, "o", "$ "] 3 | [0.50, "o", "g"] 4 | [0.55, "o", "g"] 5 | [0.60, "o", "e"] 6 | [0.65, "o", "t"] 7 | [0.70, "o", " "] 8 | [1.45, "o", "g"] 9 | [1.50, "o", "i"] 10 | [1.55, "o", "t"] 11 | [1.60, "o", "h"] 12 | [1.65, "o", "u"] 13 | [1.70, "o", "b"] 14 | [1.75, "o", "."] 15 | [1.80, "o", "c"] 16 | [1.85, "o", "o"] 17 | [1.90, "o", "m"] 18 | [1.95, "o", "/"] 19 | [2.00, "o", "g"] 20 | [2.05, "o", "o"] 21 | [2.10, "o", "h"] 22 | [2.15, "o", "u"] 23 | [2.20, "o", "g"] 24 | [2.25, "o", "o"] 25 | [2.30, "o", "i"] 26 | [2.35, "o", "o"] 27 | [2.40, "o", "/"] 28 | [2.45, "o", "h"] 29 | [2.50, "o", "u"] 30 | [2.55, "o", "g"] 31 | [2.60, "o", "o"] 32 | [2.65, "o", " "] 33 | [3.40, "o", "'"] 34 | [3.45, "o", "h"] 35 | [3.50, "o", "u"] 36 | [3.55, "o", "g"] 37 | [3.60, "o", "o"] 38 | [3.65, "o", "_"] 39 | [3.70, "o", "e"] 40 | [3.75, "o", "x"] 41 | [3.80, "o", "t"] 42 | [3.85, "o", "e"] 43 | [3.90, "o", "n"] 44 | [3.95, "o", "d"] 45 | [4.00, "o", "e"] 46 | [4.05, "o", "d"] 47 | [4.10, "o", "_"] 48 | [4.15, "o", "*"] 49 | [4.20, "o", "_"] 50 | [4.25, "o", "L"] 51 | [4.30, "o", "i"] 52 | [4.35, "o", "n"] 53 | [4.40, "o", "u"] 54 | [4.45, "o", "x"] 55 | [4.50, "o", "-"] 56 | [4.55, "o", "6"] 57 | [4.60, "o", "4"] 58 | [4.65, "o", "b"] 59 | [4.70, "o", "i"] 60 | [4.75, "o", "t"] 61 | [4.80, "o", "."] 62 | [4.85, "o", "d"] 63 | [4.90, "o", "e"] 64 | [4.95, "o", "b"] 65 | [5.00, "o", "'"] 66 | [5.90, "o", "\r\n"] 67 | [6.80, "o", "Found 1 file (13.9M) from github.com/gohugoio/hugo@v0.73.0\r\n"] 68 | [7.00, "o", " hugo_extended_0.73.0_Linux-64bit.deb waiting \r\n"] 69 | [7.25, "o", "\u001b[1A\u001b[J hugo_extended_0.73.0_Linux-64bit.deb waiting \r\n\u001b[1A\u001b[J⠋ hugo_extended_0.73.0_Linux-64bit.deb connecting \r\n"] 70 | [7.50, "o", "\u001b[1A\u001b[J⠙ hugo_extended_0.73.0_Linux-64bit.deb connecting \r\n"] 71 | [7.75, "o", "\u001b[1A\u001b[J⠹ hugo_extended_0.73.0_Linux-64bit.deb connecting \r\n"] 72 | [8.00, "o", "\u001b[1A\u001b[J⠸ hugo_extended_0.73.0_Linux-64bit.deb connecting \r\n"] 73 | [8.25, "o", "\u001b[1A\u001b[J⠼ hugo_extended_0.73.0_Linux-64bit.deb downloading (0%) \r\n"] 74 | [8.50, "o", "\u001b[1A\u001b[J⠴ hugo_extended_0.73.0_Linux-64bit.deb downloading (0%) \r\n"] 75 | [8.75, "o", "\u001b[1A\u001b[J⠦ hugo_extended_0.73.0_Linux-64bit.deb downloading (1%) \r\n"] 76 | [9.00, "o", "\u001b[1A\u001b[J⠧ hugo_extended_0.73.0_Linux-64bit.deb downloading (2%) \r\n"] 77 | [9.25, "o", "\u001b[1A\u001b[J⠇ hugo_extended_0.73.0_Linux-64bit.deb downloading (4%) \r\n"] 78 | [9.50, "o", "\u001b[1A\u001b[J⠏ hugo_extended_0.73.0_Linux-64bit.deb downloading (7%) \r\n"] 79 | [9.75, "o", "\u001b[1A\u001b[J⠋ hugo_extended_0.73.0_Linux-64bit.deb downloading (10%) \r\n"] 80 | [10.00, "o", "\u001b[1A\u001b[J⠙ hugo_extended_0.73.0_Linux-64bit.deb downloading (15%) \r\n"] 81 | [10.25, "o", "\u001b[1A\u001b[J⠹ hugo_extended_0.73.0_Linux-64bit.deb downloading (19%) \r\n"] 82 | [10.50, "o", "\u001b[1A\u001b[J⠸ hugo_extended_0.73.0_Linux-64bit.deb downloading (24%) \r\n"] 83 | [10.75, "o", "\u001b[1A\u001b[J⠼ hugo_extended_0.73.0_Linux-64bit.deb downloading (30%) \r\n"] 84 | [11.00, "o", "\u001b[1A\u001b[J⠴ hugo_extended_0.73.0_Linux-64bit.deb downloading (37%) \r\n"] 85 | [11.25, "o", "\u001b[1A\u001b[J⠦ hugo_extended_0.73.0_Linux-64bit.deb downloading (44%) \r\n"] 86 | [11.50, "o", "\u001b[1A\u001b[J⠧ hugo_extended_0.73.0_Linux-64bit.deb downloading (50%) \r\n"] 87 | [11.75, "o", "\u001b[1A\u001b[J⠇ hugo_extended_0.73.0_Linux-64bit.deb downloading (56%) \r\n"] 88 | [12.00, "o", "\u001b[1A\u001b[J⠏ hugo_extended_0.73.0_Linux-64bit.deb downloading (59%) \r\n"] 89 | [12.25, "o", "\u001b[1A\u001b[J⠋ hugo_extended_0.73.0_Linux-64bit.deb downloading (63%) \r\n"] 90 | [12.50, "o", "\u001b[1A\u001b[J⠙ hugo_extended_0.73.0_Linux-64bit.deb downloading (68%) \r\n"] 91 | [12.75, "o", "\u001b[1A\u001b[J⠹ hugo_extended_0.73.0_Linux-64bit.deb downloading (72%) \r\n"] 92 | [13.00, "o", "\u001b[1A\u001b[J⠸ hugo_extended_0.73.0_Linux-64bit.deb downloading (76%) \r\n"] 93 | [13.25, "o", "\u001b[1A\u001b[J⠼ hugo_extended_0.73.0_Linux-64bit.deb downloading (80%) \r\n"] 94 | [13.50, "o", "\u001b[1A\u001b[J⠴ hugo_extended_0.73.0_Linux-64bit.deb downloading (84%) \r\n"] 95 | [13.75, "o", "\u001b[1A\u001b[J⠦ hugo_extended_0.73.0_Linux-64bit.deb downloading (87%) \r\n"] 96 | [14.00, "o", "\u001b[1A\u001b[J⠧ hugo_extended_0.73.0_Linux-64bit.deb downloading (90%) \r\n"] 97 | [14.25, "o", "\u001b[1A\u001b[J⠇ hugo_extended_0.73.0_Linux-64bit.deb downloading (94%) \r\n"] 98 | [14.50, "o", "\u001b[1A\u001b[J⠏ hugo_extended_0.73.0_Linux-64bit.deb downloading (98%) \r\n"] 99 | [14.75, "o", "\u001b[1A\u001b[J⠋ hugo_extended_0.73.0_Linux-64bit.deb downloading (98%) \r\n"] 100 | [15.00, "o", "\u001b[1A\u001b[J⠙ hugo_extended_0.73.0_Linux-64bit.deb downloading (100%) \r\n"] 101 | [15.25, "o", "\u001b[1A\u001b[J⠹ hugo_extended_0.73.0_Linux-64bit.deb downloaded \r\n\u001b[1A\u001b[J⠸ hugo_extended_0.73.0_Linux-64bit.deb verifying (sha256) \r\n"] 102 | [15.50, "o", "\u001b[1A\u001b[J⠼ hugo_extended_0.73.0_Linux-64bit.deb verifying (sha256) \r\n"] 103 | [15.75, "o", "\u001b[1A\u001b[J⠴ hugo_extended_0.73.0_Linux-64bit.deb verifying (sha256) \r\n"] 104 | [16.00, "o", "\u001b[1A\u001b[J⠦ hugo_extended_0.73.0_Linux-64bit.deb finishing \r\n"] 105 | [16.25, "o", "\u001b[1A\u001b[J⠧ hugo_extended_0.73.0_Linux-64bit.deb finishing \r\n"] 106 | [16.50, "o", "\u001b[1A\u001b[J⠇ hugo_extended_0.73.0_Linux-64bit.deb finishing \r\n\u001b[1A\u001b[J√ hugo_extended_0.73.0_Linux-64bit.deb done (sha256 OK) \r\n\u001b[1A\u001b[J√ hugo_extended_0.73.0_Linux-64bit.deb done (sha256 OK) \r\n\u001b[1A\u001b[J√ hugo_extended_0.73.0_Linux-64bit.deb done (sha256 OK) \r\n$ "] 107 | -------------------------------------------------------------------------------- /website/assets/css/tailwind.css: -------------------------------------------------------------------------------- 1 | @import 'tailwindcss/base'; 2 | @import 'tailwindcss/components'; 3 | @import 'tailwindcss/utilities'; 4 | -------------------------------------------------------------------------------- /website/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "paths": { 5 | "~/*": ["./*"], 6 | "@/*": ["./*"], 7 | "~~/*": ["./*"], 8 | "@@/*": ["./*"] 9 | } 10 | }, 11 | "exclude": ["node_modules", ".nuxt", "dist"] 12 | } 13 | -------------------------------------------------------------------------------- /website/layouts/default.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | -------------------------------------------------------------------------------- /website/nuxt.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | mode: 'universal', 3 | target: 'server', 4 | head: { 5 | title: 'gget', 6 | meta: [ 7 | { charset: 'utf-8' }, 8 | { name: 'viewport', content: 'width=device-width, initial-scale=1' }, 9 | ] 10 | }, 11 | loading: { 12 | color: '#48BB78', 13 | height: '3px' 14 | }, 15 | buildModules: [ 16 | '@nuxtjs/tailwindcss', 17 | '@nuxtjs/fontawesome', 18 | [ 19 | '@nuxtjs/google-analytics', 20 | { 21 | id: process.env.GA_TRACKING_ID 22 | } 23 | ] 24 | ], 25 | env: { 26 | baseUrl: process.env.BASE_URL || 'http://127.0.0.1:3000/' 27 | }, 28 | fontawesome: { 29 | component: "fa", 30 | icons: { 31 | solid: ['faArrowAltCircleDown', 'faBeer', 'faCheckDouble', 'faChevronDown', 'faCode', 'faCodeBranch', 'faCopy', 'faExchangeAlt', 'faFileDownload', 'faGraduationCap', 'faHeart', 'faHome', 'faLifeRing', 'faLock', 'faPlay', 'faPlayCircle', 'faRandom', 'faServer', 'faSignature', 'faTerminal', 'faTools', 'faUserAlt'], 32 | brands: ['faApple', 'faDocker', 'faGitAlt', 'faGithub', 'faLinux', 'faTwitter', 'faWindows'], 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gget", 3 | "version": "1.0.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "nuxt", 7 | "build": "nuxt build", 8 | "start": "nuxt start", 9 | "export": "nuxt export", 10 | "serve": "nuxt serve" 11 | }, 12 | "dependencies": { 13 | "nuxt": "^2" 14 | }, 15 | "devDependencies": { 16 | "@fortawesome/free-brands-svg-icons": "^5", 17 | "@fortawesome/free-solid-svg-icons": "^5", 18 | "@nuxtjs/fontawesome": "^1", 19 | "@nuxtjs/google-analytics": "^2", 20 | "@nuxtjs/tailwindcss": "^2", 21 | "axios": "^0" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /website/pages/index.vue: -------------------------------------------------------------------------------- 1 | 447 | 448 | 588 | 589 | 643 | 644 | 645 | -------------------------------------------------------------------------------- /website/plugins/axios.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | 3 | export default axios.create({ 4 | baseURL: process.env.baseUrl 5 | }) 6 | -------------------------------------------------------------------------------- /website/static/img/card~v1~1280x640.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dpb587/gget/10b18ba1a7674a67b84b21e00199d85c234ae708/website/static/img/card~v1~1280x640.png -------------------------------------------------------------------------------- /website/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /* 2 | ** TailwindCSS Configuration File 3 | ** 4 | ** Docs: https://tailwindcss.com/docs/configuration 5 | ** Default: https://github.com/tailwindcss/tailwindcss/blob/master/stubs/defaultConfig.stub.js 6 | */ 7 | module.exports = { 8 | theme: {}, 9 | variants: {}, 10 | plugins: [], 11 | purge: { 12 | // Learn more on https://tailwindcss.com/docs/controlling-file-size/#removing-unused-css 13 | enabled: process.env.NODE_ENV === 'production', 14 | content: [ 15 | 'components/**/*.vue', 16 | 'layouts/**/*.vue', 17 | 'pages/**/*.vue', 18 | 'plugins/**/*.js', 19 | 'nuxt.config.js' 20 | ] 21 | } 22 | } 23 | --------------------------------------------------------------------------------