├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── api ├── api_app.go ├── api_http_func.go └── api_structs.go ├── app.go ├── assets ├── assets_app.go ├── assets_http_func.go └── bindata.go ├── bolt.go ├── cli └── main.go ├── commands └── server.go ├── config.go ├── config_test.go ├── docs ├── bower.md ├── composer.md ├── git.md ├── installation.md ├── maintenance.md ├── migration.md ├── npm.md ├── pkgmirror.gif ├── static.md └── usage.md ├── errors.go ├── fixtures ├── git │ └── foo.bare │ │ ├── HEAD │ │ ├── config │ │ ├── objects │ │ ├── 96 │ │ │ └── 01539ddd2dbf994af80fdc78e5df50491985d7 │ │ ├── 2d │ │ │ └── a62f8886014fb045e41b659e4fa83db5ef24d2 │ │ ├── 9b │ │ │ └── 9cc9573693611badb397b5d01a1e6645704da7 │ │ └── cb │ │ │ └── 734ed1fceda100dbe1b259f039b061e8ea2f90 │ │ └── refs │ │ ├── heads │ │ └── master │ │ └── tags │ │ └── 0.0.1 ├── mock │ ├── bower │ │ └── packages │ ├── composer │ │ ├── p │ │ │ ├── 0n3s3c │ │ │ │ └── baselibrary$f00e050833c8ac3323177dc59eb69680fc0920c1e881835a1e64adcb60ea5157.json │ │ │ ├── provider-mock$9bd35df8f2fab78bd7e7469572b7d8e5ba58d11b702232db6ce9b6f1b0bf0fe5.json │ │ │ └── symfony │ │ │ │ └── framework-standard-edition$c08c23d91489ab0c8098739084a5f2873c331cf374268af722c5867038f9c0be.json │ │ └── packages.json │ ├── invalid.json │ ├── npm │ │ ├── @types │ │ │ └── react │ │ │ │ ├── - │ │ │ │ └── react-0.0.0.tgz │ │ │ │ └── index.html │ │ ├── angular-nvd3-nb │ │ │ ├── - │ │ │ │ └── angular-nvd3-nb-1.0.5-nb.tgz │ │ │ ├── README.md │ │ │ └── index.html │ │ └── angular-oauth │ └── static │ │ └── file.txt ├── npm │ ├── gulp-app-manager.json │ ├── jsontocsv.json │ ├── knwl.js.json │ ├── math_example_bulbignz.json │ ├── qs.json │ └── repeat.json ├── packagist │ ├── p │ │ ├── 0n3s3c │ │ │ └── baselibrary$3a3dbbc33805b6748f859e8f2c517355f42e2f6d4b71daad077794842dca280c.json │ │ ├── provider-2013$370af0b17d1ec5b0325bdb0126c9007b69647fafe5df8b5ecf79241e09745841.json │ │ └── symfony │ │ │ └── framework-standard-edition$cb64bc5278d2b6bbf7c02ae4b995f3698df1a210dceb509328b4370e13f3ba33.json │ └── packages.json └── project │ ├── Makefile │ ├── composer.json │ └── package.json ├── glide.lock ├── glide.yaml ├── gui ├── .babelrc ├── .prettierrc ├── package.json ├── src │ ├── Main.js │ ├── app.js │ ├── components │ │ ├── About.js │ │ ├── CardMirror.js │ │ ├── MenuList.js │ │ ├── MirrorList.js │ │ └── index.js │ ├── redux │ │ ├── apps │ │ │ ├── guiApp.js │ │ │ └── mirrorApp.js │ │ └── containers │ │ │ ├── CardMirror.js │ │ │ ├── MenuList.js │ │ │ ├── MirrorList.js │ │ │ └── index.js │ └── static │ │ ├── index.html │ │ └── main.css └── yarn.lock ├── mirror ├── bower │ ├── bower.go │ ├── bower_app.go │ └── bower_struct.go ├── composer │ ├── composer.go │ ├── composer_app.go │ ├── composer_pat.go │ ├── composer_pat_test.go │ ├── composer_structs.go │ ├── composer_structs_test.go │ └── composer_test.go ├── git │ ├── git.go │ ├── git_app.go │ ├── git_pat.go │ ├── git_pat_test.go │ └── git_test.go ├── npm │ ├── npm.go │ ├── npm_app.go │ ├── npm_pat.go │ ├── npm_pat_test.go │ ├── npm_structs.go │ └── npm_structs_test.go └── static │ ├── static.go │ ├── static_app.go │ └── static_structs.go ├── pkgmirror.toml ├── service.go ├── sse_broker.go ├── test ├── api │ └── api_test.go ├── mirror │ ├── bower_test.go │ ├── composer_test.go │ ├── git_test.go │ ├── npm_test.go │ └── static_test.go ├── test.go └── tools │ └── invalid_struct_test.go ├── tools.go └── tools_test.go /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig helps developers define and maintain consistent 2 | # coding styles between different editors and IDEs 3 | # editorconfig.org 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | # We recommend you to keep these unchanged 9 | end_of_line = lf 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | insert_final_newline = true 13 | 14 | [*.go] 15 | indent_style = tab 16 | indent_size = 4 17 | 18 | [*.js] 19 | indent_style = space 20 | indent_size = 4 21 | 22 | [*.jsx] 23 | indent_style = space 24 | indent_size = 4 25 | 26 | [*.styl] 27 | indent_style = space 28 | indent_size = 2 29 | 30 | [*.json] 31 | insert_final_newline = false 32 | 33 | [*.md] 34 | trim_trailing_whitespace = false 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | fixtures/project/vendor 3 | fixtures/project/composer.lock 4 | fixtures/project/node_modules 5 | gui/node_modules 6 | build 7 | assets/bindata.go 8 | vendor 9 | var/ 10 | gui/.cache 11 | gui/dist -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: [1.9] 3 | sudo: false 4 | cache: 5 | directories: 6 | - gui/node_modules 7 | - vendor 8 | 9 | install: 10 | # from http://austinpray.com/ops/2015/09/20/change-travis-node-version.html 11 | - make install-backend 12 | - rm -rf ~/.nvm && git clone https://github.com/creationix/nvm.git ~/.nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source ~/.nvm/nvm.sh && nvm install 8.12.0 13 | - npm -g install yarn 14 | - make install-frontend 15 | 16 | script: 17 | - go get github.com/mattn/goveralls 18 | - make coverage-backend 19 | - goveralls -coverprofile=build/pkgmirror.coverage -service=travis-ci 20 | - make test-frontend 21 | - make build 22 | 23 | deploy: 24 | provider: releases 25 | api_key: 26 | secure: "br0UDyNOVuLwC8ZgYrKeOd4cWR1o9LlEoxhXQ8LhSCuWktf7lZy7mX7IZctzAeH26Fed9oVObCMHdwvkQw8Tg4m1eQ1UNKtrEODiEvKdMD4H4N3eOX9/k1xThaheTxkdv55U4ejwSHADkmftU03kLPoKMRgPB28G2ZJqruvO2DjuqJDG2co4yTRXY2G8n7g5vQn5BlIJoVcnGg9MCPt41wAEPvbX2tnLGvCMuHdGeKGe6cWyMbhuNkLXjsKfqdZBncFXloLxLdkBlnYBOdWOgpLsXdlQUCW4wDKmUxCiKK8rELQvB3vO87WjnO0k9UpBpjdBAj0KJYv0SOfmuMML1BmkQyV+MNR8V2/yFpX99u0q19mwVhaopSfwllABCa8zQmfnpmDHle8Qzz9b91kqosuwYMTBzuCKZW8zlMv+JNDrXVpwe0NSh4qrCj6klW+hFyGm2uegwnfADUXOhVnKUT45MX7u4nG3/netT2+mQ3GUM8XmsM6Nv3RvR7yx0S0i2SXxHwlSkUXurjVmgXs3lkC+AtC8h9oXxYKVqK2HEjw4u0ChkexbQ/EHDzk6ovRfcGu16rksIw+smHLcY/DcFG5f/dNcFntckZa97IUuAkuHchXT9Ogv3+0qFAqhZiyko2IY41qJTeIhmGVB9svEPsTcSFzj2jHPuNiycU1UMd0=" 27 | file: 28 | - build/darwin-amd64-pkgmirror 29 | - build/linux-amd64-pkgmirror 30 | - build/linux-386-pkgmirror 31 | - build/linux-arm-pkgmirror 32 | - build/linux-arm64-pkgmirror 33 | skip_cleanup: true 34 | overwrite: true 35 | on: 36 | tags: true 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Thomas Rabaix 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished 8 | to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help format test install update build release assets 2 | 3 | GO_BINDATA_PREFIX = $(shell pwd)/gui/dist 4 | GO_BINDATA_PATHS = $(shell pwd)/gui/dist 5 | GO_BINDATA_IGNORE = "(.*)\.(go|DS_Store)" 6 | GO_BINDATA_OUTPUT = $(shell pwd)/assets/bindata.go 7 | GO_BINDATA_PACKAGE = assets 8 | GO_PROJECTS_PATHS = ./ ./test ./test/mirror ./test/tools ./test/api ./api ./assets ./cli ./mirror/composer ./mirror/git ./mirror/npm ./mirror/bower ./mirror/static ./commands 9 | GO_PKG = ./,./mirror/composer,./mirror/git,./mirror/npm,./mirror/bower,./mirror/static,./api 10 | GO_FILES = $(shell find $(GO_PROJECTS_PATHS) -maxdepth 1 -type f -name "*.go") 11 | JS_FILES = $(shell find ./gui/src -type f -name "*.js") 12 | 13 | SHA1=$(shell git rev-parse HEAD) 14 | 15 | help: ## Display this help 16 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 17 | 18 | 19 | format: format-frontend format-backend 20 | echo "Done!" 21 | 22 | format-frontend: ## Format code to respect CS 23 | cd gui && yarn prettier --single-quote --trailing-comma es5 --write "src/**/*.js" 24 | 25 | format-backend: ## Format code to respect CS 26 | `go env GOPATH`/bin/goimports -w $(GO_FILES) 27 | gofmt -l -w -s $(GO_FILES) 28 | go vet $(GO_PROJECTS_PATHS) 29 | 30 | coverage-backend: ## run coverage tests 31 | mkdir -p build/coverage 32 | rm -rf build/coverage/*.cov 33 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/main.cov ./ 34 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/composer.cov ./mirror/composer 35 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/git.cov ./mirror/git 36 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/npm.cov ./mirror/npm 37 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/bower.cov ./mirror/bower 38 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/static.cov ./mirror/static 39 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/functional_mirror.cov ./test/mirror 40 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/functional_api.cov ./test/api 41 | go test -v -timeout 60s -coverpkg $(GO_PKG) -covermode count -coverprofile=build/coverage/functional_tools.cov ./test/tools 42 | gocovmerge build/coverage/* > build/pkgmirror.coverage 43 | go tool cover -html=./build/pkgmirror.coverage -o build/pkgmirror.html 44 | 45 | test-backend: ## Run backend tests 46 | go test -v -race -timeout 60s $(GO_PROJECTS_PATHS) 47 | go vet $(GO_PROJECTS_PATHS) 48 | 49 | test-frontend: ## Run frontend tests 50 | exit 0 51 | 52 | test: test-backend 53 | 54 | run: bin-dev ## Run server 55 | go run -race cli/main.go run -file ./pkgmirror.toml -log-level=info 56 | 57 | install: install-backend install-frontend 58 | 59 | install-backend: ## Install backend dependencies 60 | go get github.com/wadey/gocovmerge 61 | go get golang.org/x/tools/cmd/cover 62 | go get github.com/aktau/github-release 63 | go get golang.org/x/tools/cmd/goimports 64 | go get -u github.com/jteeuwen/go-bindata/... 65 | go get github.com/Masterminds/glide 66 | glide install 67 | 68 | install-frontend: ## Install frontend dependencies 69 | cd gui && yarn --no-progress 70 | 71 | update: ## Update dependencies 72 | glide update 73 | 74 | bin-dev: ## Generate bin dev assets file 75 | `go env GOPATH`/bin/go-bindata -dev -o $(GO_BINDATA_OUTPUT) -prefix $(GO_BINDATA_PREFIX) -pkg $(GO_BINDATA_PACKAGE) -ignore $(GO_BINDATA_IGNORE) $(GO_BINDATA_PATHS) 76 | 77 | bin: assets ## Generate bin assets file 78 | `go env GOPATH`/bin/go-bindata -o $(GO_BINDATA_OUTPUT) -prefix $(GO_BINDATA_PREFIX) -pkg $(GO_BINDATA_PACKAGE) -ignore $(GO_BINDATA_IGNORE) $(GO_BINDATA_PATHS) 79 | 80 | assets: ## build assets 81 | rm -rf gui/dist/* 82 | cd gui && yarn parcel build src/static/index.html --no-source-maps 83 | 84 | watch: ## build assets 85 | rm -rf gui/build/* 86 | cd gui && yarn parcel watch src/static/index.html 87 | 88 | build: bin ## build binaries 89 | GOOS=darwin GOARCH=amd64 go build -ldflags "-X main.RefLog=$(SHA1) -X main.Version=$(TRAVIS_TAG) -s -w" -o build/darwin-amd64-pkgmirror cli/main.go 90 | GOOS=linux GOARCH=amd64 go build -ldflags "-X main.RefLog=$(SHA1) -X main.Version=$(TRAVIS_TAG) -s -w" -o build/linux-amd64-pkgmirror cli/main.go 91 | GOOS=linux GOARCH=386 go build -ldflags "-X main.RefLog=$(SHA1) -X main.Version=$(TRAVIS_TAG) -s -w" -o build/linux-386-pkgmirror cli/main.go 92 | GOOS=linux GOARCH=arm go build -ldflags "-X main.RefLog=$(SHA1) -X main.Version=$(TRAVIS_TAG) -s -w" -o build/linux-arm-pkgmirror cli/main.go 93 | GOOS=linux GOARCH=arm64 go build -ldflags "-X main.RefLog=$(SHA1) -X main.Version=$(TRAVIS_TAG) -s -w" -o build/linux-arm64-pkgmirror cli/main.go -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pkg Mirrors 2 | =========== 3 | 4 | [![Build Status](https://travis-ci.org/rande/pkgmirror.svg?branch=master)](https://travis-ci.org/rande/pkgmirror) 5 | [![Coverage Status](https://coveralls.io/repos/github/rande/pkgmirror/badge.svg)](https://coveralls.io/github/rande/pkgmirror) 6 | 7 | This project aims to provides mirroring features for: 8 | - composer: a php package manager. 9 | - git: clone mirror and sync repo repository. 10 | - npm: a NodeJS package manager. 11 | - bower: a front-end package manager. 12 | 13 | *Please note*, if you only need to proxy packagist, please consider [Toran Proxy](https://toranproxy.com/) 14 | as a strong alternative. 15 | 16 | 17 | Documentations 18 | -------------- 19 | 20 | ### User guide 21 | 22 | * [Installation](docs/installation.md) 23 | * [Maintenance](docs/maintenance.md) 24 | * [Usage](docs/usage.md) 25 | 26 | ### Internals 27 | 28 | * [Composer](docs/composer.md) 29 | * [Git](docs/git.md) 30 | * [Npm](docs/npm.md) 31 | * [Bower](docs/bower.md) 32 | * [Static](docs/static.md) 33 | 34 | GUI 35 | --- 36 | 37 | ![alt text](docs/pkgmirror.gif "GUI PkgMirror") 38 | -------------------------------------------------------------------------------- /api/api_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package api 7 | 8 | import ( 9 | "encoding/json" 10 | "sync" 11 | 12 | log "github.com/Sirupsen/logrus" 13 | "github.com/rande/goapp" 14 | "github.com/rande/pkgmirror" 15 | "goji.io" 16 | "goji.io/pat" 17 | ) 18 | 19 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 20 | 21 | l.Register(func(app *goapp.App) error { 22 | app.Set("pkgmirror.channel.state", func(app *goapp.App) interface{} { 23 | return make(chan pkgmirror.State) 24 | }) 25 | 26 | app.Set("pkgmirror.sse.broker", func(app *goapp.App) interface{} { 27 | return pkgmirror.NewSseBroker() 28 | }) 29 | 30 | return nil 31 | }) 32 | 33 | l.Prepare(func(app *goapp.App) error { 34 | mux := app.Get("mux").(*goji.Mux) 35 | mux.HandleFuncC(pat.Get("/api/mirrors"), Api_GET_MirrorServices(app)) 36 | mux.HandleFunc(pat.Get("/api/sse"), Api_GET_Sse(app)) 37 | mux.HandleFuncC(pat.Get("/api/ping"), Api_GET_Ping(app)) 38 | 39 | return nil 40 | }) 41 | 42 | l.Run(func(app *goapp.App, state *goapp.GoroutineState) error { 43 | ch := app.Get("pkgmirror.channel.state").(chan pkgmirror.State) 44 | brk := app.Get("pkgmirror.sse.broker").(*pkgmirror.SseBroker) 45 | logger := app.Get("logger").(*log.Logger) 46 | 47 | logger.Info("Start the SSE Broker") 48 | // start the broken 49 | go brk.Listen() 50 | 51 | states := map[string]pkgmirror.State{} 52 | 53 | l := sync.Mutex{} 54 | 55 | // send the current state 56 | brk.OnConnect(func() { 57 | l.Lock() 58 | for _, s := range states { 59 | data, _ := json.Marshal(&s) 60 | 61 | brk.Notifier <- data 62 | } 63 | l.Unlock() 64 | }) 65 | 66 | for { 67 | select { 68 | case s := <-ch: 69 | logger.WithFields(log.Fields{ 70 | "id": s.Id, 71 | "message": s.Message, 72 | "status": s.Status, 73 | }).Debug("Receive message") 74 | 75 | l.Lock() 76 | states[s.Id] = s 77 | l.Unlock() 78 | 79 | data, _ := json.Marshal(&s) 80 | 81 | brk.Notifier <- data 82 | case <-state.In: 83 | return nil // exit 84 | } 85 | } 86 | }) 87 | 88 | } 89 | -------------------------------------------------------------------------------- /api/api_http_func.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package api 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | 12 | "github.com/rande/goapp" 13 | "github.com/rande/pkgmirror" 14 | "golang.org/x/net/context" 15 | ) 16 | 17 | func Api_GET_MirrorServices(app *goapp.App) func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 18 | config := app.Get("config").(*pkgmirror.Config) 19 | 20 | return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 21 | w.Header().Set("Content-Type", "application/json") 22 | 23 | d := []*ServiceMirror{} 24 | 25 | for code, conf := range config.Git { 26 | s := &ServiceMirror{} 27 | s.Id = fmt.Sprintf("pkgmirror.git.%s", code) 28 | s.Icon = conf.Icon 29 | s.Type = "git" 30 | s.Name = code 31 | s.SourceUrl = conf.Server 32 | s.TargetUrl = fmt.Sprintf("%s/git/%s", config.PublicServer, conf.Server) 33 | s.Enabled = conf.Enabled 34 | s.Usage = fmt.Sprintf(` 35 | You can also download a zip file with the following url: 36 | 37 | %s/path/repository/REFENCE.zip 38 | 39 | The reference can be anything: a branch, a tag or a commit. Please note, tag and commit are 40 | stored on dedicated cache location. 41 | 42 | You can clone repository with the following command: 43 | 44 | git clone %s/path/repository.git 45 | 46 | `, s.TargetUrl, s.TargetUrl) 47 | 48 | d = append(d, s) 49 | } 50 | 51 | for code, conf := range config.Npm { 52 | s := &ServiceMirror{} 53 | s.Id = fmt.Sprintf("pkgmirror.npm.%s", code) 54 | s.Icon = conf.Icon 55 | s.Type = "npm" 56 | s.Name = code 57 | s.SourceUrl = conf.Server 58 | s.TargetUrl = fmt.Sprintf("%s/npm/%s", config.PublicServer, code) 59 | s.Enabled = conf.Enabled 60 | s.Usage = fmt.Sprintf(` 61 | You need to set the registry to: 62 | 63 | npm set registry %s 64 | 65 | That's it! Now any packages will be retrieve from the mirror. Only downloaded archive files will 66 | be stored on a dedicated cache location. 67 | 68 | Please note, the configuration is global to all projects running in the current environment. 69 | 70 | `, s.TargetUrl) 71 | d = append(d, s) 72 | } 73 | 74 | for code, conf := range config.Composer { 75 | s := &ServiceMirror{} 76 | s.Id = fmt.Sprintf("pkgmirror.composer.%s", code) 77 | s.Icon = conf.Icon 78 | s.Type = "composer" 79 | s.Name = code 80 | s.SourceUrl = conf.Server 81 | s.TargetUrl = fmt.Sprintf("%s/composer/%s", config.PublicServer, code) 82 | s.Enabled = conf.Enabled 83 | s.Usage = fmt.Sprintf(` 84 | You need to declare the mirror in your composer.json file: 85 | 86 | "repositories":[ 87 | { "packagist": false }, 88 | { "type": "composer", "url": "%s"} 89 | ], 90 | 91 | That's it! 92 | 93 | Please note, the composer mirror alter github path to point to the local git mirror. Make sure 94 | the github mirror is properly configured. 95 | `, s.TargetUrl) 96 | 97 | d = append(d, s) 98 | } 99 | 100 | for code, conf := range config.Bower { 101 | s := &ServiceMirror{} 102 | s.Id = fmt.Sprintf("pkgmirror.bower.%s", code) 103 | s.Icon = conf.Icon 104 | s.Type = "bower" 105 | s.Name = code 106 | s.SourceUrl = conf.Server 107 | s.TargetUrl = fmt.Sprintf("%s/bower/%s", config.PublicServer, code) 108 | s.Enabled = conf.Enabled 109 | s.Usage = fmt.Sprintf(` 110 | You need to declare the mirror in your .bowerrc file: 111 | 112 | { 113 | "registry": { 114 | "search": ["%s"], 115 | "register": "%s" 116 | } 117 | } 118 | 119 | That's it! 120 | 121 | Please note, the bower mirror alter github path to point to the local git mirror. Make sure 122 | the github mirror is properly configured. 123 | `, s.TargetUrl, s.TargetUrl) 124 | 125 | d = append(d, s) 126 | } 127 | 128 | for code, conf := range config.Static { 129 | s := &ServiceMirror{} 130 | s.Id = fmt.Sprintf("pkgmirror.static.%s", code) 131 | s.Icon = conf.Icon 132 | s.Type = "static" 133 | s.Name = code 134 | s.SourceUrl = conf.Server 135 | s.TargetUrl = fmt.Sprintf("%s/static/%s", config.PublicServer, code) 136 | s.Enabled = conf.Enabled 137 | s.Usage = fmt.Sprintf(` 138 | You just need to reference the server as %s/myfile.zip, the static handle will retrieve the file 139 | from %s/myfile.zip and store a copy on the mirror server. 140 | `, s.TargetUrl, s.SourceUrl) 141 | 142 | d = append(d, s) 143 | } 144 | 145 | pkgmirror.Serialize(w, d) 146 | } 147 | } 148 | 149 | func Api_GET_Sse(app *goapp.App) func(w http.ResponseWriter, r *http.Request) { 150 | brk := app.Get("pkgmirror.sse.broker").(*pkgmirror.SseBroker) 151 | 152 | return brk.Handler 153 | } 154 | 155 | func Api_GET_Ping(app *goapp.App) func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 156 | return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 157 | w.Write([]byte("pong")) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /api/api_structs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package api 7 | 8 | type ServiceMirror struct { 9 | Id string 10 | Type string 11 | Name string 12 | SourceUrl string 13 | TargetUrl string 14 | Icon string 15 | Enabled bool 16 | Usage string 17 | } 18 | -------------------------------------------------------------------------------- /app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | import ( 9 | "errors" 10 | "fmt" 11 | "net/http" 12 | 13 | "github.com/NYTimes/gziphandler" 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/bakins/logrus-middleware" 16 | "github.com/rande/goapp" 17 | "goji.io" 18 | ) 19 | 20 | func GetApp(conf *Config, l *goapp.Lifecycle) (*goapp.App, error) { 21 | 22 | app := goapp.NewApp() 23 | 24 | // init logger 25 | logger := log.New() 26 | if level, err := log.ParseLevel(conf.LogLevel); err != nil { 27 | return app, errors.New(fmt.Sprintf("Unable to parse the log level: %s", conf.LogLevel)) 28 | } else { 29 | logger.Level = level 30 | } 31 | 32 | if len(conf.DataDir) == 0 { 33 | return app, errors.New("Please configure DataDir") 34 | } 35 | 36 | app.Set("logger", func(app *goapp.App) interface{} { 37 | return logger 38 | }) 39 | 40 | app.Set("config", func(app *goapp.App) interface{} { 41 | return conf 42 | }) 43 | 44 | app.Set("bolt.compacter", func(app *goapp.App) interface{} { 45 | logger := app.Get("logger").(*log.Logger) 46 | 47 | return &BoltCompacter{ 48 | Logger: logger, 49 | TxMaxSize: 65536, 50 | } 51 | }) 52 | 53 | app.Set("mux", func(app *goapp.App) interface{} { 54 | m := goji.NewMux() 55 | 56 | m.Use(func(h http.Handler) http.Handler { 57 | gzip := gziphandler.GzipHandler(h) 58 | 59 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 | skip := false 61 | if len(r.URL.Path) > 4 && r.URL.Path[1:4] == "npm" { 62 | skip = true 63 | } 64 | 65 | if len(r.URL.Path) > 8 && r.URL.Path[1:9] == "composer" { 66 | skip = true 67 | } 68 | 69 | if r.URL.Path == "/api/sse" { 70 | skip = true 71 | } 72 | 73 | if skip { 74 | h.ServeHTTP(w, r) 75 | } else { 76 | gzip.ServeHTTP(w, r) 77 | } 78 | }) 79 | }) 80 | 81 | m.Use(func(h http.Handler) http.Handler { 82 | lm := &logrusmiddleware.Middleware{ 83 | Logger: logger, 84 | Name: "pkgmirror", 85 | } 86 | 87 | return lm.Handler(h, "http") 88 | }) 89 | 90 | return m 91 | }) 92 | 93 | return app, nil 94 | } 95 | -------------------------------------------------------------------------------- /assets/assets_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package assets 7 | 8 | import ( 9 | "github.com/rande/goapp" 10 | "github.com/rande/pkgmirror" 11 | "goji.io" 12 | "goji.io/pat" 13 | ) 14 | 15 | // required by go-bindata 16 | var rootDir = "./gui/dist" 17 | 18 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 19 | l.Prepare(func(app *goapp.App) error { 20 | mux := app.Get("mux").(*goji.Mux) 21 | mux.HandleFuncC(pat.Get("/*"), Assets_GET_File(app)) 22 | 23 | return nil 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /assets/assets_http_func.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package assets 7 | 8 | import ( 9 | "net/http" 10 | "strings" 11 | 12 | "fmt" 13 | 14 | "github.com/rande/goapp" 15 | "golang.org/x/net/context" 16 | ) 17 | 18 | var contentTypes = map[string]string{ 19 | "js": "application/javascript", 20 | "css": "text/css", 21 | "svg": "image/svg+xml", 22 | "eot": "application/vnd.ms-fontobject", 23 | "woff": "application/x-font-woff", 24 | "woff2": "application/font-woff2", 25 | "ttf": "application/x-font-ttf", 26 | "png": "image/png", 27 | "jpg": "image/jpg", 28 | "gif": "image/gif", 29 | } 30 | 31 | func Assets_GET_File(app *goapp.App) func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 32 | return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 33 | path := "" 34 | if r.URL.Path == "/" { 35 | path = "index.html" 36 | } else { 37 | path = r.URL.Path[1:] 38 | } 39 | 40 | if asset, err := Asset(path); err != nil { 41 | w.WriteHeader(404) 42 | w.Write([]byte(fmt.Sprintf("Page not found

Page not found

Page: %s
", path))) 43 | } else { 44 | ext := path[(strings.LastIndex(path, ".") + 1):] 45 | 46 | if _, ok := contentTypes[ext]; ok { 47 | w.Header().Set("Content-Type", contentTypes[ext]) 48 | } 49 | 50 | w.Write(asset) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /assets/bindata.go: -------------------------------------------------------------------------------- 1 | // Code generated by go-bindata. 2 | // sources: 3 | // gui/build/app.js 4 | // gui/build/index.html 5 | // gui/build/main.css 6 | // DO NOT EDIT! 7 | 8 | package assets 9 | 10 | import ( 11 | "fmt" 12 | "io/ioutil" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | ) 17 | 18 | // bindataRead reads the given file from disk. It returns an error on failure. 19 | func bindataRead(path, name string) ([]byte, error) { 20 | buf, err := ioutil.ReadFile(path) 21 | if err != nil { 22 | err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err) 23 | } 24 | return buf, err 25 | } 26 | 27 | type asset struct { 28 | bytes []byte 29 | info os.FileInfo 30 | } 31 | 32 | // Asset loads and returns the asset for the given name. 33 | // It returns an error if the asset could not be found or 34 | // could not be loaded. 35 | func Asset(name string) ([]byte, error) { 36 | cannonicalName := strings.Replace(name, "\\", "/", -1) 37 | if f, ok := _bindata[cannonicalName]; ok { 38 | a, err := f() 39 | if err != nil { 40 | return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) 41 | } 42 | return a.bytes, nil 43 | } 44 | return nil, fmt.Errorf("Asset %s not found", name) 45 | } 46 | 47 | // MustAsset is like Asset but panics when Asset would return an error. 48 | // It simplifies safe initialization of global variables. 49 | func MustAsset(name string) []byte { 50 | a, err := Asset(name) 51 | if err != nil { 52 | panic("asset: Asset(" + name + "): " + err.Error()) 53 | } 54 | 55 | return a 56 | } 57 | 58 | // AssetInfo loads and returns the asset info for the given name. 59 | // It returns an error if the asset could not be found or 60 | // could not be loaded. 61 | func AssetInfo(name string) (os.FileInfo, error) { 62 | cannonicalName := strings.Replace(name, "\\", "/", -1) 63 | if f, ok := _bindata[cannonicalName]; ok { 64 | a, err := f() 65 | if err != nil { 66 | return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) 67 | } 68 | return a.info, nil 69 | } 70 | return nil, fmt.Errorf("AssetInfo %s not found", name) 71 | } 72 | 73 | // AssetNames returns the names of the assets. 74 | func AssetNames() []string { 75 | names := make([]string, 0, len(_bindata)) 76 | for name := range _bindata { 77 | names = append(names, name) 78 | } 79 | return names 80 | } 81 | 82 | // _bindata is a table, holding each asset generator, mapped to its name. 83 | var _bindata = map[string]func() (*asset, error){} 84 | 85 | // AssetDir returns the file names below a certain 86 | // directory embedded in the file by go-bindata. 87 | // For example if you run go-bindata on data/... and data contains the 88 | // following hierarchy: 89 | // data/ 90 | // foo.txt 91 | // img/ 92 | // a.png 93 | // b.png 94 | // then AssetDir("data") would return []string{"foo.txt", "img"} 95 | // AssetDir("data/img") would return []string{"a.png", "b.png"} 96 | // AssetDir("foo.txt") and AssetDir("notexist") would return an error 97 | // AssetDir("") will return []string{"data"}. 98 | func AssetDir(name string) ([]string, error) { 99 | node := _bintree 100 | if len(name) != 0 { 101 | cannonicalName := strings.Replace(name, "\\", "/", -1) 102 | pathList := strings.Split(cannonicalName, "/") 103 | for _, p := range pathList { 104 | node = node.Children[p] 105 | if node == nil { 106 | return nil, fmt.Errorf("Asset %s not found", name) 107 | } 108 | } 109 | } 110 | if node.Func != nil { 111 | return nil, fmt.Errorf("Asset %s not found", name) 112 | } 113 | rv := make([]string, 0, len(node.Children)) 114 | for childName := range node.Children { 115 | rv = append(rv, childName) 116 | } 117 | return rv, nil 118 | } 119 | 120 | type bintree struct { 121 | Func func() (*asset, error) 122 | Children map[string]*bintree 123 | } 124 | 125 | var _bintree = &bintree{nil, map[string]*bintree{}} 126 | 127 | // RestoreAsset restores an asset under the given directory 128 | func RestoreAsset(dir, name string) error { 129 | data, err := Asset(name) 130 | if err != nil { 131 | return err 132 | } 133 | info, err := AssetInfo(name) 134 | if err != nil { 135 | return err 136 | } 137 | err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) 138 | if err != nil { 139 | return err 140 | } 141 | err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) 142 | if err != nil { 143 | return err 144 | } 145 | err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) 146 | if err != nil { 147 | return err 148 | } 149 | return nil 150 | } 151 | 152 | // RestoreAssets restores an asset under the given directory recursively 153 | func RestoreAssets(dir, name string) error { 154 | children, err := AssetDir(name) 155 | // File 156 | if err != nil { 157 | return RestoreAsset(dir, name) 158 | } 159 | // Dir 160 | for _, child := range children { 161 | err = RestoreAssets(dir, filepath.Join(name, child)) 162 | if err != nil { 163 | return err 164 | } 165 | } 166 | return nil 167 | } 168 | 169 | func _filePath(dir, name string) string { 170 | cannonicalName := strings.Replace(name, "\\", "/", -1) 171 | return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) 172 | } 173 | -------------------------------------------------------------------------------- /bolt.go: -------------------------------------------------------------------------------- 1 | package pkgmirror 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | log "github.com/Sirupsen/logrus" 10 | "github.com/boltdb/bolt" 11 | ) 12 | 13 | func OpenDatabaseWithBucket(basePath string, bucket []byte) (db *bolt.DB, err error) { 14 | if err = os.MkdirAll(basePath, 0755); err != nil { 15 | return 16 | } 17 | 18 | path := fmt.Sprintf("%s/%s.db", basePath, bucket) 19 | 20 | db, err = bolt.Open(path, 0600, &bolt.Options{ 21 | Timeout: 1 * time.Second, 22 | ReadOnly: false, 23 | }) 24 | 25 | if err != nil { 26 | return 27 | } 28 | 29 | err = db.Update(func(tx *bolt.Tx) error { 30 | _, err := tx.CreateBucketIfNotExists(bucket) 31 | 32 | return err 33 | }) 34 | 35 | return 36 | } 37 | 38 | // adapted from : https://github.com/boltdb/bolt/blob/master/cmd/bolt/main.go 39 | 40 | var ( 41 | 42 | // ErrPathRequired is returned when the path to a Bolt database is not specified. 43 | ErrPathRequired = errors.New("path required") 44 | 45 | // ErrFileNotFound is returned when a Bolt database does not exist. 46 | ErrFileNotFound = errors.New("file not found") 47 | 48 | ErrUnableToCloseDatabase = errors.New("Unable to close the database") 49 | ) 50 | 51 | type BoltCompacter struct { 52 | TxMaxSize int64 53 | Logger *log.Logger 54 | } 55 | 56 | // Run executes the command. 57 | func (bc *BoltCompacter) Compact(srcPath string) (err error) { 58 | now := time.Now() 59 | bckPath := fmt.Sprintf("%s.%d-%d-%d-%d.backup", srcPath, now.Year(), now.Month(), now.Day(), now.Unix()) 60 | dstPath := fmt.Sprintf("%s.%d-%d-%d-%d.compacted", srcPath, now.Year(), now.Month(), now.Day(), now.Unix()) 61 | 62 | logger := bc.Logger.WithFields(log.Fields{ 63 | "method": "compacter", 64 | "srcPath": srcPath, 65 | "dstPath": dstPath, 66 | }) 67 | 68 | // Require database paths. 69 | if srcPath == "" { 70 | logger.Error("srcPath is not defined") 71 | 72 | return ErrPathRequired 73 | } 74 | 75 | // Ensure source file exists. 76 | fi, err := os.Stat(srcPath) 77 | if os.IsNotExist(err) { 78 | logger.Error("srcPath does not exist") 79 | 80 | return ErrFileNotFound 81 | } else if err != nil { 82 | return err 83 | } 84 | initialSize := fi.Size() 85 | 86 | // Open source database. 87 | src, err := bolt.Open(srcPath, 0444, nil) 88 | if err != nil { 89 | logger.Error("unable to open the src database") 90 | return err 91 | } 92 | defer src.Close() 93 | 94 | // Open destination database. 95 | dst, err := bolt.Open(dstPath, fi.Mode(), nil) 96 | if err != nil { 97 | logger.Error("unable to open the dst database") 98 | return err 99 | } 100 | defer dst.Close() 101 | 102 | // Run compaction. 103 | if err := bc.compact(dst, src); err != nil { 104 | logger.Error("unable to compact the database") 105 | return err 106 | } 107 | 108 | // Report stats on new size. 109 | fi, err = os.Stat(dstPath) 110 | if err != nil { 111 | logger.Error("unable to get the stat on the compacted database") 112 | return err 113 | } else if fi.Size() == 0 { 114 | return fmt.Errorf("zero db size") 115 | } 116 | 117 | logger.WithFields(log.Fields{ 118 | "stat": fmt.Sprintf("%d -> %d bytes (gain=%.2fx)\n", initialSize, fi.Size(), float64(initialSize)/float64(fi.Size())), 119 | }).Info("Compact action ended!") 120 | 121 | // mv srcPath => bckPath 122 | if err := os.Rename(srcPath, bckPath); err != nil { 123 | return err 124 | } 125 | 126 | if err := os.Rename(dstPath, srcPath); err != nil { 127 | return err 128 | } 129 | 130 | // delete backup 131 | os.Remove(bckPath) 132 | 133 | return nil 134 | } 135 | 136 | func (bc *BoltCompacter) compact(dst, src *bolt.DB) error { 137 | // commit regularly, or we'll run out of memory for large datasets if using one transaction. 138 | var size int64 139 | tx, err := dst.Begin(true) 140 | if err != nil { 141 | return err 142 | } 143 | defer tx.Rollback() 144 | 145 | if err := bc.walk(src, func(keys [][]byte, k, v []byte, seq uint64) error { 146 | // On each key/value, check if we have exceeded tx size. 147 | sz := int64(len(k) + len(v)) 148 | if size+sz > bc.TxMaxSize && bc.TxMaxSize != 0 { 149 | // Commit previous transaction. 150 | if err := tx.Commit(); err != nil { 151 | return err 152 | } 153 | 154 | // Start new transaction. 155 | tx, err = dst.Begin(true) 156 | if err != nil { 157 | return err 158 | } 159 | size = 0 160 | } 161 | size += sz 162 | 163 | // Create bucket on the root transaction if this is the first level. 164 | nk := len(keys) 165 | if nk == 0 { 166 | bkt, err := tx.CreateBucket(k) 167 | if err != nil { 168 | return err 169 | } 170 | if err := bkt.SetSequence(seq); err != nil { 171 | return err 172 | } 173 | return nil 174 | } 175 | 176 | // Create buckets on subsequent levels, if necessary. 177 | b := tx.Bucket(keys[0]) 178 | if nk > 1 { 179 | for _, k := range keys[1:] { 180 | b = b.Bucket(k) 181 | } 182 | } 183 | 184 | // If there is no value then this is a bucket call. 185 | if v == nil { 186 | bkt, err := b.CreateBucket(k) 187 | if err != nil { 188 | return err 189 | } 190 | if err := bkt.SetSequence(seq); err != nil { 191 | return err 192 | } 193 | return nil 194 | } 195 | 196 | // Otherwise treat it as a key/value pair. 197 | return b.Put(k, v) 198 | }); err != nil { 199 | return err 200 | } 201 | 202 | return tx.Commit() 203 | } 204 | 205 | // walkFunc is the type of the function called for keys (buckets and "normal" 206 | // values) discovered by Walk. keys is the list of keys to descend to the bucket 207 | // owning the discovered key/value pair k/v. 208 | type walkFunc func(keys [][]byte, k, v []byte, seq uint64) error 209 | 210 | // walk walks recursively the bolt database db, calling walkFn for each key it finds. 211 | func (bc *BoltCompacter) walk(db *bolt.DB, walkFn walkFunc) error { 212 | return db.View(func(tx *bolt.Tx) error { 213 | return tx.ForEach(func(name []byte, b *bolt.Bucket) error { 214 | return bc.walkBucket(b, nil, name, nil, b.Sequence(), walkFn) 215 | }) 216 | }) 217 | } 218 | 219 | func (bc *BoltCompacter) walkBucket(b *bolt.Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error { 220 | // Execute callback. 221 | if err := fn(keypath, k, v, seq); err != nil { 222 | return err 223 | } 224 | 225 | // If this is not a bucket then stop. 226 | if v != nil { 227 | return nil 228 | } 229 | 230 | // Iterate over each child key/value. 231 | keypath = append(keypath, k) 232 | return b.ForEach(func(k, v []byte) error { 233 | if v == nil { 234 | bkt := b.Bucket(k) 235 | return bc.walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn) 236 | } 237 | return bc.walkBucket(b, keypath, k, v, b.Sequence(), fn) 238 | }) 239 | } 240 | -------------------------------------------------------------------------------- /cli/main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package main 7 | 8 | import ( 9 | "fmt" 10 | "os" 11 | 12 | "github.com/mitchellh/cli" 13 | "github.com/rande/pkgmirror/commands" 14 | ) 15 | 16 | var ( 17 | Version = "0.0.1-Dev" 18 | RefLog = "master" 19 | ) 20 | 21 | func main() { 22 | ui := &cli.BasicUi{Writer: os.Stdout} 23 | 24 | c := cli.NewCLI("pkgmirror", fmt.Sprintf("%s - %s", Version, RefLog)) 25 | c.Args = os.Args[1:] 26 | 27 | c.Commands = map[string]cli.CommandFactory{ 28 | "run": func() (cli.Command, error) { 29 | return &commands.ServerCommand{ 30 | Ui: ui, 31 | }, nil 32 | }, 33 | } 34 | 35 | exitStatus, _ := c.Run() 36 | 37 | os.Exit(exitStatus) 38 | } 39 | -------------------------------------------------------------------------------- /commands/server.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package commands 7 | 8 | import ( 9 | "flag" 10 | "fmt" 11 | "net/http" 12 | "net/http/pprof" 13 | "os" 14 | "strings" 15 | 16 | "github.com/BurntSushi/toml" 17 | "github.com/mitchellh/cli" 18 | "github.com/rande/goapp" 19 | "github.com/rande/pkgmirror" 20 | "github.com/rande/pkgmirror/api" 21 | "github.com/rande/pkgmirror/assets" 22 | "github.com/rande/pkgmirror/mirror/bower" 23 | "github.com/rande/pkgmirror/mirror/composer" 24 | "github.com/rande/pkgmirror/mirror/git" 25 | "github.com/rande/pkgmirror/mirror/npm" 26 | "github.com/rande/pkgmirror/mirror/static" 27 | "goji.io" 28 | "goji.io/pat" 29 | ) 30 | 31 | type ServerCommand struct { 32 | Ui cli.Ui 33 | Verbose bool 34 | Commands map[string]cli.CommandFactory 35 | ConfFile string 36 | LogLevel string 37 | } 38 | 39 | func (c *ServerCommand) Run(args []string) int { 40 | cmdFlags := flag.NewFlagSet("run", flag.ContinueOnError) 41 | cmdFlags.Usage = func() { 42 | c.Ui.Output(c.Help()) 43 | } 44 | 45 | cmdFlags.BoolVar(&c.Verbose, "verbose", false, "") 46 | cmdFlags.StringVar(&c.LogLevel, "log-level", "warning", "The log level") 47 | cmdFlags.StringVar(&c.ConfFile, "file", "/etc/pkgmirror.toml", "The configuration file") 48 | 49 | if err := cmdFlags.Parse(args); err != nil { 50 | return 1 51 | } 52 | 53 | config := &pkgmirror.Config{ 54 | CacheDir: fmt.Sprintf("%s/pkgmirror", os.TempDir()), 55 | LogLevel: "info", 56 | } 57 | 58 | if _, err := toml.DecodeFile(c.ConfFile, config); err != nil { 59 | c.Ui.Error(fmt.Sprintf("Unable to parse configuration file: %s", c.ConfFile)) 60 | 61 | return 1 62 | } 63 | 64 | c.Ui.Info("Configure app") 65 | 66 | l := goapp.NewLifecycle() 67 | 68 | app, err := pkgmirror.GetApp(config, l) 69 | 70 | if err != nil { 71 | c.Ui.Error(err.Error()) 72 | 73 | return 1 74 | } 75 | 76 | composer.ConfigureApp(config, l) 77 | git.ConfigureApp(config, l) 78 | npm.ConfigureApp(config, l) 79 | bower.ConfigureApp(config, l) 80 | static.ConfigureApp(config, l) 81 | api.ConfigureApp(config, l) 82 | assets.ConfigureApp(config, l) 83 | 84 | l.Run(func(app *goapp.App, state *goapp.GoroutineState) error { 85 | c.Ui.Info(fmt.Sprintf("Start HTTP Server (bind: %s)", config.InternalServer)) 86 | 87 | mux := app.Get("mux").(*goji.Mux) 88 | 89 | //mux.HandleFunc(pat.Get("/debug/pprof/cmdline"), http.HandlerFunc(pprof.Cmdline)) 90 | //mux.HandleFunc(pat.Get("/debug/pprof/profile"), http.HandlerFunc(pprof.Profile)) 91 | //mux.HandleFunc(pat.Get("/debug/pprof/symbol"), http.HandlerFunc(pprof.Symbol)) 92 | 93 | mux.HandleFunc(pat.Get("/debug/pprof/*"), http.HandlerFunc(pprof.Index)) 94 | 95 | http.ListenAndServe(config.InternalServer, mux) 96 | 97 | return nil 98 | }) 99 | 100 | c.Ui.Info("Start app lifecycle") 101 | 102 | return l.Go(app) 103 | } 104 | 105 | func (c *ServerCommand) Synopsis() string { 106 | return "Run the mirroring server." 107 | } 108 | 109 | func (c *ServerCommand) Help() string { 110 | return strings.TrimSpace(` 111 | Usage: pkgmirror run [options] 112 | 113 | Run the mirror server 114 | 115 | Options: 116 | -file The configuration file (default: /etc/pkgmirror.toml) 117 | -verbose Add verbose information to the output 118 | -log-level Log level (defaul: warning) 119 | possible values: debug, info, warning, error, fatal and panic 120 | `) 121 | } 122 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | type ComposerConfig struct { 9 | Server string 10 | Enabled bool 11 | Icon string 12 | } 13 | 14 | type BowerConfig struct { 15 | Server string 16 | Enabled bool 17 | Icon string 18 | } 19 | 20 | type NpmConfig struct { 21 | Server string 22 | Enabled bool 23 | Icon string 24 | Fallbacks []*struct { 25 | Server string 26 | } 27 | } 28 | 29 | type GitConfig struct { 30 | Server string 31 | Enabled bool 32 | Icon string 33 | Clone string 34 | } 35 | 36 | type StaticConfig struct { 37 | Server string 38 | Enabled bool 39 | Icon string 40 | } 41 | 42 | type Config struct { 43 | DataDir string 44 | LogDir string 45 | CacheDir string 46 | PublicServer string 47 | InternalServer string 48 | LogLevel string 49 | Composer map[string]*ComposerConfig 50 | Npm map[string]*NpmConfig 51 | Git map[string]*GitConfig 52 | Bower map[string]*BowerConfig 53 | Static map[string]*StaticConfig 54 | } 55 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | import ( 9 | "testing" 10 | 11 | "github.com/BurntSushi/toml" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test_Config(t *testing.T) { 16 | c := &Config{} 17 | 18 | confStr := ` 19 | DataDir = "/var/lib/pkgmirror" 20 | PublicServer = "https://mirror.example.com" 21 | InternalServer = "localhost:8000" 22 | 23 | [Composer] 24 | [Composer.packagist] 25 | Server = "https://packagist.org" 26 | Enabled = true 27 | 28 | [Composer.satis] 29 | Server = "https://satis.internal.org" 30 | Enabled = false 31 | 32 | [Npm] 33 | [Npm.npm] 34 | Server = "https://registry.npmjs.org" 35 | 36 | [Git] 37 | [Git.github] 38 | Server = "github.com" 39 | Clone = "git@gitbub.com:" 40 | Enabled = true 41 | 42 | [Static] 43 | [Static.drupal] 44 | Server = "drupal.org" 45 | ` 46 | 47 | _, err := toml.Decode(confStr, c) 48 | 49 | assert.NoError(t, err) 50 | assert.Equal(t, "/var/lib/pkgmirror", c.DataDir) 51 | assert.Equal(t, 2, len(c.Composer)) 52 | assert.Equal(t, "https://satis.internal.org", c.Composer["satis"].Server) 53 | assert.Equal(t, false, c.Composer["satis"].Enabled) 54 | assert.Equal(t, "https://packagist.org", c.Composer["packagist"].Server) 55 | assert.Equal(t, true, c.Composer["packagist"].Enabled) 56 | 57 | assert.Equal(t, 1, len(c.Npm)) 58 | assert.Equal(t, "https://registry.npmjs.org", c.Npm["npm"].Server) 59 | assert.Equal(t, false, c.Npm["npm"].Enabled) 60 | 61 | assert.Equal(t, 1, len(c.Git)) 62 | assert.Equal(t, "github.com", c.Git["github"].Server) 63 | assert.Equal(t, "git@gitbub.com:", c.Git["github"].Clone) 64 | assert.Equal(t, true, c.Git["github"].Enabled) 65 | 66 | assert.Equal(t, 1, len(c.Static)) 67 | assert.Equal(t, "drupal.org", c.Static["drupal"].Server) 68 | assert.Equal(t, false, c.Static["drupal"].Enabled) 69 | } 70 | -------------------------------------------------------------------------------- /docs/bower.md: -------------------------------------------------------------------------------- 1 | Bower mirroring 2 | ============= 3 | 4 | Mirroring Workflow 5 | ------------------ 6 | 7 | 1. Load https://registry.bower.io/packages json. It is a 4.3MB json file with all informations. 8 | 9 | [ 10 | { 11 | "name": "10digit-geo", 12 | "url": "https://github.com/10digit/geo.git" 13 | }, 14 | { 15 | "name": "10digit-invoices", 16 | "url": "https://github.com/10digit/invoices.git" 17 | }, 18 | { 19 | "name": "10digit-legal", 20 | "url": "https://github.com/10digit/legal.git" 21 | } 22 | 23 | 2. Update the local metadata. 24 | 25 | 26 | Entry Points 27 | ------------ 28 | 29 | * Get package information: ``/bower/bower/packages/package_name`` 30 | * Download all packages: ``/bower/bower/packages`` 31 | -------------------------------------------------------------------------------- /docs/composer.md: -------------------------------------------------------------------------------- 1 | Composer mirroring 2 | ================== 3 | 4 | The mirroring will fetch the latest information from packagist.org and keep the data in a local storage (boltdb). The 5 | archive paths can also be updated with the local git mirroring. 6 | 7 | Composer workflow 8 | ----------------- 9 | 10 | 1. Load https://packagist.org/packages.json, which contains definitions to some providers. 11 | 12 | provider-includes: { 13 | p/provider-2013$%hash%.json: { 14 | sha256: "81839b9e7c94fdecc520e0e33f8e47b092079568ccfa319650db0e353412bfc3" 15 | }, 16 | p/provider-2014$%hash%.json: { 17 | sha256: "27fb04c654fb35ac2cb50183cc03861396cdacfc57e5ce94735e71a44a393bc4" 18 | }, 19 | p/provider-2015$%hash%.json: { 20 | sha256: "9c5310ed37ea7fd7243e26a62b150f0c7c257065236a208301712f524a6e68e9" 21 | }, 22 | p/provider-2015-07$%hash%.json: { 23 | sha256: "c2b3d17ececc1cab2cdf039c5016513c51e83f6d0998ebf4ee4d37eb401f5b4d" 24 | }, 25 | p/provider-2015-10$%hash%.json: { 26 | sha256: "898cee210f3b6ee1b8ae98ac4875dbe9632c480b66ccc7382b21ff75ca2fad5a" 27 | }, 28 | p/provider-2016-01$%hash%.json: { 29 | sha256: "a2ebcb11c730eeb56c42af336e6654a2b58f7c62e9edaa3528fa4ad6def8bafe" 30 | }, 31 | p/provider-2016-04$%hash%.json: { 32 | sha256: "2dd634aa1adabfb1c82e6f93ac65bb35a120f651e6dd139787e3e09521427467" 33 | }, 34 | p/provider-archived$%hash%.json: { 35 | sha256: "dcf5bde9f42034979a3475a33d0282b60ce8db28c4b1ab98728a6c7b8c467e00" 36 | }, 37 | p/provider-latest$%hash%.json: { 38 | sha256: "09fc55f7e0e166e7a96d9d07460f87b88fd1aa78ba8bd4454c3c9e953d7e3253" 39 | } 40 | } 41 | 42 | The sha256 value is the hash of the target file. 43 | 44 | 2. Load the provider files, each file contains references to package file. 45 | 46 | { 47 | "providers": { 48 | "0s1r1s\/dev-shortcuts-bundle": { 49 | "sha256": "6c7710a1ca26d3c0f9dfc4c34bc3d6e71ed88d8783847ed82079601401e29b18" 50 | }, 51 | "0x20h\/monoconf": { 52 | "sha256": "9515a0ee8fce44be80ed35292384c2f908cabbf6a710099f4743b710bc47607e" 53 | }, 54 | "11ya\/excelbundle": { 55 | "sha256": "65dccb7f2d57c09c19519c1b3cdf7cbace1dfbf46f43736c2afcb95658d9c0f1" 56 | }, 57 | .... 58 | } 59 | 60 | 3. Load the package information. 61 | 62 | Mirroring workflow 63 | ------------------ 64 | 65 | 1. Load the packages.json file 66 | 2. Iterate over the providers. 67 | 4. Download the package definition and 68 | - alter path if required, 69 | - compute new hash 70 | - store the package in data layer using bzip compression to save bandwidth and local storage. 71 | 5. Update packages.json and providers.json to use the new hash and recompute the final hash for each provider. 72 | 6. Clean old references 73 | 74 | Storage 75 | ------- 76 | 77 | The storage layer uses [boltdb](https://github.com/boltdb/bolt). A packagist.org mirror is about 512MB on disk, 78 | the file is located in the ``PATH/composer/packagist.db``. 79 | -------------------------------------------------------------------------------- /docs/git.md: -------------------------------------------------------------------------------- 1 | Git Mirroring 2 | ============= 3 | 4 | The git mirroring service allows to mirror git repositories into a local server. 5 | 6 | Mirroring Workflow 7 | ------------------ 8 | 9 | All repositories are stored in the ``DataDir/git/hostname`` path. So for github.com it will be: ``DataDir/git/github.com``. 10 | 11 | 1. Iterate over each hostname 12 | 2. Start a goroutinne for each hostname 13 | 3. Iterate over folder ending by ``.git`` (up to 3 nested levels) 14 | 4. Run the ``fetch`` and ``update-server-info`` commands on each mirror 15 | 16 | Entry Points 17 | ------------ 18 | 19 | ### Clone repository 20 | 21 | The current implementation provides support for the [smart http protocol](https://git-scm.com/book/tr/v2/Git-on-the-Server-The-Protocols), so 22 | it is possible to only clone over http/https. 23 | 24 | git clone https://mirror.example.com/git/github.com/rande/pkgmirror.git 25 | 26 | ### Archive 27 | 28 | You can also download a zip for a specific version: 29 | 30 | curl https://mirror.example.com/git/github.com/rande/pkgmirror/master.git 31 | curl https://mirror.example.com/git/github.com/rande/pkgmirror/9c34490d5fb421d45bb8634b84308995b407fb4b.git 32 | 33 | Please note, only semver tags and commits are cached. 34 | 35 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | Dependencies installation 5 | ------------------------- 6 | 7 | 1. Install dependencies 8 | 9 | apt-get install git 10 | 11 | 2. Make sure the git client have valid credentials to clone/fetch remote repositories. 12 | 13 | * Github: create a ssh key and add the key on github. Please note, when mirroring repository, you need 14 | to use the ``git@github.com:/vendor/project.git`` protocol. For instance, ``git clone --mirror git@github.com:rande/pkgmirror.git``. 15 | 16 | 17 | PkgMirror installation 18 | ---------------------- 19 | 20 | 1. Download the latest version from the [releases page](https://github.com/rande/pkgmirror/releases) 21 | 2. Create a configuration file ``pkgmirror.toml`` 22 | 23 | DataDir = "/usr/local/web/pkgmirror/data" 24 | CacheDir = "/usr/local/web/pkgmirror/cache" 25 | PublicServer = "https://mirror.example.com" 26 | InternalServer = ":8000" 27 | 28 | 3. Create require folders 29 | 30 | mkdir -p /usr/local/web/{pkgmirror/data,pkgmirror/cache} 31 | 32 | 4. Start the process with a process manager 33 | 34 | ./pkgmirror -file pkgmirror.toml 35 | 36 | If you migrate from [ekino/phpmirroring](https://github.com/ekino/php-mirroring), please read [the migration guide](migration.md) -------------------------------------------------------------------------------- /docs/maintenance.md: -------------------------------------------------------------------------------- 1 | Maintenance 2 | =========== 3 | 4 | Backup 5 | ------ 6 | 7 | For now there is no internal mechanism to protect data integrity while backuping, so you need to stop 8 | the service, backup the data folder and restart the folder. 9 | 10 | Supervision 11 | ----------- 12 | 13 | Add health check endpoint... 14 | 15 | -------------------------------------------------------------------------------- /docs/migration.md: -------------------------------------------------------------------------------- 1 | Migration from ekino/phpmirroring 2 | ================================= 3 | 4 | The project ``ekino/php-mirroring`` is deprecated, the current page explains how to migrate your project from ``ekino/php-mirroring`` to ``pkgmirror``. 5 | 6 | Archive Update 7 | -------------- 8 | 9 | The old urls look likes: ``http://oldserver.com/cache.php/github.com/doctrine/cache/47cdc76ceb95cc591d9c79a36dc3794975b5d136.zip`` 10 | 11 | The new urls now are: ``http://newserver.com/git/github.com/doctrine/cache/47cdc76ceb95cc591d9c79a36dc3794975b5d136.zip`` 12 | 13 | 14 | Repository Update 15 | ----------------- 16 | 17 | The old urls look likes: ``git@oldserver.com:/mirrors/github.com/doctrine/cache.git`` 18 | 19 | The new urls now are: ``http://newserver.com/git/github.com/doctrine/cache.git`` 20 | 21 | 22 | Migration 23 | --------- 24 | 25 | You can update your ``composer.lock`` file by running the command : ``cat packages.lock | sed -e 's|http://oldserver.com/cache.php|https://newserver.com/git|' | sed -e 's|git@oldserver.com:/mirrors|https://newserver.com/git|'``. 26 | 27 | If the output is ok, you can redirect the output to the ``composer.lock`` file 28 | 29 | You also need to update the ``composer.json`` file: 30 | 31 | Before: 32 | 33 | "repositories":[ 34 | { "packagist": false }, 35 | { "type": "composer", "url": "http://oldserver.com"} 36 | ], 37 | 38 | After: 39 | 40 | "repositories":[ 41 | { "packagist": false }, 42 | { "type": "composer", "url": "http://newserver.com/packagist"} 43 | ], 44 | 45 | If you use the same domain name, you can clear local composer cache to avoid any issues: ``rm -rf ~/.composer/cache`` 46 | -------------------------------------------------------------------------------- /docs/npm.md: -------------------------------------------------------------------------------- 1 | NPM mirroring 2 | ============= 3 | 4 | Mirroring Workflow 5 | ------------------ 6 | 7 | > The url `https://registry.npmjs.org/-/all` is not available anymore, so it not possible to have 8 | > a full copy of the npm's registry anymore. 9 | > Pkgmirror will store on-demand and sync local packages. 10 | 11 | 1. On demand, the proxy will load the remote version if the package is new. 12 | 2. Update the local data with the modified date, if changed then update related package reference. 13 | 3. The package update will download the package information from ``https://registry.npmjs.org/package_name`` and update tarbal reference to point to the local entry point. 14 | 15 | 16 | Entry Points 17 | ------------ 18 | 19 | * Get package information: ``/npm/package_name`` 20 | * Download archive: ``/npm/package_name/-/package_name-version.tgz`` 21 | -------------------------------------------------------------------------------- /docs/pkgmirror.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/docs/pkgmirror.gif -------------------------------------------------------------------------------- /docs/static.md: -------------------------------------------------------------------------------- 1 | Static mirroring 2 | ================ 3 | 4 | This service is a simple proxy to remote file. Each server need to be configured. 5 | 6 | For now, there is no specific configuration, the current behavior is: 7 | 8 | - if ``statusCode == 200`` the file will be stored into the local cache 9 | - if ``statusCode == 404`` the file will no be stored and a 404 code will be send to the suer 10 | - if ``statusCode == 302`` the internal http lib will follow redirect and the final will be stored with the initial provided path. 11 | - any other code will result of an "Internal Server Error" -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | Main configuration 5 | ------------------ 6 | 7 | DataDir = "/var/lib/pkgmirror/data" 8 | CacheDir = "/var/lib/pkgmirror/cache" 9 | PublicServer = "https://mirrors.example.com" 10 | InternalServer = ":8000" 11 | 12 | Composer 13 | -------- 14 | 15 | To add a new repository, for instance, the official one: 16 | 17 | [Composer] 18 | [Composer.packagist] 19 | Server = "https://packagist.org" 20 | Enabled = true 21 | Icon = "https://getcomposer.org/img/logo-composer-transparent.png" 22 | 23 | [Composer.drupal8] 24 | Server = "https://packages.drupal.org/8" 25 | Enabled = true 26 | Icon = "https://www.drupal.org/files/druplicon-small.png" 27 | 28 | [Composer.drupal7] 29 | Server = "https://packages.drupal.org/7" 30 | Enabled = true 31 | Icon = "https://www.drupal.org/files/druplicon-small.png" 32 | 33 | 34 | Next, you need to declare the mirror in your ``composer.json`` file: 35 | 36 | { 37 | "repositories":[ 38 | { "packagist": false }, 39 | { "type": "composer", "url": "https://localhost/composer/packagist"} 40 | { "type": "composer", "url": "https://localhost/composer/drupal8"} 41 | ], 42 | "require": { 43 | "sonata-project/exporter": "*" 44 | } 45 | } 46 | 47 | The ``packagist`` key is used here as an example. 48 | 49 | 50 | > You also need to setup `git` and `static` configuration to be able to download assets or clone repository. 51 | 52 | Npm 53 | --- 54 | 55 | To add new repository, for instance, https://registry.npmjs.org 56 | 57 | [Npm] 58 | [Npm.npm] 59 | Server = "https://registry.npmjs.org" 60 | Enabled = true 61 | Icon = "https://cldup.com/Rg6WLgqccB.svg" 62 | 63 | Next, you need to declare the registry in npm 64 | 65 | npm set registry https://localhost/npm/npm 66 | 67 | Git 68 | --- 69 | 70 | To add new servers: 71 | 72 | [Git] 73 | [Git.github] 74 | Server = "github.com" 75 | Clone = "git@gitbub.com:{path}" 76 | Enabled = true 77 | Icon = "https://assets-cdn.github.com/images/modules/logos_page/GitHub-Mark.png" 78 | 79 | [Git.drupal] 80 | Server = "drupal.org" 81 | Clone = "https://git.drupal.org/{path}" 82 | Enabled = true 83 | Icon = "https://www.drupal.org/files/druplicon-small.png" 84 | 85 | 86 | If the ``Clone`` settings is not set, you need to manually add git repository: 87 | 88 | 1. Connect to the server 89 | 2. Clone a repository 90 | 91 | git clone --mirror git@github.com:rande/gonode.git ./data/git/github.com/rande/gonode.git 92 | 93 | 94 | > In order to clone, make sure you have a proper ssh key setup and remote ssh fingerprint accepted on the server. 95 | 96 | Bower 97 | ----- 98 | 99 | To add a new repository, for instance, https://registry.bower.io: 100 | 101 | [Bower] 102 | [Bower.bower] 103 | Server = "https://registry.bower.io" 104 | Enabled = true 105 | Icon = "https://bower.io/img/bower-logo.svg" 106 | 107 | You need to declare the mirror in your .bowerrc file: 108 | 109 | { 110 | "registry": { 111 | "search": ["https://localhost/bower/bower"], 112 | "register": "https://localhost/bower/bower" 113 | } 114 | } 115 | 116 | Static 117 | ------ 118 | 119 | To add a new server: 120 | 121 | [Static] 122 | [Static.drupal] 123 | Server = "https://ftp.drupal.org/files/projects" 124 | Icon = "https://www.drupal.org/files/druplicon-small.png" 125 | 126 | You can now download file from ``https://localhos/static/drupal/panopoly-7.x-1.40-core.tar.gz`` 127 | 128 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | import ( 9 | "errors" 10 | ) 11 | 12 | var ( 13 | SyncInProgressError = errors.New("A synchronization is already running") 14 | DatabaseLockedError = errors.New("The database is locked") 15 | EmptyKeyError = errors.New("No value available") 16 | ResourceNotFoundError = errors.New("Resource not found") 17 | EmptyDataError = errors.New("Empty data") 18 | SameKeyError = errors.New("Same key") 19 | HttpError = errors.New("Http error") 20 | InvalidPackageError = errors.New("Invalid package error") 21 | InvalidReferenceError = errors.New("Invalid reference") 22 | ) 23 | -------------------------------------------------------------------------------- /fixtures/git/foo.bare/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /fixtures/git/foo.bare/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /fixtures/git/foo.bare/objects/2d/a62f8886014fb045e41b659e4fa83db5ef24d2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/fixtures/git/foo.bare/objects/2d/a62f8886014fb045e41b659e4fa83db5ef24d2 -------------------------------------------------------------------------------- /fixtures/git/foo.bare/objects/96/01539ddd2dbf994af80fdc78e5df50491985d7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/fixtures/git/foo.bare/objects/96/01539ddd2dbf994af80fdc78e5df50491985d7 -------------------------------------------------------------------------------- /fixtures/git/foo.bare/objects/9b/9cc9573693611badb397b5d01a1e6645704da7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/fixtures/git/foo.bare/objects/9b/9cc9573693611badb397b5d01a1e6645704da7 -------------------------------------------------------------------------------- /fixtures/git/foo.bare/objects/cb/734ed1fceda100dbe1b259f039b061e8ea2f90: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/fixtures/git/foo.bare/objects/cb/734ed1fceda100dbe1b259f039b061e8ea2f90 -------------------------------------------------------------------------------- /fixtures/git/foo.bare/refs/heads/master: -------------------------------------------------------------------------------- 1 | 9b9cc9573693611badb397b5d01a1e6645704da7 2 | -------------------------------------------------------------------------------- /fixtures/git/foo.bare/refs/tags/0.0.1: -------------------------------------------------------------------------------- 1 | 2da62f8886014fb045e41b659e4fa83db5ef24d2 2 | -------------------------------------------------------------------------------- /fixtures/mock/bower/packages: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "10digit-geo", 4 | "url": "https://github.com/10digit/geo.git" 5 | }, 6 | { 7 | "name": "10digit-invoices", 8 | "url": "https://github.com/10digit/invoices.git" 9 | }, 10 | { 11 | "name": "10digit-legal", 12 | "url": "https://github.com/10digit/legal.git" 13 | } 14 | ] -------------------------------------------------------------------------------- /fixtures/mock/composer/p/0n3s3c/baselibrary$f00e050833c8ac3323177dc59eb69680fc0920c1e881835a1e64adcb60ea5157.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": { 3 | "0n3s3c\/baselibrary": { 4 | "0.5.0": { 5 | "name": "0n3s3c\/baselibrary", 6 | "description": "Library for working with objects in PHP", 7 | "keywords": [ 8 | "library", 9 | "collection" 10 | ], 11 | "homepage": "", 12 | "version": "0.5.0", 13 | "version_normalized": "0.5.0.0", 14 | "license": [ 15 | "MIT" 16 | ], 17 | "authors": [ 18 | { 19 | "name": "Joshua Jones", 20 | "email": "joshua.jones.software@gmail.com" 21 | } 22 | ], 23 | "source": { 24 | "type": "git", 25 | "url": "https:\/\/github.com\/0N3S3C\/BaseLibrary.git", 26 | "reference": "27892d3e65147f2eb706dec13c5d9e454a692ce6" 27 | }, 28 | "dist": { 29 | "type": "zip", 30 | "url": "https:\/\/api.github.com\/repos\/0N3S3C\/BaseLibrary\/zipball\/27892d3e65147f2eb706dec13c5d9e454a692ce6", 31 | "reference": "27892d3e65147f2eb706dec13c5d9e454a692ce6", 32 | "shasum": "" 33 | }, 34 | "type": "library", 35 | "time": "2016-03-25T17:29:35+00:00", 36 | "autoload": { 37 | "psr-4": { 38 | "Base\\": "src\/Base" 39 | } 40 | }, 41 | "require": { 42 | "php": ">=5.5.0" 43 | }, 44 | "require-dev": { 45 | "phpunit\/phpunit": "^5.0" 46 | }, 47 | "uid": 752383 48 | }, 49 | "0.5.1": { 50 | "name": "0n3s3c\/baselibrary", 51 | "description": "Library for working with objects in PHP", 52 | "keywords": [ 53 | "library", 54 | "collection" 55 | ], 56 | "homepage": "", 57 | "version": "0.5.1", 58 | "version_normalized": "0.5.1.0", 59 | "license": [ 60 | "MIT" 61 | ], 62 | "authors": [ 63 | { 64 | "name": "Joshua Jones", 65 | "email": "joshua.jones.software@gmail.com" 66 | } 67 | ], 68 | "source": { 69 | "type": "git", 70 | "url": "https:\/\/github.com\/0N3S3C\/BaseLibrary.git", 71 | "reference": "8de06188fdf335651ff2114a1f7e4fb343da4f0d" 72 | }, 73 | "dist": { 74 | "type": "zip", 75 | "url": "https:\/\/api.github.com\/repos\/0N3S3C\/BaseLibrary\/zipball\/8de06188fdf335651ff2114a1f7e4fb343da4f0d", 76 | "reference": "8de06188fdf335651ff2114a1f7e4fb343da4f0d", 77 | "shasum": "" 78 | }, 79 | "type": "library", 80 | "time": "2016-03-28T12:57:25+00:00", 81 | "autoload": { 82 | "psr-4": { 83 | "Base\\": "src\/Base" 84 | } 85 | }, 86 | "require": { 87 | "php": ">=5.5.0" 88 | }, 89 | "require-dev": { 90 | "phpunit\/phpunit": "^5.0" 91 | }, 92 | "uid": 754198 93 | } 94 | } 95 | } 96 | } -------------------------------------------------------------------------------- /fixtures/mock/composer/p/provider-mock$9bd35df8f2fab78bd7e7469572b7d8e5ba58d11b702232db6ce9b6f1b0bf0fe5.json: -------------------------------------------------------------------------------- 1 | { 2 | "providers": { 3 | "0n3s3c\/baselibrary": { 4 | "sha256": "f00e050833c8ac3323177dc59eb69680fc0920c1e881835a1e64adcb60ea5157" 5 | }, 6 | "symfony\/framework-standard-edition": { 7 | "sha256": "c08c23d91489ab0c8098739084a5f2873c331cf374268af722c5867038f9c0be" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /fixtures/mock/composer/packages.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": [], 3 | "notify": "\/downloads\/%package%", 4 | "notify-batch": "\/downloads\/", 5 | "providers-url": "\/composer\/p\/%package%$%hash%.json", 6 | "search": "\/search.json?q=%query%", 7 | "provider-includes": { 8 | "p\/provider-mock$%hash%.json": { 9 | "sha256": "9bd35df8f2fab78bd7e7469572b7d8e5ba58d11b702232db6ce9b6f1b0bf0fe5" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /fixtures/mock/invalid.json: -------------------------------------------------------------------------------- 1 | { 2 | "foo": "bar" 3 | -------------------------------------------------------------------------------- /fixtures/mock/npm/@types/react/-/react-0.0.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/fixtures/mock/npm/@types/react/-/react-0.0.0.tgz -------------------------------------------------------------------------------- /fixtures/mock/npm/angular-nvd3-nb/-/angular-nvd3-nb-1.0.5-nb.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rande/pkgmirror/452f8747413a59a666566c90cf960c0f1ae3647e/fixtures/mock/npm/angular-nvd3-nb/-/angular-nvd3-nb-1.0.5-nb.tgz -------------------------------------------------------------------------------- /fixtures/mock/npm/angular-nvd3-nb/README.md: -------------------------------------------------------------------------------- 1 | The golang ServeFile will look for an index.html file if a folder is found. 2 | 3 | So the ``/npm/angular-nvd3-nb`` request will be ``/npm/angular-nvd3-nb/index.html`` -------------------------------------------------------------------------------- /fixtures/mock/npm/angular-oauth: -------------------------------------------------------------------------------- 1 | { 2 | "_id": "angular-oauth", 3 | "_rev": "3-bcf2c8638122d39e4039e7e7e119faf3", 4 | "name": "angular-oauth", 5 | "time": { 6 | "modified": "2015-01-12T21:06:46.770Z", 7 | "created": "2015-01-12T21:06:46.770Z", 8 | "1.0.0": "2015-01-12T21:06:46.770Z", 9 | "unpublished": { 10 | "name": "penso", 11 | "time": "2015-01-12T21:22:47.817Z", 12 | "tags": { 13 | "latest": "1.0.0" 14 | }, 15 | "maintainers": [ 16 | { 17 | "name": "penso", 18 | "email": "rui.penso@gmail.com" 19 | } 20 | ], 21 | "description": "Angular OAuth", 22 | "versions": [ 23 | "1.0.0" 24 | ] 25 | } 26 | }, 27 | "_attachments": { 28 | 29 | } 30 | } -------------------------------------------------------------------------------- /fixtures/mock/static/file.txt: -------------------------------------------------------------------------------- 1 | This is a sample test file. -------------------------------------------------------------------------------- /fixtures/npm/gulp-app-manager.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id": "gulp-app-manager", 3 | "_rev": "6-b042b4d6432c241ca895208b47b6aee4", 4 | "name": "gulp-app-manager", 5 | "time": { 6 | "modified": "2015-11-23T12:12:06.113Z", 7 | "created": "2015-11-20T19:18:41.563Z", 8 | "0.0.1": "2015-11-20T19:18:41.563Z", 9 | "0.0.2": "2015-11-20T19:31:53.809Z", 10 | "0.0.3": "2015-11-23T11:09:03.657Z", 11 | "unpublished": { 12 | "name": "ricardogobbosouza", 13 | "time": "2015-11-23T12:12:18.460Z", 14 | "tags": { 15 | "latest": "0.0.3" 16 | }, 17 | "maintainers": [ 18 | { 19 | "name": "ricardogobbosouza", 20 | "email": "ricardogobbosouza@yahoo.com.br" 21 | } 22 | ], 23 | "versions": [ 24 | "0.0.3" 25 | ] 26 | } 27 | }, 28 | "_attachments": {} 29 | } -------------------------------------------------------------------------------- /fixtures/npm/math_example_bulbignz.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id": "math_example_bulbignz", 3 | "_rev": "2-5d5bd3e3170af30f145f6aa86599fecf", 4 | "name": "math_example_bulbignz", 5 | "description": "An example of creating a package", 6 | "dist-tags": { 7 | "latest": "0.0.0" 8 | }, 9 | "versions": { 10 | "0.0.0": { 11 | "name": "math_example_bulbignz", 12 | "version": "0.0.0", 13 | "description": "An example of creating a package", 14 | "main": "bin/main.js", 15 | "scripts": { 16 | "test": "echo \"Error: no test specified\" && exit 1" 17 | }, 18 | "repository": "", 19 | "keywords": [ 20 | "math", 21 | "example", 22 | "addition", 23 | "substraction", 24 | "mumtiplication", 25 | "division", 26 | "fibonacci" 27 | ], 28 | "author": { 29 | "name": "BulbiGNZ" 30 | }, 31 | "license": "ISC", 32 | "_id": "math_example_bulbignz@0.0.0", 33 | "dist": { 34 | "shasum": "046c474d1f3ade30505328b22e834cb6d30b8d45", 35 | "tarball": "https://registry.npmjs.org/math_example_bulbignz/-/math_example_bulbignz-0.0.0.tgz" 36 | }, 37 | "_from": ".", 38 | "_npmVersion": "1.4.3", 39 | "_npmUser": { 40 | "name": "bulbignz", 41 | "email": "mario.vu@gmail.com" 42 | }, 43 | "maintainers": [ 44 | { 45 | "name": "bulbignz", 46 | "email": "mario.vu@gmail.com" 47 | } 48 | ], 49 | "directories": {} 50 | } 51 | }, 52 | "readme": "math_example package\r\n====================\r\n\r\nblablabla\r\n- **addition** Adds two numbers and return the result.\r\n- etc.", 53 | "maintainers": [ 54 | { 55 | "name": "bulbignz", 56 | "email": "mario.vu@gmail.com" 57 | } 58 | ], 59 | "time": { 60 | "modified": "2014-04-13T22:31:41.228Z", 61 | "created": "2014-04-13T22:31:41.228Z", 62 | "0.0.0": "2014-04-13T22:31:41.228Z" 63 | }, 64 | "keywords": [ 65 | "math", 66 | "example", 67 | "addition", 68 | "substraction", 69 | "mumtiplication", 70 | "division", 71 | "fibonacci" 72 | ], 73 | "author": { 74 | "name": "BulbiGNZ" 75 | }, 76 | "license": "ISC", 77 | "readmeFilename": "README.md", 78 | "_attachments": {} 79 | } -------------------------------------------------------------------------------- /fixtures/packagist/p/0n3s3c/baselibrary$3a3dbbc33805b6748f859e8f2c517355f42e2f6d4b71daad077794842dca280c.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": { 3 | "0n3s3c\/baselibrary": { 4 | "0.5.0": { 5 | "name": "0n3s3c\/baselibrary", 6 | "description": "Library for working with objects in PHP", 7 | "keywords": [ 8 | "library", 9 | "collection" 10 | ], 11 | "homepage": "", 12 | "version": "0.5.0", 13 | "version_normalized": "0.5.0.0", 14 | "license": [ 15 | "MIT" 16 | ], 17 | "authors": [ 18 | { 19 | "name": "Joshua Jones", 20 | "email": "joshua.jones.software@gmail.com" 21 | } 22 | ], 23 | "source": { 24 | "type": "git", 25 | "url": "https:\/\/github.com\/0N3S3C\/BaseLibrary.git", 26 | "reference": "27892d3e65147f2eb706dec13c5d9e454a692ce6" 27 | }, 28 | "dist": { 29 | "type": "zip", 30 | "url": "https:\/\/api.github.com\/repos\/0N3S3C\/BaseLibrary\/zipball\/27892d3e65147f2eb706dec13c5d9e454a692ce6", 31 | "reference": "27892d3e65147f2eb706dec13c5d9e454a692ce6", 32 | "shasum": "" 33 | }, 34 | "type": "library", 35 | "time": "2016-03-25T17:29:35+00:00", 36 | "autoload": { 37 | "psr-4": { 38 | "Base\\": "src\/Base" 39 | } 40 | }, 41 | "require": { 42 | "php": ">=5.5.0" 43 | }, 44 | "require-dev": { 45 | "phpunit\/phpunit": "^5.0" 46 | }, 47 | "uid": 752383 48 | }, 49 | "0.5.1": { 50 | "name": "0n3s3c\/baselibrary", 51 | "description": "Library for working with objects in PHP", 52 | "keywords": [ 53 | "library", 54 | "collection" 55 | ], 56 | "homepage": "", 57 | "version": "0.5.1", 58 | "version_normalized": "0.5.1.0", 59 | "license": [ 60 | "MIT" 61 | ], 62 | "authors": [ 63 | { 64 | "name": "Joshua Jones", 65 | "email": "joshua.jones.software@gmail.com" 66 | } 67 | ], 68 | "source": { 69 | "type": "git", 70 | "url": "https:\/\/github.com\/0N3S3C\/BaseLibrary.git", 71 | "reference": "8de06188fdf335651ff2114a1f7e4fb343da4f0d" 72 | }, 73 | "dist": { 74 | "type": "zip", 75 | "url": "https:\/\/api.github.com\/repos\/0N3S3C\/BaseLibrary\/zipball\/8de06188fdf335651ff2114a1f7e4fb343da4f0d", 76 | "reference": "8de06188fdf335651ff2114a1f7e4fb343da4f0d", 77 | "shasum": "" 78 | }, 79 | "type": "library", 80 | "time": "2016-03-28T12:57:25+00:00", 81 | "autoload": { 82 | "psr-4": { 83 | "Base\\": "src\/Base" 84 | } 85 | }, 86 | "require": { 87 | "php": ">=5.5.0" 88 | }, 89 | "require-dev": { 90 | "phpunit\/phpunit": "^5.0" 91 | }, 92 | "uid": 754198 93 | } 94 | } 95 | } 96 | } -------------------------------------------------------------------------------- /fixtures/packagist/packages.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": [], 3 | "notify": "\/downloads\/%package%", 4 | "notify-batch": "\/downloads\/", 5 | "providers-url": "\/p\/%package%$%hash%.json", 6 | "search": "\/search.json?q=%query%", 7 | "provider-includes": { 8 | "p\/provider-2013$%hash%.json": { 9 | "sha256": "370af0b17d1ec5b0325bdb0126c9007b69647fafe5df8b5ecf79241e09745841" 10 | }, 11 | "p\/provider-2014$%hash%.json": { 12 | "sha256": "5cb5c5557ce3f5e779ae4a73abbbe820347be844666d9bb1cfc24af4609da2f7" 13 | }, 14 | "p\/provider-2015$%hash%.json": { 15 | "sha256": "1da7d6dbac884ff87bba52104ed9312738c7b030d7d357e6d267163da3f0e840" 16 | }, 17 | "p\/provider-2015-07$%hash%.json": { 18 | "sha256": "2ddaa9220d0f1af9e28d4d8ed6783e03d47fdba463b046794cb5edb89929815b" 19 | }, 20 | "p\/provider-2015-10$%hash%.json": { 21 | "sha256": "d1f8b859c696ba1b1b12b70ee6a2672a0f5b6b35e7f387aef45c5b34859d4b07" 22 | }, 23 | "p\/provider-2016-01$%hash%.json": { 24 | "sha256": "992ca91dab9b17af3b3e71aad9777d13712c9e6ee7c37ab35559759cb087979d" 25 | }, 26 | "p\/provider-2016-04$%hash%.json": { 27 | "sha256": "9cafd372835295e5ef009c5700d99cd39b4ba133ba3794641620c8f22995feeb" 28 | }, 29 | "p\/provider-archived$%hash%.json": { 30 | "sha256": "dcf5bde9f42034979a3475a33d0282b60ce8db28c4b1ab98728a6c7b8c467e00" 31 | }, 32 | "p\/provider-latest$%hash%.json": { 33 | "sha256": "1fdefd362a06a8d75dec117153bd4fbf1a12f5307dbcb99d9732648a3d795139" 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /fixtures/project/Makefile: -------------------------------------------------------------------------------- 1 | install-dist: ## Install backend dependencies 2 | rm -rf ~/.composer composer.lock vendor && composer install --prefer-dist -vvv 3 | 4 | install-source: ## Install backend dependencies 5 | rm -rf ~/.composer composer.lock vendor && composer install --prefer-source -vvv -------------------------------------------------------------------------------- /fixtures/project/composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "repositories":[ 3 | { "packagist": false }, 4 | { "type": "composer", "url": "http://localhost:8000/packagist"} 5 | ], 6 | 7 | "require": { 8 | "sonata-project/exporter": "*" 9 | }, 10 | "config": { 11 | "secure-http": false 12 | } 13 | } -------------------------------------------------------------------------------- /fixtures/project/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "dependencies": { 7 | "babel-core": "^6.5.2", 8 | "babel-loader": "^6.2.1", 9 | "babel-plugin-add-module-exports": "^0.1.2", 10 | "babel-plugin-transform-decorators-legacy": "^1.3.4", 11 | "babel-plugin-transform-react-display-name": "^6.3.13", 12 | "babel-plugin-transform-runtime": "^6.3.13", 13 | "babel-polyfill": "^6.3.14", 14 | "babel-preset-es2015": "^6.3.13", 15 | "babel-preset-react": "^6.3.13", 16 | "babel-preset-stage-0": "^6.3.13", 17 | "babel-register": "^6.3.13", 18 | "babel-runtime": "^6.3.19", 19 | "body-parser": "^1.14.1", 20 | "compression": "^1.6.0", 21 | "express": "^4.13.3", 22 | "express-session": "^1.12.1", 23 | "file-loader": "^0.8.5", 24 | "hoist-non-react-statics": "^1.0.3", 25 | "http-proxy": "^1.12.0", 26 | "invariant": "^2.2.0", 27 | "less": "^2.5.3", 28 | "less-loader": "^2.2.1", 29 | "lru-memoize": "^1.0.0", 30 | "map-props": "^1.0.0", 31 | "multireducer": "^2.0.0", 32 | "piping": "^0.3.0", 33 | "pretty-error": "^1.2.0", 34 | "react": "^0.14.2", 35 | "react-bootstrap": "^0.28.1", 36 | "react-dom": "^0.14.1", 37 | "react-helmet": "^2.2.0", 38 | "react-inline-css": "^2.0.0", 39 | "react-redux": "^4.0.0", 40 | "react-router": "2.0.0", 41 | "react-router-bootstrap": "^0.20.1", 42 | "react-router-redux": "^4.0.0", 43 | "redux": "^3.0.4", 44 | "redux-async-connect": "^1.0.0-rc2", 45 | "redux-form": "^3.0.12", 46 | "scroll-behavior": "^0.3.2", 47 | "serialize-javascript": "^1.1.2", 48 | "serve-favicon": "^2.3.0", 49 | "socket.io": "^1.3.7", 50 | "socket.io-client": "^1.3.7", 51 | "superagent": "^1.4.0", 52 | "url-loader": "^0.5.7", 53 | "warning": "^2.1.0", 54 | "webpack-isomorphic-tools": "^2.2.18" 55 | }, 56 | "devDependencies": { 57 | "autoprefixer-loader": "^3.1.0", 58 | "babel-eslint": "^5.0.0-beta6", 59 | "babel-plugin-react-transform": "^2.0.0", 60 | "babel-plugin-typecheck": "^3.6.0", 61 | "better-npm-run": "0.0.8", 62 | "bootstrap-sass": "^3.3.5", 63 | "bootstrap-sass-loader": "^1.0.9", 64 | "chai": "^3.3.0", 65 | "clean-webpack-plugin": "^0.1.6", 66 | "concurrently": "^0.1.1", 67 | "css-loader": "^0.23.1", 68 | "eslint": "1.10.3", 69 | "eslint-config-airbnb": "0.1.0", 70 | "eslint-loader": "^1.0.0", 71 | "eslint-plugin-import": "^0.8.0", 72 | "eslint-plugin-react": "^3.5.0", 73 | "extract-text-webpack-plugin": "^0.9.1", 74 | "font-awesome": "^4.4.0", 75 | "font-awesome-webpack": "0.0.4", 76 | "json-loader": "^0.5.4", 77 | "karma": "^0.13.10", 78 | "karma-cli": "^0.1.1", 79 | "karma-mocha": "^0.2.0", 80 | "karma-mocha-reporter": "^1.1.1", 81 | "karma-phantomjs-launcher": "^0.2.1", 82 | "karma-sourcemap-loader": "^0.3.5", 83 | "karma-webpack": "^1.7.0", 84 | "mocha": "^2.3.3", 85 | "node-sass": "^3.4.2", 86 | "phantomjs": "^1.9.18", 87 | "phantomjs-polyfill": "0.0.1", 88 | "react-a11y": "^0.2.6", 89 | "react-addons-test-utils": "^0.14.0", 90 | "react-transform-catch-errors": "^1.0.0", 91 | "react-transform-hmr": "^1.0.1", 92 | "redbox-react": "^1.1.1", 93 | "redux-devtools": "^3.0.0-beta-3", 94 | "redux-devtools-dock-monitor": "^1.0.0-beta-3", 95 | "redux-devtools-log-monitor": "^1.0.0-beta-3", 96 | "sass-loader": "^3.1.2", 97 | "sinon": "^1.17.2", 98 | "strip-loader": "^0.1.0", 99 | "style-loader": "^0.13.0", 100 | "timekeeper": "0.0.5", 101 | "webpack": "^1.12.9", 102 | "webpack-dev-middleware": "^1.4.0", 103 | "webpack-hot-middleware": "^2.5.0" 104 | }, 105 | "scripts": { 106 | "test": "echo \"Error: no test specified\" && exit 1" 107 | }, 108 | "author": "", 109 | "license": "ISC" 110 | } 111 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: be9a945aa6f914e7fa9102bac9afb55efcfe22940538e573a052867f0929c9bb 2 | updated: 2018-10-04T11:41:40.000507+02:00 3 | imports: 4 | - name: github.com/AaronO/go-git-http 5 | version: a8b8273a5ac1dbfb412cd2b70382badb1aceeadb 6 | repo: https://github.com/rande/go-git-http.git 7 | vcs: git 8 | - name: github.com/armon/go-radix 9 | version: 4239b77079c7b5d1243b7b4736304ce8ddb6f0f2 10 | - name: github.com/aws/aws-sdk-go 11 | version: 7557129b846473b8edd52853c077d21b7934b699 12 | subpackages: 13 | - aws 14 | - aws/awserr 15 | - aws/awsutil 16 | - aws/client 17 | - aws/client/metadata 18 | - aws/corehandlers 19 | - aws/credentials 20 | - aws/credentials/ec2rolecreds 21 | - aws/credentials/endpointcreds 22 | - aws/credentials/stscreds 23 | - aws/defaults 24 | - aws/ec2metadata 25 | - aws/request 26 | - aws/session 27 | - aws/signer/v4 28 | - private/endpoints 29 | - private/protocol 30 | - private/protocol/query 31 | - private/protocol/query/queryutil 32 | - private/protocol/rest 33 | - private/protocol/restxml 34 | - private/protocol/xml/xmlutil 35 | - private/waiter 36 | - service/s3 37 | - service/sts 38 | - name: github.com/bakins/logrus-middleware 39 | version: 2f037bcda984f172af9c62c9048a5e6c12fa3179 40 | - name: github.com/bgentry/speakeasy 41 | version: 675b82c74c0ed12283ee81ba8a534c8982c07b85 42 | - name: github.com/boltdb/bolt 43 | version: 2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8 44 | - name: github.com/BurntSushi/toml 45 | version: 3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005 46 | - name: github.com/davecgh/go-spew 47 | version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 48 | subpackages: 49 | - spew 50 | - name: github.com/go-ini/ini 51 | version: 6e4869b434bd001f6983749881c7ead3545887d8 52 | - name: github.com/jmespath/go-jmespath 53 | version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d 54 | - name: github.com/mattn/go-isatty 55 | version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 56 | - name: github.com/mitchellh/cli 57 | version: 074e243518e473e0887217008dd013e07755be8c 58 | - name: github.com/NYTimes/gziphandler 59 | version: f6438dbf4a82c56684964b03956aa727b0d7816b 60 | - name: github.com/pmezard/go-difflib 61 | version: d8ed2627bdf02c080bf22230dbb337003b7aba2d 62 | subpackages: 63 | - difflib 64 | - name: github.com/rande/goapp 65 | version: 52b532af1ee53a5c7863590bb85f688b6163ac97 66 | - name: github.com/rande/gonode 67 | version: f9e1126252df14ef4e80752ba393bd0db5c7313c 68 | subpackages: 69 | - core/vault 70 | - name: github.com/Sirupsen/logrus 71 | version: ba1b36c82c5e05c4f912a88eab0dcd91a171688f 72 | - name: github.com/stretchr/objx 73 | version: cbeaeb16a013161a98496fad62933b1d21786672 74 | - name: github.com/stretchr/testify 75 | version: f35b8ab0b5a2cef36673838d662e249dd9c94686 76 | subpackages: 77 | - assert 78 | - mock 79 | - name: goji.io 80 | version: bc2e9e7dc2769805d1b6ce4dc48ec23a03ae82ec 81 | subpackages: 82 | - internal 83 | - pat 84 | - pattern 85 | - name: golang.org/x/net 86 | version: 8b4af36cd21a1f85a7484b49feb7c79363106d8e 87 | subpackages: 88 | - context 89 | - name: golang.org/x/sys 90 | version: 002cbb5f952456d0c50e0d2aff17ea5eca716979 91 | subpackages: 92 | - unix 93 | testImports: [] 94 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/rande/pkgmirror 2 | owners: 3 | - name: Thomas Rabaix 4 | email: thomas.rabaix@gmail.com 5 | homepage: https://thomas.rabaix.net 6 | excludeDirs: 7 | - gui 8 | - build 9 | - docs 10 | - fixtures 11 | import: 12 | - package: github.com/BurntSushi/toml 13 | version: ^0.2.0 14 | - package: github.com/NYTimes/gziphandler 15 | - package: github.com/Sirupsen/logrus 16 | version: ^0.10.0 17 | - package: github.com/bakins/logrus-middleware 18 | - package: github.com/boltdb/bolt 19 | version: ^1.2.1 20 | - package: github.com/mitchellh/cli 21 | - package: github.com/rande/goapp 22 | - package: github.com/rande/gonode 23 | subpackages: 24 | - core/vault 25 | - package: goji.io 26 | version: ^1.1.0 27 | subpackages: 28 | - pat 29 | - pattern 30 | - package: golang.org/x/net 31 | subpackages: 32 | - context 33 | - package: github.com/AaronO/go-git-http 34 | version: fix_remaining_git_process 35 | repo: https://github.com/rande/go-git-http.git 36 | vcs: git 37 | - package: github.com/stretchr/testify 38 | version: ^1.1.3 39 | subpackages: 40 | - assert 41 | -------------------------------------------------------------------------------- /gui/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015", "react", "stage-0"] 3 | } -------------------------------------------------------------------------------- /gui/.prettierrc: -------------------------------------------------------------------------------- 1 | trailingComma: "es5" 2 | tabWidth: 4 3 | semi: false 4 | singleQuote: true -------------------------------------------------------------------------------- /gui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pkgmirror", 3 | "version": "0.0.1", 4 | "description": "pkgmirror", 5 | "scripts": { 6 | "start": "webpack-dev-server --config webpack-dev-server.config.js --progress --inline --colors", 7 | "build": "webpack --config webpack-production.config.js --progress --colors" 8 | }, 9 | "private": true, 10 | "dependencies": { 11 | "material-ui": "0.20.2", 12 | "prettier": "^1.18.2", 13 | "react": "15.3.2", 14 | "react-dom": "15.3.2", 15 | "react-markdown": "2.4.2", 16 | "react-redux": "4.4.5", 17 | "react-router": "2.8.1", 18 | "react-router-redux": "4.0.6", 19 | "react-tap-event-plugin": "1.0.0", 20 | "redux": "3.6.0" 21 | }, 22 | "devDependencies": { 23 | "babel-cli": "6.26.0", 24 | "babel-core": "6.26.3", 25 | "babel-loader": "8.0.6", 26 | "babel-preset-es2015": "6.24.1", 27 | "babel-preset-react": "6.24.1", 28 | "babel-preset-stage-0": "6.24.1", 29 | "json-loader": "0.5.7", 30 | "parcel-bundler": "^1.12.3", 31 | "react-hot-loader": "4.12.7", 32 | "react-transmit": "3.2.0", 33 | "whatwg-fetch": "3.0.0" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /gui/src/Main.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import React from 'react' 7 | import MuiThemeProvider from 'material-ui/styles/MuiThemeProvider' 8 | import AppBar from 'material-ui/AppBar' 9 | import Drawer from 'material-ui/Drawer' 10 | import { SMALL } from 'material-ui/utils/withWidth' 11 | import { connect } from 'react-redux' 12 | import { Router, Route, IndexRoute } from 'react-router' 13 | import { push } from 'react-router-redux' 14 | 15 | import { MirrorList, MenuList, CardMirror } from './redux/containers' 16 | import { toggleDrawer, hideDrawer } from './redux/apps/guiApp' 17 | import About from './components/About' 18 | 19 | const Container = props =>
{props.children}
20 | 21 | Container.propTypes = { 22 | children: React.PropTypes.any, 23 | } 24 | 25 | const Main = props => { 26 | let DrawerOpen = false 27 | let marginLeft = 0 28 | if (props.width === SMALL && props.DrawerOpen) { 29 | DrawerOpen = true 30 | } 31 | 32 | if (props.width !== SMALL) { 33 | DrawerOpen = true 34 | marginLeft = 300 35 | } 36 | 37 | return ( 38 | 39 |
40 | 46 | 47 | 48 | 53 | 54 | 55 | 56 | 57 |
61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 |
69 |
70 | 71 | ) 72 | } 73 | 74 | Main.propTypes = { 75 | Theme: React.PropTypes.object, 76 | Title: React.PropTypes.string, 77 | DrawerOpen: React.PropTypes.bool, 78 | toggleDrawer: React.PropTypes.func, 79 | history: React.PropTypes.object, 80 | width: React.PropTypes.number.isRequired, 81 | } 82 | 83 | const mapStateToProps = state => ({ ...state.guiApp }) 84 | 85 | const mapDispatchToProps = dispatch => ({ 86 | toggleDrawer: () => { 87 | dispatch(toggleDrawer()) 88 | }, 89 | }) 90 | 91 | export default connect( 92 | mapStateToProps, 93 | mapDispatchToProps 94 | )(Main) 95 | -------------------------------------------------------------------------------- /gui/src/app.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import React from 'react' 7 | import { render } from 'react-dom' 8 | import { createStore, combineReducers, applyMiddleware } from 'redux' 9 | import { Provider } from 'react-redux' 10 | import { 11 | syncHistoryWithStore, 12 | routerReducer, 13 | routerMiddleware, 14 | } from 'react-router-redux' 15 | 16 | import injectTapEventPlugin from 'react-tap-event-plugin' 17 | 18 | import { hashHistory } from 'react-router' 19 | 20 | import Main from './Main' // Our custom react component 21 | import { mirrorApp, addList, updateState } from './redux/apps/mirrorApp' 22 | import { guiApp, resizeApp } from './redux/apps/guiApp' 23 | 24 | // Needed for onTouchTap 25 | // http://stackoverflow.com/a/34015469/988941 26 | injectTapEventPlugin() 27 | 28 | const middleware = routerMiddleware(hashHistory) 29 | const reducers = combineReducers({ 30 | mirrorApp, 31 | guiApp, 32 | routing: routerReducer, 33 | }) 34 | 35 | const store = createStore(reducers, applyMiddleware(middleware)) 36 | 37 | if (window) { 38 | let deferTimer 39 | window.addEventListener('resize', () => { 40 | clearTimeout(deferTimer) 41 | deferTimer = setTimeout(() => { 42 | store.dispatch(resizeApp(window.innerWidth)) 43 | }, 200) 44 | }) 45 | 46 | // init the state 47 | store.dispatch(resizeApp(window.innerWidth)) 48 | } 49 | 50 | const history = syncHistoryWithStore(hashHistory, store) 51 | // history.listen(location => { 52 | // // console.log("Render Main", location); 53 | // }); 54 | 55 | fetch('/api/mirrors').then(res => { 56 | res.json().then(data => { 57 | store.dispatch(addList(data)) 58 | }) 59 | }) 60 | 61 | const ev = new EventSource('/api/sse') 62 | ev.onmessage = em => { 63 | try { 64 | const data = JSON.parse(em.data) 65 | store.dispatch(updateState(data)) 66 | } catch (e) { 67 | console.error(e) 68 | } 69 | } 70 | 71 | render( 72 | 73 |
74 | , 75 | document.getElementById('app') 76 | ) 77 | -------------------------------------------------------------------------------- /gui/src/components/About.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import React from 'react' 7 | 8 | const About = () => ( 9 |
10 |

PkgMirror

11 |

12 | This project has been created by{' '} 13 | Thomas Rabaix to avoid 14 | downtime while working with remote dependencies.
15 |
16 | The backend is coded using the Golang programming language for 17 | syncing the different sources. On the frontend, ReactJS is used with 18 | Material-UI.
19 |
20 | Feel free to contribute on:{' '} 21 | 22 | https://github.com/rande/pkgmirror 23 | 24 | . 25 |

26 |

Licence

27 | Copyright (c) 2016 Thomas Rabaix
28 |
29 | Permission is hereby granted, free of charge, to any person obtaining a 30 | copy of this software and associated documentation files (the 31 | "Software"), to deal in the Software without restriction, including 32 | without limitation the rights to use, copy, modify, merge, publish, 33 | distribute, sublicense, and/or sell copies of the Software, and to 34 | permit persons to whom the Software is furnished to do so, subject to 35 | the following conditions:
36 |
37 | The above copyright notice and this permission notice shall be included 38 | in all copies or substantial portions of the Software. 39 |
40 |
41 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 42 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 43 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 44 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 45 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 46 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 47 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 48 |
49 | ) 50 | 51 | export default About 52 | -------------------------------------------------------------------------------- /gui/src/components/CardMirror.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import React from 'react' 7 | 8 | import { Card, CardHeader, CardText } from 'material-ui/Card' 9 | import Markdown from 'react-markdown' 10 | import Avatar from 'material-ui/Avatar' 11 | 12 | const CardMirror = props => ( 13 | 14 | 22 | } 23 | /> 24 | 25 | 26 | 27 | 28 | ) 29 | 30 | CardMirror.propTypes = { 31 | mirror: React.PropTypes.object, 32 | } 33 | 34 | export default CardMirror 35 | -------------------------------------------------------------------------------- /gui/src/components/MenuList.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import React from 'react' 7 | import Avatar from 'material-ui/Avatar' 8 | import { List, ListItem } from 'material-ui/List' 9 | import Dashboard from 'material-ui/svg-icons/action/dashboard' 10 | import Info from 'material-ui/svg-icons/action/info' 11 | 12 | const MenuList = props => { 13 | const mirrorsItems = props.mirrors.map((mirror, pos) => ( 14 | 19 | } 20 | onTouchTap={() => { 21 | props.onTouchStart(mirror) 22 | }} 23 | insetChildren={false} 24 | /> 25 | )) 26 | 27 | const items = [ 28 | 36 | } 37 | onTouchTap={() => { 38 | props.homepage() 39 | }} 40 | />, 41 | 42 | ...mirrorsItems, 43 | 44 | 49 | } 50 | onTouchTap={() => { 51 | props.about() 52 | }} 53 | />, 54 | ] 55 | 56 | return {items} 57 | } 58 | 59 | MenuList.propTypes = { 60 | mirrors: React.PropTypes.array, 61 | onTouchStart: React.PropTypes.func, 62 | homepage: React.PropTypes.func, 63 | about: React.PropTypes.func, 64 | } 65 | 66 | export default MenuList 67 | -------------------------------------------------------------------------------- /gui/src/components/MirrorList.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import React from 'react' 7 | import { GridList, GridTile } from 'material-ui/GridList' 8 | import { MEDIUM, LARGE } from 'material-ui/utils/withWidth' 9 | 10 | import ActionInfo from 'material-ui/svg-icons/action/info' 11 | import ContentBlock from 'material-ui/svg-icons/content/block' 12 | import PlayCircle from 'material-ui/svg-icons/av/loop' 13 | import PauseCircle from 'material-ui/svg-icons/av/pause-circle-filled' 14 | 15 | const MirrorList = props => { 16 | const { mirrors, events } = props 17 | 18 | let cols = 2, 19 | cellHeight = 150 20 | 21 | if (props.width === MEDIUM) { 22 | cols = 2 23 | cellHeight = 200 24 | } 25 | 26 | if (props.width === LARGE) { 27 | cols = 4 28 | cellHeight = 300 29 | } 30 | 31 | return ( 32 | 33 | {mirrors.map((mirror, pos) => { 34 | let rightIcon = 35 | 36 | if (!mirror.Enabled) { 37 | rightIcon = 38 | } 39 | 40 | let text = mirror.Type 41 | 42 | if (mirror.Id in events) { 43 | if (events[mirror.Id].Status === 1) { 44 | rightIcon = 45 | } else { 46 | if (events[mirror.Id].Status === 2) { 47 | rightIcon = 48 | } 49 | } 50 | 51 | text += ` - ${events[mirror.Id].Message}` 52 | } 53 | 54 | return ( 55 | { 63 | props.onTouchStart(mirror) 64 | }} 65 | > 66 | 67 | 68 | ) 69 | })} 70 | 71 | ) 72 | } 73 | 74 | MirrorList.propTypes = { 75 | mirrors: React.PropTypes.array, 76 | events: React.PropTypes.object, 77 | onTouchStart: React.PropTypes.func, 78 | width: React.PropTypes.integer, 79 | } 80 | 81 | export default MirrorList 82 | -------------------------------------------------------------------------------- /gui/src/components/index.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import MirrorList from './MirrorList' 7 | import MenuList from './MenuList' 8 | import CardMirror from './CardMirror' 9 | 10 | export { MirrorList, MenuList, CardMirror } 11 | -------------------------------------------------------------------------------- /gui/src/redux/apps/guiApp.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import getMuiTheme from 'material-ui/styles/getMuiTheme' 7 | import { deepOrange500 } from 'material-ui/styles/colors' 8 | import { SMALL, MEDIUM, LARGE } from 'material-ui/utils/withWidth' 9 | 10 | export const GUI_TOGGLE_DRAWER = 'GUI_TOGGLE_DRAWER' 11 | export const GUI_HIDE_DRAWER = 'GUI_HIDE_DRAWER' 12 | export const GUI_RESIZE_WINDOW = 'GUI_RESIZE_WINDOW' 13 | 14 | export const toggleDrawer = () => ({ 15 | type: GUI_TOGGLE_DRAWER, 16 | }) 17 | 18 | export const hideDrawer = () => ({ 19 | type: GUI_HIDE_DRAWER, 20 | }) 21 | 22 | export const resizeApp = innerWidth => ({ 23 | type: GUI_RESIZE_WINDOW, 24 | innerWidth, 25 | }) 26 | 27 | const defaultState = { 28 | DrawerOpen: false, 29 | Title: 'PkgMirror', 30 | Theme: getMuiTheme({ 31 | palette: { 32 | accent1Color: deepOrange500, 33 | }, 34 | }), 35 | width: SMALL, 36 | } 37 | 38 | export function guiApp(state = defaultState, action) { 39 | switch (action.type) { 40 | case GUI_TOGGLE_DRAWER: 41 | return { ...state, DrawerOpen: !state.DrawerOpen } 42 | 43 | case GUI_HIDE_DRAWER: 44 | return { ...state, DrawerOpen: false } 45 | 46 | case GUI_RESIZE_WINDOW: 47 | const largeWidth = 992 48 | const mediumWidth = 768 49 | 50 | let width 51 | 52 | if (action.innerWidth >= largeWidth) { 53 | width = LARGE 54 | } else if (action.innerWidth >= mediumWidth) { 55 | width = MEDIUM 56 | } else { 57 | // innerWidth < 768 58 | width = SMALL 59 | } 60 | 61 | return { ...state, width } 62 | default: 63 | return state 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /gui/src/redux/apps/mirrorApp.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | export const MIRROR_ADD_LIST = 'MIRROR_ADD_LIST' 7 | export const MIRROR_UPDATE_STATE = 'MIRROR_UPDATE_STATE' 8 | 9 | export const updateState = mirror => ({ 10 | type: MIRROR_UPDATE_STATE, 11 | mirror, 12 | }) 13 | 14 | export const addList = list => ({ 15 | type: MIRROR_ADD_LIST, 16 | list, 17 | }) 18 | 19 | const defaultState = { 20 | mirrors: [ 21 | { 22 | Id: 'fake.id', 23 | Type: 'redux', 24 | Name: 'github', 25 | SourceUrl: 'http://redux.js.org', 26 | TargetUrl: 'http://redux.js.org', 27 | Icon: 'http://freeiconbox.com/icon/256/34429.png', 28 | Enabled: true, 29 | }, 30 | ], 31 | events: {}, 32 | } 33 | 34 | export function mirrorApp(state = defaultState, action) { 35 | switch (action.type) { 36 | case MIRROR_ADD_LIST: 37 | return { ...state, mirrors: action.list } 38 | 39 | case MIRROR_UPDATE_STATE: { 40 | const s = { ...state } 41 | s.events = { ...state.events } 42 | s.events[action.mirror.Id] = action.mirror 43 | 44 | return s 45 | } 46 | 47 | default: 48 | return state 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gui/src/redux/containers/CardMirror.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import { connect } from 'react-redux' 7 | 8 | import CardMirror from '../../components/CardMirror' 9 | 10 | const mapStateToProps = (state, ownProps) => { 11 | let mirror = {} 12 | 13 | state.mirrorApp.mirrors.every(v => { 14 | if (ownProps.params.id === v.Id) { 15 | mirror = v 16 | 17 | return false 18 | } 19 | 20 | return true 21 | }) 22 | 23 | return { 24 | mirror, 25 | } 26 | } 27 | 28 | export default connect(mapStateToProps)(CardMirror) 29 | -------------------------------------------------------------------------------- /gui/src/redux/containers/MenuList.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import { connect } from 'react-redux' 7 | import { push } from 'react-router-redux' 8 | 9 | import MenuList from '../../components/MenuList' 10 | import { hideDrawer } from '../apps/guiApp' 11 | 12 | const mapStateToProps = state => ({ 13 | mirrors: state.mirrorApp.mirrors, 14 | }) 15 | 16 | const mapDispatchToProps = dispatch => ({ 17 | homepage: () => { 18 | dispatch(push('/')) 19 | dispatch(hideDrawer()) 20 | }, 21 | about: () => { 22 | dispatch(push('/about')) 23 | dispatch(hideDrawer()) 24 | }, 25 | onTouchStart: mirror => { 26 | dispatch(push(`/mirror/${mirror.Id}`)) 27 | dispatch(hideDrawer()) 28 | }, 29 | }) 30 | 31 | export default connect( 32 | mapStateToProps, 33 | mapDispatchToProps 34 | )(MenuList) 35 | -------------------------------------------------------------------------------- /gui/src/redux/containers/MirrorList.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import { connect } from 'react-redux' 7 | import { push } from 'react-router-redux' 8 | 9 | import MirrorList from '../../components/MirrorList' 10 | import { hideDrawer } from '../apps/guiApp' 11 | 12 | const mapStateToProps = state => ({ 13 | mirrors: state.mirrorApp.mirrors, 14 | events: state.mirrorApp.events, 15 | width: state.guiApp.width, 16 | }) 17 | 18 | const mapDispatchToProps = dispatch => ({ 19 | onTouchStart: mirror => { 20 | dispatch(push(`/mirror/${mirror.Id}`)) 21 | dispatch(hideDrawer()) 22 | }, 23 | homepage: () => { 24 | dispatch(push('/')) 25 | }, 26 | }) 27 | 28 | export default connect( 29 | mapStateToProps, 30 | mapDispatchToProps 31 | )(MirrorList) 32 | -------------------------------------------------------------------------------- /gui/src/redux/containers/index.js: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | import MirrorList from './MirrorList' 7 | import MenuList from './MenuList' 8 | import CardMirror from './CardMirror' 9 | 10 | export { MirrorList, MenuList, CardMirror } 11 | -------------------------------------------------------------------------------- /gui/src/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | PkgMirror - Status Page 8 | 9 | 10 | 11 | 15 | 16 | 17 | 18 | 19 |
20 | 21 | 22 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /gui/src/static/main.css: -------------------------------------------------------------------------------- 1 | html { 2 | font-family: 'Roboto', sans-serif; 3 | } 4 | 5 | body { 6 | font-size: 13px; 7 | line-height: 20px; 8 | margin: 0px; 9 | } 10 | 11 | pre { 12 | white-space: pre-wrap; /* Since CSS 2.1 */ 13 | white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ 14 | white-space: -pre-wrap; /* Opera 4-6 */ 15 | white-space: -o-pre-wrap; /* Opera 7 */ 16 | word-wrap: break-word; /* Internet Explorer 5.5+ */ 17 | } -------------------------------------------------------------------------------- /mirror/bower/bower.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package bower 7 | 8 | import ( 9 | "encoding/json" 10 | "fmt" 11 | "io" 12 | "time" 13 | 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/boltdb/bolt" 16 | "github.com/rande/goapp" 17 | "github.com/rande/pkgmirror" 18 | "github.com/rande/pkgmirror/mirror/git" 19 | ) 20 | 21 | type BowerConfig struct { 22 | SourceServer string 23 | PublicServer string 24 | Code []byte 25 | Path string 26 | } 27 | 28 | func NewBowerService() *BowerService { 29 | return &BowerService{ 30 | Config: &BowerConfig{ 31 | SourceServer: "https://registry.bower.io", 32 | Code: []byte("bower"), 33 | Path: "./data/bower", 34 | }, 35 | } 36 | } 37 | 38 | type BowerService struct { 39 | DB *bolt.DB 40 | Config *BowerConfig 41 | Logger *log.Entry 42 | lock bool 43 | StateChan chan pkgmirror.State 44 | BoltCompacter *pkgmirror.BoltCompacter 45 | } 46 | 47 | func (bs *BowerService) Init(app *goapp.App) (err error) { 48 | bs.Logger.Info("Init") 49 | 50 | return bs.openDatabase() 51 | } 52 | 53 | func (bs *BowerService) openDatabase() (err error) { 54 | if bs.DB, err = pkgmirror.OpenDatabaseWithBucket(bs.Config.Path, bs.Config.Code); err != nil { 55 | bs.Logger.WithFields(log.Fields{ 56 | log.ErrorKey: err, 57 | "path": bs.Config.Path, 58 | "bucket": string(bs.Config.Code), 59 | "action": "Init", 60 | }).Error("Unable to open the internal database") 61 | 62 | return err 63 | } 64 | 65 | return nil 66 | } 67 | 68 | func (bs *BowerService) optimize() error { 69 | bs.lock = true 70 | 71 | path := bs.DB.Path() 72 | 73 | bs.DB.Close() 74 | 75 | if err := bs.BoltCompacter.Compact(path); err != nil { 76 | return err 77 | } 78 | 79 | err := bs.openDatabase() 80 | 81 | bs.lock = false 82 | 83 | return err 84 | } 85 | 86 | func (bs *BowerService) Serve(state *goapp.GoroutineState) error { 87 | bs.Logger.Info("Starting Bower Service") 88 | 89 | syncEnd := make(chan bool) 90 | 91 | iteration := 0 92 | sync := func() { 93 | bs.Logger.Info("Starting a new sync...") 94 | 95 | bs.SyncPackages() 96 | 97 | iteration++ 98 | 99 | // optimize every 10 iteration 100 | if iteration > 9 { 101 | bs.Logger.Info("Starting database optimization") 102 | bs.optimize() 103 | iteration = 0 104 | } 105 | 106 | syncEnd <- true 107 | } 108 | 109 | // start the first sync 110 | go sync() 111 | 112 | for { 113 | select { 114 | case <-state.In: 115 | bs.DB.Close() 116 | return nil 117 | 118 | case <-syncEnd: 119 | bs.StateChan <- pkgmirror.State{ 120 | Message: "Wait for a new run", 121 | Status: pkgmirror.STATUS_HOLD, 122 | } 123 | 124 | bs.Logger.Info("Wait before starting a new sync...") 125 | 126 | // we recursively call sync unless a state.In comes in to exist the current 127 | // go routine (ie, the Serve function). This might not close the sync processus 128 | // completely. We need to have a proper channel (queue mode) for git fetch. 129 | // This will probably make this current code obsolete. 130 | go func() { 131 | time.Sleep(60 * 15 * time.Second) 132 | sync() 133 | }() 134 | } 135 | } 136 | } 137 | 138 | func (bs *BowerService) SyncPackages() error { 139 | logger := bs.Logger.WithFields(log.Fields{ 140 | "action": "SyncPackages", 141 | }) 142 | 143 | logger.Info("Starting SyncPackages") 144 | 145 | bs.StateChan <- pkgmirror.State{ 146 | Message: "Syncing packages", 147 | Status: pkgmirror.STATUS_RUNNING, 148 | } 149 | 150 | pkgs := make(Packages, 0) 151 | 152 | logger.Info("Loading bower packages") 153 | 154 | bs.StateChan <- pkgmirror.State{ 155 | Message: "Loading packages list", 156 | Status: pkgmirror.STATUS_RUNNING, 157 | } 158 | 159 | if err := pkgmirror.LoadRemoteStruct(fmt.Sprintf("%s/packages", bs.Config.SourceServer), &pkgs); err != nil { 160 | logger.WithFields(log.Fields{ 161 | "path": "packages", 162 | log.ErrorKey: err.Error(), 163 | }).Error("Error loading bower packages list") 164 | 165 | return err // an error occurs avoid empty file 166 | } 167 | 168 | logger.Info("End loading packages information!") 169 | 170 | for _, pkg := range pkgs { 171 | bs.DB.Update(func(tx *bolt.Tx) error { 172 | b := tx.Bucket(bs.Config.Code) 173 | 174 | logger := bs.Logger.WithFields(log.Fields{ 175 | "package": pkg.Name, 176 | }) 177 | 178 | saved := &Package{} 179 | data := b.Get([]byte(pkg.Name)) 180 | 181 | if len(data) > 0 { 182 | if err := json.Unmarshal(data, saved); err != nil { 183 | logger.WithError(err).Info("Error while unmarshaling current package") 184 | } else { 185 | if saved.SourceUrl == pkg.Url { 186 | logger.Debug("Skip package!") 187 | 188 | return nil // same package no change, avoid io 189 | } 190 | } 191 | } 192 | 193 | bs.StateChan <- pkgmirror.State{ 194 | Message: fmt.Sprintf("Save package information: %s", pkg.Name), 195 | Status: pkgmirror.STATUS_RUNNING, 196 | } 197 | 198 | pkg.SourceUrl = pkg.Url 199 | pkg.Url = git.GitRewriteRepository(bs.Config.PublicServer, pkg.Url) 200 | 201 | data, _ = json.Marshal(pkg) 202 | 203 | // store the path 204 | if err := b.Put([]byte(pkg.Name), data); err != nil { 205 | logger.WithError(err).Error("Error updating/creating definition") 206 | 207 | return err 208 | } 209 | 210 | logger.Info("Package saved!") 211 | 212 | return nil 213 | }) 214 | } 215 | 216 | bs.StateChan <- pkgmirror.State{ 217 | Message: "End package synchronisation", 218 | Status: pkgmirror.STATUS_HOLD, 219 | } 220 | 221 | return nil 222 | } 223 | 224 | func (bs *BowerService) Get(name string) ([]byte, error) { 225 | var data []byte 226 | 227 | bs.Logger.WithFields(log.Fields{ 228 | "action": "Get", 229 | "key": name, 230 | }).Info("Get package data") 231 | 232 | err := bs.DB.View(func(tx *bolt.Tx) error { 233 | b := tx.Bucket(bs.Config.Code) 234 | 235 | raw := b.Get([]byte(name)) 236 | 237 | if len(raw) == 0 { 238 | return pkgmirror.EmptyKeyError 239 | } 240 | 241 | data = make([]byte, len(raw)) 242 | 243 | copy(data, raw) 244 | 245 | return nil 246 | }) 247 | 248 | return data, err 249 | } 250 | 251 | func (bs *BowerService) WriteList(w io.Writer) error { 252 | err := bs.DB.View(func(tx *bolt.Tx) error { 253 | b := tx.Bucket(bs.Config.Code) 254 | 255 | c := b.Cursor() 256 | 257 | k, v := c.First() 258 | 259 | w.Write([]byte{'['}) 260 | 261 | if k == nil { 262 | w.Write([]byte{']'}) 263 | 264 | return nil 265 | } 266 | 267 | w.Write(v) 268 | 269 | for k, v := c.Next(); k != nil; k, v = c.Next() { 270 | w.Write([]byte{','}) 271 | w.Write(v) 272 | } 273 | w.Write([]byte{']'}) 274 | 275 | return nil 276 | }) 277 | 278 | return err 279 | } 280 | -------------------------------------------------------------------------------- /mirror/bower/bower_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package bower 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | 12 | log "github.com/Sirupsen/logrus" 13 | "github.com/rande/goapp" 14 | "github.com/rande/pkgmirror" 15 | "goji.io" 16 | "goji.io/pat" 17 | "golang.org/x/net/context" 18 | ) 19 | 20 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 21 | 22 | l.Register(func(app *goapp.App) error { 23 | logger := app.Get("logger").(*log.Logger) 24 | 25 | for name, conf := range config.Bower { 26 | if !conf.Enabled { 27 | continue 28 | } 29 | 30 | app.Set(fmt.Sprintf("pkgmirror.bower.%s", name), func(name string, conf *pkgmirror.BowerConfig) func(app *goapp.App) interface{} { 31 | return func(app *goapp.App) interface{} { 32 | s := NewBowerService() 33 | s.Config.Code = []byte(name) 34 | s.Config.Path = fmt.Sprintf("%s/bower", config.DataDir) 35 | s.Config.PublicServer = config.PublicServer 36 | s.Config.SourceServer = conf.Server 37 | s.Logger = logger.WithFields(log.Fields{ 38 | "handler": "bower", 39 | "server": s.Config.SourceServer, 40 | "code": name, 41 | }) 42 | s.StateChan = pkgmirror.GetStateChannel(fmt.Sprintf("pkgmirror.bower.%s", name), app.Get("pkgmirror.channel.state").(chan pkgmirror.State)) 43 | s.BoltCompacter = app.Get("bolt.compacter").(*pkgmirror.BoltCompacter) 44 | 45 | if err := s.Init(app); err != nil { 46 | panic(err) 47 | } 48 | 49 | return s 50 | } 51 | }(name, conf)) 52 | } 53 | 54 | return nil 55 | }) 56 | 57 | l.Prepare(func(app *goapp.App) error { 58 | for name, conf := range config.Bower { 59 | if !conf.Enabled { 60 | continue 61 | } 62 | 63 | ConfigureHttp(name, conf, app) 64 | } 65 | 66 | return nil 67 | }) 68 | 69 | for name, conf := range config.Bower { 70 | if !conf.Enabled { 71 | continue 72 | } 73 | 74 | l.Run(func(name string) func(app *goapp.App, state *goapp.GoroutineState) error { 75 | return func(app *goapp.App, state *goapp.GoroutineState) error { 76 | s := app.Get(fmt.Sprintf("pkgmirror.bower.%s", name)).(pkgmirror.MirrorService) 77 | s.Serve(state) 78 | 79 | return nil 80 | } 81 | }(name)) 82 | } 83 | } 84 | 85 | func ConfigureHttp(name string, conf *pkgmirror.BowerConfig, app *goapp.App) { 86 | mux := app.Get("mux").(*goji.Mux) 87 | bowerService := app.Get(fmt.Sprintf("pkgmirror.bower.%s", name)).(*BowerService) 88 | 89 | mux.HandleFuncC(pat.Get(fmt.Sprintf("/bower/%s/packages", name)), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 90 | w.Header().Set("Content-Type", "application/json") 91 | if err := bowerService.WriteList(w); err != nil { 92 | pkgmirror.SendWithHttpCode(w, 500, err.Error()) 93 | } 94 | }) 95 | 96 | mux.HandleFuncC(pat.Get(fmt.Sprintf("/bower/%s/packages/:name", name)), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 97 | if data, err := bowerService.Get(fmt.Sprintf("%s", pat.Param(ctx, "name"))); err != nil { 98 | pkgmirror.SendWithHttpCode(w, 404, err.Error()) 99 | } else { 100 | w.Header().Set("Content-Type", "application/json") 101 | w.Write(data) 102 | } 103 | }) 104 | } 105 | -------------------------------------------------------------------------------- /mirror/bower/bower_struct.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package bower 7 | 8 | type Package struct { 9 | Name string `json:"name"` 10 | Url string `json:"url"` 11 | SourceUrl string `json:"source_url"` 12 | } 13 | 14 | type Packages []*Package 15 | -------------------------------------------------------------------------------- /mirror/composer/composer_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package composer 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | "net/url" 12 | "strings" 13 | 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/rande/goapp" 16 | "github.com/rande/pkgmirror" 17 | "goji.io" 18 | "goji.io/pat" 19 | "golang.org/x/net/context" 20 | ) 21 | 22 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 23 | 24 | l.Register(func(app *goapp.App) error { 25 | 26 | logger := app.Get("logger").(*log.Logger) 27 | 28 | for name, conf := range config.Composer { 29 | if !conf.Enabled { 30 | continue 31 | } 32 | 33 | app.Set(fmt.Sprintf("pkgmirror.composer.%s", name), func(name string, conf *pkgmirror.ComposerConfig) func(app *goapp.App) interface{} { 34 | return func(app *goapp.App) interface{} { 35 | 36 | s := NewComposerService() 37 | 38 | if u, err := url.Parse(conf.Server); err != nil { 39 | panic(err) 40 | } else { 41 | s.Config.BasePublicServer = fmt.Sprintf("%s://%s", u.Scheme, u.Host) 42 | } 43 | 44 | s.Config.Path = fmt.Sprintf("%s/composer", config.DataDir) 45 | s.Config.PublicServer = config.PublicServer 46 | s.Config.SourceServer = conf.Server 47 | 48 | s.Config.Code = []byte(name) 49 | s.Logger = logger.WithFields(log.Fields{ 50 | "handler": "composer", 51 | "server": s.Config.SourceServer, 52 | "code": name, 53 | }) 54 | s.StateChan = pkgmirror.GetStateChannel(fmt.Sprintf("pkgmirror.composer.%s", name), app.Get("pkgmirror.channel.state").(chan pkgmirror.State)) 55 | s.BoltCompacter = app.Get("bolt.compacter").(*pkgmirror.BoltCompacter) 56 | 57 | if err := s.Init(app); err != nil { 58 | panic(err) 59 | } 60 | 61 | return s 62 | } 63 | }(name, conf)) 64 | } 65 | 66 | return nil 67 | }) 68 | 69 | l.Prepare(func(app *goapp.App) error { 70 | // BC Compatible 71 | mux := app.Get("mux").(*goji.Mux) 72 | mux.HandleFuncC(pat.Get("/packagist/*"), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 73 | http.Redirect(w, r, "/composer"+r.URL.EscapedPath(), http.StatusMovedPermanently) 74 | }) 75 | 76 | for name, conf := range config.Composer { 77 | if !conf.Enabled { 78 | continue 79 | } 80 | 81 | ConfigureHttp(name, conf, app) 82 | } 83 | 84 | return nil 85 | }) 86 | 87 | for name, conf := range config.Composer { 88 | if !conf.Enabled { 89 | continue 90 | } 91 | 92 | l.Run(func(name string) func(app *goapp.App, state *goapp.GoroutineState) error { 93 | return func(app *goapp.App, state *goapp.GoroutineState) error { 94 | s := app.Get(fmt.Sprintf("pkgmirror.composer.%s", name)).(pkgmirror.MirrorService) 95 | s.Serve(state) 96 | 97 | return nil 98 | } 99 | }(name)) 100 | } 101 | } 102 | 103 | // http://localhost:8000/composer/drupal8/drupal/provider-2011-2%24e22123ab0815d43cedb1309f7ad7b803127ac9679f7aaa9b281cf768f6806ae2.json 104 | 105 | func ConfigureHttp(name string, conf *pkgmirror.ComposerConfig, app *goapp.App) { 106 | mux := app.Get("mux").(*goji.Mux) 107 | logger := app.Get("logger").(*log.Logger).WithFields(log.Fields{ 108 | "handler": "composer", 109 | "code": name, 110 | }) 111 | 112 | composerService := app.Get(fmt.Sprintf("pkgmirror.composer.%s", name)).(*ComposerService) 113 | 114 | mux.HandleFuncC(pat.Get(fmt.Sprintf("/composer/%s(/|)", name)), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 115 | http.Redirect(w, r, fmt.Sprintf("/composer/%s/packages.json", name), http.StatusMovedPermanently) 116 | }) 117 | 118 | mux.HandleFuncC(NewPackagePat(name), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 119 | pkg := fmt.Sprintf("%s/%s$%s", pat.Param(ctx, "vendor"), pat.Param(ctx, "package"), pat.Param(ctx, "ref")) 120 | 121 | if refresh := r.FormValue("refresh"); len(refresh) > 0 { 122 | w.Header().Set("Content-Type", "application/json") 123 | 124 | if err := composerService.UpdatePackage(pkg); err != nil { 125 | pkgmirror.SendWithHttpCode(w, 500, err.Error()) 126 | } else { 127 | pkgmirror.SendWithHttpCode(w, 200, "Package updated") 128 | } 129 | 130 | return 131 | } 132 | 133 | if data, err := composerService.Get(pkg); err != nil { 134 | pkgmirror.SendWithHttpCode(w, 404, err.Error()) 135 | } else { 136 | w.Header().Set("Content-Type", "application/json") 137 | w.Header().Set("Content-Encoding", "gzip") 138 | w.Write(data) 139 | } 140 | }) 141 | 142 | mux.HandleFuncC(NewPackageInfoPat(name), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 143 | if pi, err := composerService.GetPackage(fmt.Sprintf("%s/%s", pat.Param(ctx, "vendor"), pat.Param(ctx, "package"))); err != nil { 144 | pkgmirror.SendWithHttpCode(w, 404, err.Error()) 145 | } else { 146 | switch pat.Param(ctx, "format") { 147 | case "html": 148 | http.Redirect(w, r, fmt.Sprintf("/composer/%s/p/%s.json", name, pi.GetTargetKey()), http.StatusFound) 149 | } 150 | } 151 | }) 152 | 153 | baseUrlLen := len(fmt.Sprintf("/composer/%s/", name)) 154 | 155 | // catch all for this element (drupal element) 156 | mux.HandleFuncC(pat.Get(fmt.Sprintf("/composer/%s/*", name)), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 157 | logger.Debug(r.URL.Path) 158 | 159 | format := "html" 160 | url := r.URL.Path[baseUrlLen:] 161 | key := url 162 | hash := "" 163 | ref := "" 164 | 165 | if i := strings.Index(url, "."); i > 0 { 166 | format = url[i+1:] 167 | key = url[:i] 168 | } 169 | 170 | if i := strings.Index(key, "$"); i > 0 { 171 | hash = key[i+1:] 172 | ref = key[:i] 173 | } 174 | 175 | logger.WithFields(log.Fields{ 176 | "url": url, 177 | "format": format, 178 | "key": key, 179 | "ref": ref, 180 | "hash": hash, 181 | }).Debug(url) 182 | 183 | if data, err := composerService.Get(url); err != nil { 184 | pkgmirror.SendWithHttpCode(w, 404, err.Error()) 185 | } else { 186 | w.Header().Set("Content-Type", "application/json") 187 | w.Header().Set("Content-Encoding", "gzip") 188 | w.Write(data) 189 | } 190 | }) 191 | } 192 | -------------------------------------------------------------------------------- /mirror/composer/composer_pat.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package composer 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | "regexp" 12 | 13 | "goji.io" 14 | "goji.io/pattern" 15 | "golang.org/x/net/context" 16 | ) 17 | 18 | func NewPackagePat(code string) goji.Pattern { 19 | return &PackagePat{ 20 | Pattern: regexp.MustCompile(fmt.Sprintf(`\/composer\/%s\/p\/([^\/]*)\/([^\/]*)\$([^\/]*)\.json`, code)), 21 | } 22 | } 23 | 24 | type PackagePat struct { 25 | Pattern *regexp.Regexp 26 | } 27 | 28 | func (pp *PackagePat) Match(ctx context.Context, r *http.Request) context.Context { 29 | if results := pp.Pattern.FindStringSubmatch(r.URL.Path); len(results) == 0 { 30 | return nil 31 | } else { 32 | return &packagePatMatch{ctx, results[1], results[2], results[3], "json"} 33 | } 34 | } 35 | 36 | func NewPackageInfoPat(code string) goji.Pattern { 37 | return &PackageInfoPat{ 38 | Pattern: regexp.MustCompile(fmt.Sprintf(`\/composer\/%s\/p\/([^\/]*)\/([^\/]*)(.json|)`, code)), 39 | } 40 | } 41 | 42 | type PackageInfoPat struct { 43 | Pattern *regexp.Regexp 44 | } 45 | 46 | func (pp *PackageInfoPat) Match(ctx context.Context, r *http.Request) context.Context { 47 | if results := pp.Pattern.FindStringSubmatch(r.URL.Path); len(results) == 0 { 48 | return nil 49 | } else { 50 | format := "html" 51 | 52 | if len(results[3]) > 0 { 53 | format = results[3][1:] 54 | } 55 | 56 | return &packagePatMatch{ctx, results[1], results[2], "", format} 57 | } 58 | } 59 | 60 | type packagePatMatch struct { 61 | context.Context 62 | Vendor string 63 | Package string 64 | Ref string 65 | Format string 66 | } 67 | 68 | func (m packagePatMatch) Value(key interface{}) interface{} { 69 | 70 | switch key { 71 | case pattern.AllVariables: 72 | return map[pattern.Variable]string{ 73 | "vendor": m.Vendor, 74 | "package": m.Package, 75 | "ref": m.Ref, 76 | "format": m.Format, 77 | } 78 | case pattern.Variable("vendor"): 79 | return m.Vendor 80 | case pattern.Variable("package"): 81 | return m.Package 82 | case pattern.Variable("ref"): 83 | return m.Ref 84 | case pattern.Variable("format"): 85 | return m.Format 86 | } 87 | 88 | return m.Context.Value(key) 89 | } 90 | -------------------------------------------------------------------------------- /mirror/composer/composer_pat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package composer 7 | 8 | import ( 9 | "net/http" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "goji.io/pattern" 14 | "golang.org/x/net/context" 15 | ) 16 | 17 | func mustReq(method, path string) (context.Context, *http.Request) { 18 | req, err := http.NewRequest(method, path, nil) 19 | if err != nil { 20 | panic(err) 21 | } 22 | ctx := pattern.SetPath(context.Background(), req.URL.EscapedPath()) 23 | 24 | return ctx, req 25 | } 26 | 27 | func Test_Composer_Pat_Definition(t *testing.T) { 28 | p := NewPackagePat("packagist") 29 | 30 | c, r := mustReq("GET", "/composer/packagist/p/kevinlebrun/colors.php%24f8ef02dddbd0bb7f78a2775e7188415e128d7b147f2a5630784c75cfc46a1a7e.json") 31 | 32 | result := p.Match(c, r) 33 | 34 | assert.NotNil(t, result) 35 | assert.Equal(t, "kevinlebrun", result.Value(pattern.Variable("vendor"))) 36 | assert.Equal(t, "colors.php", result.Value(pattern.Variable("package"))) 37 | assert.Equal(t, "f8ef02dddbd0bb7f78a2775e7188415e128d7b147f2a5630784c75cfc46a1a7e", result.Value(pattern.Variable("ref"))) 38 | assert.Equal(t, "json", result.Value(pattern.Variable("format"))) 39 | } 40 | 41 | func Test_Composer_Pat_PackageInformation(t *testing.T) { 42 | p := NewPackageInfoPat("packagist") 43 | 44 | c, r := mustReq("GET", "/composer/packagist/p/kevinlebrun/colors.php") 45 | 46 | result := p.Match(c, r) 47 | 48 | assert.NotNil(t, result) 49 | assert.Equal(t, "kevinlebrun", result.Value(pattern.Variable("vendor"))) 50 | assert.Equal(t, "colors.php", result.Value(pattern.Variable("package"))) 51 | assert.Equal(t, "html", result.Value(pattern.Variable("format"))) 52 | } 53 | 54 | func Test_Composer_Pat_AllVariables(t *testing.T) { 55 | p := NewPackagePat("packagist") 56 | 57 | c, r := mustReq("GET", "/composer/packagist/p/kevinlebrun/colors.php%24f8ef02dddbd0bb7f78a2775e7188415e128d7b147f2a5630784c75cfc46a1a7e.json") 58 | 59 | result := p.Match(c, r) 60 | 61 | assert.NotNil(t, result) 62 | 63 | vars := result.Value(pattern.AllVariables).(map[pattern.Variable]string) 64 | 65 | assert.Equal(t, "kevinlebrun", vars["vendor"]) 66 | assert.Equal(t, "colors.php", vars["package"]) 67 | assert.Equal(t, "f8ef02dddbd0bb7f78a2775e7188415e128d7b147f2a5630784c75cfc46a1a7e", vars["ref"]) 68 | assert.Equal(t, "json", vars["format"]) 69 | } 70 | 71 | func Test_Composer_Pat_OtherVariable(t *testing.T) { 72 | p := NewPackagePat("packagist") 73 | 74 | c, r := mustReq("GET", "/composer/packagist/p/kevinlebrun/colors.php%24f8ef02dddbd0bb7f78a2775e7188415e128d7b147f2a5630784c75cfc46a1a7e.json") 75 | 76 | result := p.Match(c, r) 77 | 78 | assert.NotNil(t, result) 79 | 80 | assert.Nil(t, result.Value(pattern.Variable("foo"))) 81 | } 82 | -------------------------------------------------------------------------------- /mirror/composer/composer_structs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package composer 7 | 8 | import ( 9 | "encoding/json" 10 | "fmt" 11 | "time" 12 | ) 13 | 14 | type ProviderInclude map[string]struct { 15 | Sha256 string `json:"sha256"` 16 | } 17 | 18 | type PackagesResult struct { 19 | Packages json.RawMessage `json:"packages"` 20 | Notify string `json:"notify"` 21 | NotifyBatch string `json:"notify-batch"` 22 | ProvidersURL string `json:"providers-url"` 23 | Search string `json:"search"` 24 | ProviderIncludes ProviderInclude `json:"provider-includes"` 25 | } 26 | 27 | type ProvidersResult struct { 28 | Providers map[string]struct { 29 | Sha256 string `json:"sha256"` 30 | } `json:"providers"` 31 | Code string `json:"-"` 32 | } 33 | 34 | // package description 35 | type Package struct { 36 | Name string `json:"name,omitempty"` 37 | Abandoned *json.RawMessage `json:"abandoned,omitempty"` 38 | Description string `json:"description,omitempty"` 39 | Keywords *json.RawMessage `json:"keywords,omitempty"` 40 | Homepage string `json:"homepage,omitempty"` 41 | Version string `json:"version,omitempty"` 42 | VersionNormalized string `json:"version_normalized,omitempty"` 43 | License *json.RawMessage `json:"license,omitempty"` 44 | Bin []string `json:"bin,omitempty"` 45 | Authors []struct { 46 | Name string `json:"name"` 47 | Email string `json:"email"` 48 | Homepage string `json:"homepage"` 49 | Role string `json:"role"` 50 | } `json:"authors,omitempty"` 51 | Source struct { 52 | Type string `json:"type"` 53 | URL string `json:"url"` 54 | Reference string `json:"reference"` 55 | } `json:"source,omitempty"` 56 | Dist struct { 57 | Type string `json:"type"` 58 | URL string `json:"url"` 59 | Reference string `json:"reference"` 60 | Shasum string `json:"shasum"` 61 | } `json:"dist,omitempty"` 62 | Extra *json.RawMessage `json:"extra,omitempty"` 63 | TargetDir string `json:"target-dir,omitempty"` 64 | Type string `json:"type,omitempty"` 65 | Time time.Time `json:"time,omitempty"` 66 | Autoload *json.RawMessage `json:"autoload,omitempty"` 67 | Replace *json.RawMessage `json:"replace,omitempty"` 68 | Conflict *json.RawMessage `json:"conflict,omitempty"` 69 | Provide *json.RawMessage `json:"provide,omitempty"` 70 | Require *json.RawMessage `json:"require,omitempty"` 71 | RequireDev *json.RawMessage `json:"require-dev,omitempty"` 72 | Suggest *json.RawMessage `json:"suggest,omitempty"` 73 | UID *json.RawMessage `json:"uid,omitempty"` 74 | } 75 | 76 | // used to load the packages.json file 77 | type PackageResult struct { 78 | Packages map[string]map[string]*Package `json:"packages"` 79 | } 80 | 81 | type PackageInformation struct { 82 | Server string `json:"server"` 83 | PackageResult PackageResult `json:"-"` 84 | Package string `json:"package"` 85 | Exist bool `json:"-"` 86 | HashSource string `json:"hash_source"` 87 | HashTarget string `json:"hash_target"` 88 | Url string `json:"-"` 89 | } 90 | 91 | func (pi *PackageInformation) GetTargetKey() string { 92 | return fmt.Sprintf("%s$%s", pi.Package, pi.HashTarget) 93 | } 94 | -------------------------------------------------------------------------------- /mirror/composer/composer_structs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package composer 7 | 8 | import ( 9 | "testing" 10 | 11 | "github.com/rande/pkgmirror" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func LoadTestStruct(t *testing.T, file string, v interface{}) { 16 | err := pkgmirror.LoadStruct(file, v) 17 | 18 | assert.NoError(t, err) 19 | } 20 | 21 | func Test_Load_Packages(t *testing.T) { 22 | p := &PackagesResult{} 23 | 24 | LoadTestStruct(t, "../../fixtures/packagist/packages.json", p) 25 | 26 | assert.Equal(t, "/downloads/", p.NotifyBatch) 27 | 28 | assert.Equal(t, 9, len(p.ProviderIncludes)) 29 | } 30 | 31 | func Test_Load_Providers(t *testing.T) { 32 | p := &ProvidersResult{} 33 | 34 | LoadTestStruct(t, "../../fixtures/packagist/p/provider-2013$370af0b17d1ec5b0325bdb0126c9007b69647fafe5df8b5ecf79241e09745841.json", p) 35 | 36 | assert.Equal(t, 7585, len(p.Providers)) 37 | } 38 | 39 | func Test_Load_Package(t *testing.T) { 40 | p := &PackageResult{} 41 | 42 | LoadTestStruct(t, "../../fixtures/packagist/p/0n3s3c/baselibrary$3a3dbbc33805b6748f859e8f2c517355f42e2f6d4b71daad077794842dca280c.json", p) 43 | 44 | assert.Equal(t, 1, len(p.Packages)) 45 | assert.Equal(t, 2, len(p.Packages["0n3s3c/baselibrary"])) 46 | assert.Equal(t, "Library for working with objects in PHP", p.Packages["0n3s3c/baselibrary"]["0.5.0"].Description) 47 | } 48 | 49 | func Test_Load_Package_Project(t *testing.T) { 50 | p := &PackageResult{} 51 | 52 | LoadTestStruct(t, "../../fixtures/packagist/p/symfony/framework-standard-edition$cb64bc5278d2b6bbf7c02ae4b995f3698df1a210dceb509328b4370e13f3ba33.json", p) 53 | 54 | assert.Equal(t, 1, len(p.Packages)) 55 | assert.Equal(t, 158, len(p.Packages["symfony/framework-standard-edition"])) 56 | assert.Equal(t, "The \"Symfony Standard Edition\" distribution", p.Packages["symfony/framework-standard-edition"]["2.8.x-dev"].Description) 57 | } 58 | -------------------------------------------------------------------------------- /mirror/composer/composer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package composer 7 | -------------------------------------------------------------------------------- /mirror/git/git.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package git 7 | 8 | import ( 9 | "bytes" 10 | "fmt" 11 | "io" 12 | "os" 13 | "os/exec" 14 | "path/filepath" 15 | "regexp" 16 | "strings" 17 | "sync" 18 | "time" 19 | 20 | log "github.com/Sirupsen/logrus" 21 | "github.com/boltdb/bolt" 22 | "github.com/rande/goapp" 23 | "github.com/rande/gonode/core/vault" 24 | "github.com/rande/pkgmirror" 25 | ) 26 | 27 | var ( 28 | DRUPAL_ARCHIVE = regexp.MustCompile(`https:\/\/ftp.drupal.org\/files\/projects/(.*)\.zip`) 29 | BITBUCKET_ARCHIVE = regexp.MustCompile(`http(s|):\/\/([\w-\.]+)\/([\w\.\d-]+)\/([\w-\.\d]+)\/get\/([\w]+)\.zip`) 30 | GITHUB_ARCHIVE = regexp.MustCompile(`http(s|):\/\/api\.([\w-\.]+)\/repos\/([\w\.\d-]+)\/([\w\.\d-]+)\/zipball\/([\w]+)`) 31 | GITLAB_ARCHIVE = regexp.MustCompile(`http(s|):\/\/([\w-\.]+)\/([\w-\.\d]+)\/([\w-\.\d]+)\/repository\/archive.zip\?ref=([\w]+)`) 32 | 33 | GIT_REPOSITORY = regexp.MustCompile(`^(((git|http(s|)):\/\/|git@))([\w-\.]+@|)([\w-\.]+)(\/|:)([\w-\.\/]+?)(\.git|)$`) 34 | SVN_REPOSITORY = regexp.MustCompile(`(svn:\/\/(.*)|(.*)\.svn\.(.*))`) 35 | 36 | CACHEABLE_REF = regexp.MustCompile(`([\w\d]{40}|[\w\d]+\.[\w\d]+\.[\w\d]+(-[\w\d]+|))`) 37 | IS_REF = regexp.MustCompile(`^[a-zA-Z0-9\.\-]{1,40}$`) 38 | ) 39 | 40 | func NewGitService() *GitService { 41 | return &GitService{ 42 | Config: &GitConfig{ 43 | DataDir: "./data/git", 44 | Binary: "git", 45 | SourceServer: "git@github.com:%s", 46 | PublicServer: "http://localhost:8000", 47 | }, 48 | Vault: &vault.Vault{ 49 | Algo: "no_op", 50 | Driver: &vault.DriverFs{ 51 | Root: "./cache/git", 52 | }, 53 | }, 54 | } 55 | } 56 | 57 | type GitConfig struct { 58 | PublicServer string 59 | SourceServer string 60 | Server string 61 | DataDir string 62 | Binary string 63 | Clone string 64 | } 65 | 66 | type GitService struct { 67 | DB *bolt.DB 68 | Config *GitConfig 69 | Logger *log.Entry 70 | Vault *vault.Vault 71 | StateChan chan pkgmirror.State 72 | } 73 | 74 | func (gs *GitService) Init(app *goapp.App) error { 75 | os.MkdirAll(string(filepath.Separator)+gs.Config.DataDir, 0755) 76 | 77 | return nil 78 | } 79 | 80 | func (gs *GitService) Serve(state *goapp.GoroutineState) error { 81 | syncEnd := make(chan bool) 82 | 83 | sync := func() { 84 | gs.Logger.Info("Starting a new sync...") 85 | 86 | gs.syncRepositories() 87 | 88 | syncEnd <- true 89 | } 90 | 91 | // start the first sync 92 | go sync() 93 | 94 | for { 95 | select { 96 | case <-state.In: 97 | return nil 98 | 99 | case <-syncEnd: 100 | gs.StateChan <- pkgmirror.State{ 101 | Message: "Wait for a new run", 102 | Status: pkgmirror.STATUS_HOLD, 103 | } 104 | 105 | gs.Logger.Info("Wait before starting a new sync...") 106 | 107 | // we recursively call sync unless a state.In comes in to exist the current 108 | // go routine (ie, the Serve function). This might not close the sync processus 109 | // completely. We need to have a proper channel (queue mode) for git fetch. 110 | // This will probably make this current code obsolete. 111 | go func() { 112 | time.Sleep(60 * time.Second) 113 | sync() 114 | }() 115 | } 116 | } 117 | } 118 | 119 | func (gs *GitService) syncRepositories() { 120 | service := fmt.Sprintf("%s/%s", gs.Config.DataDir, gs.Config.Server) 121 | 122 | gs.Logger.WithFields(log.Fields{ 123 | "action": "SyncRepositories", 124 | "datadir": service, 125 | }).Info("Sync service's repositories") 126 | 127 | searchPaths := []string{ 128 | fmt.Sprintf("%s/*.git", service), 129 | fmt.Sprintf("%s/*/*.git", service), 130 | fmt.Sprintf("%s/*/*/*.git", service), 131 | } 132 | 133 | paths := []string{} 134 | for _, searchPath := range searchPaths { 135 | if p, err := filepath.Glob(searchPath); err != nil { 136 | continue 137 | } else { 138 | paths = append(paths, p...) 139 | } 140 | } 141 | 142 | for _, path := range paths { 143 | // remove the base path 144 | path := path[len(service):] // compute in the fetchRepository - SRP 145 | 146 | gs.fetchRepository(path) 147 | } 148 | } 149 | 150 | func (gs *GitService) fetchRepository(path string) error { 151 | service := fmt.Sprintf("%s/%s", gs.Config.DataDir, gs.Config.Server) 152 | 153 | dir := fmt.Sprintf("%s/%s", service, path) 154 | 155 | logger := gs.Logger.WithFields(log.Fields{ 156 | "path": path, 157 | "action": "fetchRepository", 158 | }) 159 | 160 | gs.StateChan <- pkgmirror.State{ 161 | Message: fmt.Sprintf("Fetch %s", dir[len(service):]), 162 | Status: pkgmirror.STATUS_RUNNING, 163 | } 164 | 165 | logger.Info("fetch repository") 166 | 167 | var outbuf, errbuf bytes.Buffer 168 | 169 | cmd := exec.Command(gs.Config.Binary, "fetch") 170 | cmd.Dir = dir 171 | cmd.Stdout = &outbuf 172 | cmd.Stderr = &errbuf 173 | 174 | if err := cmd.Start(); err != nil { 175 | logger.WithFields(log.Fields{ 176 | log.ErrorKey: err, 177 | "stderr": errbuf.String(), 178 | "stdout": outbuf.String(), 179 | }).Error("Error while starting the fetch command") 180 | 181 | return err 182 | } 183 | 184 | if err := cmd.Wait(); err != nil { 185 | logger.WithFields(log.Fields{ 186 | log.ErrorKey: err, 187 | "stderr": errbuf.String(), 188 | "stdout": outbuf.String(), 189 | }).Error("Error while waiting the fetch command") 190 | 191 | return err 192 | } 193 | 194 | gs.Logger.WithFields(log.Fields{ 195 | "path": path, 196 | "action": "SyncRepositories", 197 | }).Debug("Complete the fetch command") 198 | 199 | return nil 200 | } 201 | 202 | func (gs *GitService) WriteArchive(w io.Writer, path, ref string) error { 203 | if CACHEABLE_REF.Match([]byte(ref)) { 204 | return gs.cacheArchive(w, path, ref) 205 | } else { 206 | return gs.writeArchive(w, path, ref) 207 | } 208 | } 209 | 210 | func (gs *GitService) cacheArchive(w io.Writer, path, ref string) error { 211 | logger := gs.Logger.WithFields(log.Fields{ 212 | "path": path, 213 | "ref": ref, 214 | "action": "cacheArchive", 215 | }) 216 | 217 | if !IS_REF.Match([]byte(ref)) { 218 | return pkgmirror.InvalidReferenceError 219 | } 220 | 221 | vaultKey := fmt.Sprintf("%s:%s/%s", gs.Config.Server, path, ref) 222 | 223 | if !gs.Vault.Has(vaultKey) { 224 | logger.Info("Create vault entry") 225 | 226 | var wg sync.WaitGroup 227 | 228 | if !gs.Has(path) { // repository exists, nothing to do 229 | logger.Info("Git folder does not exist") 230 | 231 | if len(gs.Config.Clone) == 0 { 232 | return pkgmirror.ResourceNotFoundError // not configured, so skip clone 233 | } 234 | 235 | if err := gs.Clone(path); err != nil { 236 | return err 237 | } 238 | } 239 | 240 | if !gs.hasRef(path, ref) { 241 | logger.Info("Reference does not exist, try to fetch remote repository") 242 | 243 | if err := gs.fetchRepository(path); err != nil { 244 | return err 245 | } 246 | 247 | if !gs.hasRef(path, ref) { 248 | return pkgmirror.InvalidReferenceError 249 | } 250 | } 251 | 252 | pr, pw := io.Pipe() 253 | wg.Add(1) 254 | 255 | go func() { 256 | meta := vault.NewVaultMetadata() 257 | meta["path"] = path 258 | meta["ref"] = ref 259 | 260 | if _, err := gs.Vault.Put(vaultKey, meta, pr); err != nil { 261 | logger.WithError(err).Info("Error while writing into vault") 262 | 263 | gs.Vault.Remove(vaultKey) 264 | } 265 | 266 | wg.Done() 267 | }() 268 | 269 | if err := gs.writeArchive(pw, path, ref); err != nil { 270 | logger.WithError(err).Info("Error while writing archive") 271 | 272 | pw.Close() 273 | pr.Close() 274 | 275 | gs.Vault.Remove(vaultKey) 276 | 277 | return err 278 | } else { 279 | pw.Close() 280 | } 281 | 282 | wg.Wait() 283 | 284 | pr.Close() 285 | } 286 | 287 | logger.Info("Read vault entry") 288 | if _, err := gs.Vault.Get(vaultKey, w); err != nil { 289 | return err 290 | } 291 | 292 | return nil 293 | } 294 | 295 | func (gs *GitService) dataFolder() string { 296 | return gs.Config.DataDir + string(filepath.Separator) + gs.Config.Server 297 | } 298 | 299 | func (gs *GitService) hasRef(path, ref string) bool { 300 | if !IS_REF.Match([]byte(ref)) { 301 | return false 302 | } 303 | 304 | cmd := exec.Command(gs.Config.Binary, "rev-parse", "--quiet", "--verify", ref) 305 | cmd.Dir = gs.dataFolder() + string(filepath.Separator) + path 306 | 307 | if err := cmd.Start(); err != nil { 308 | return false 309 | } 310 | 311 | if err := cmd.Wait(); err != nil { 312 | return false 313 | } 314 | 315 | return true 316 | } 317 | 318 | func (gs *GitService) writeArchive(w io.Writer, path, ref string) error { 319 | logger := gs.Logger.WithFields(log.Fields{ 320 | "path": gs.dataFolder() + string(filepath.Separator) + path, 321 | "action": "writeArchive", 322 | "cmd": fmt.Sprintf("%s archive --format=zip %s", gs.Config.Binary, ref), 323 | }) 324 | 325 | cmd := exec.Command(gs.Config.Binary, "archive", "--format=zip", ref) 326 | cmd.Dir = gs.dataFolder() + string(filepath.Separator) + path 327 | 328 | stdout, _ := cmd.StdoutPipe() 329 | 330 | if err := cmd.Start(); err != nil { 331 | logger.WithError(err).Error("Error while starting the archive command") 332 | 333 | return err 334 | } 335 | 336 | if _, err := io.Copy(w, stdout); err != nil { 337 | logger.WithError(err).Error("Error while reading stdout from the archive command") 338 | } 339 | 340 | if err := cmd.Wait(); err != nil { 341 | logger.WithError(err).Error("Error while waiting the archive command") 342 | 343 | return err 344 | } 345 | 346 | logger.Info("Complete the archive command") 347 | 348 | return nil 349 | } 350 | 351 | func (gs *GitService) Has(path string) bool { 352 | gitPath := gs.dataFolder() + string(filepath.Separator) + path 353 | 354 | has := true 355 | if _, err := os.Stat(gitPath); os.IsNotExist(err) { 356 | has = false 357 | } 358 | 359 | gs.Logger.WithFields(log.Fields{ 360 | "path": gitPath, 361 | "action": "Has", 362 | "has": has, 363 | }).Debug("Has repository?") 364 | 365 | return has 366 | } 367 | 368 | func (gs *GitService) Clone(path string) error { 369 | gitPath := gs.dataFolder() + string(filepath.Separator) + path 370 | remote := strings.Replace(gs.Config.Clone, "{path}", path, -1) 371 | 372 | if gs.Config.Clone == remote { 373 | // same key, no replacement 374 | return pkgmirror.SameKeyError 375 | } 376 | 377 | logger := gs.Logger.WithFields(log.Fields{ 378 | "path": path, 379 | "action": "Clone", 380 | "remote": remote, 381 | }) 382 | 383 | logger.Info("Starting cloning remote repository") 384 | 385 | cmd := exec.Command(gs.Config.Binary, "clone", "--mirror", remote, gitPath) 386 | 387 | logger.WithField("cmd", cmd.Args).Debug("Run command") 388 | 389 | if err := cmd.Start(); err != nil { 390 | logger.WithError(err).Error("Error while starting to clone the remote repository") 391 | 392 | return err 393 | } 394 | 395 | if err := cmd.Wait(); err != nil { 396 | logger.WithError(err).Error("Error while cloning the remote repository") 397 | 398 | return err 399 | } 400 | 401 | return nil 402 | } 403 | 404 | func GitRewriteArchive(publicServer, path string) string { 405 | if results := GITHUB_ARCHIVE.FindStringSubmatch(path); len(results) == 6 { 406 | return fmt.Sprintf("%s/git/%s/%s/%s/%s.zip", publicServer, results[2], results[3], results[4], results[5]) 407 | } 408 | 409 | if results := BITBUCKET_ARCHIVE.FindStringSubmatch(path); len(results) == 6 { 410 | return fmt.Sprintf("%s/git/%s/%s/%s/%s.zip", publicServer, results[2], results[3], results[4], results[5]) 411 | } 412 | 413 | if results := GITLAB_ARCHIVE.FindStringSubmatch(path); len(results) == 6 { 414 | return fmt.Sprintf("%s/git/%s/%s/%s/%s.zip", publicServer, results[2], results[3], results[4], results[5]) 415 | } 416 | 417 | if results := DRUPAL_ARCHIVE.FindStringSubmatch(path); len(results) == 2 { 418 | return fmt.Sprintf("%s/static/drupal/%s.zip", publicServer, results[1]) 419 | } 420 | 421 | return publicServer 422 | } 423 | 424 | func GitRewriteRepository(publicServer, path string) string { 425 | if results := SVN_REPOSITORY.FindStringSubmatch(path); len(results) > 1 { 426 | return path // svn not supported 427 | } 428 | 429 | if results := GIT_REPOSITORY.FindStringSubmatch(path); len(results) > 1 { 430 | return fmt.Sprintf("%s/git/%s/%s.git", publicServer, results[6], results[8]) 431 | } 432 | 433 | return publicServer 434 | } 435 | -------------------------------------------------------------------------------- /mirror/git/git_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package git 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | "regexp" 12 | 13 | "github.com/AaronO/go-git-http" 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/rande/goapp" 16 | "github.com/rande/gonode/core/vault" 17 | "github.com/rande/pkgmirror" 18 | "goji.io" 19 | "goji.io/pat" 20 | "golang.org/x/net/context" 21 | ) 22 | 23 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 24 | 25 | l.Register(func(app *goapp.App) error { 26 | logger := app.Get("logger").(*log.Logger) 27 | 28 | v := &vault.Vault{ 29 | Algo: "no_op", 30 | Driver: &vault.DriverFs{ 31 | Root: fmt.Sprintf("%s/git", config.CacheDir), 32 | }, 33 | } 34 | 35 | for name, conf := range config.Git { 36 | if !conf.Enabled { 37 | continue 38 | } 39 | 40 | app.Set(fmt.Sprintf("pkgmirror.git.%s", name), func(name string, conf *pkgmirror.GitConfig) func(app *goapp.App) interface{} { 41 | 42 | return func(app *goapp.App) interface{} { 43 | s := NewGitService() 44 | s.Config.Server = conf.Server 45 | s.Config.PublicServer = config.PublicServer 46 | s.Config.DataDir = fmt.Sprintf("%s/git", config.DataDir) 47 | s.Config.Clone = conf.Clone 48 | s.Vault = v 49 | s.Logger = logger.WithFields(log.Fields{ 50 | "handler": "git", 51 | "code": name, 52 | }) 53 | s.StateChan = pkgmirror.GetStateChannel(fmt.Sprintf("pkgmirror.git.%s", name), app.Get("pkgmirror.channel.state").(chan pkgmirror.State)) 54 | 55 | if err := s.Init(app); err != nil { 56 | panic(err) 57 | } 58 | 59 | return s 60 | } 61 | }(name, conf)) 62 | } 63 | 64 | return nil 65 | }) 66 | 67 | l.Prepare(func(app *goapp.App) error { 68 | for name, conf := range config.Git { 69 | if !conf.Enabled { 70 | continue 71 | } 72 | 73 | ConfigureHttp(name, conf, app) 74 | } 75 | 76 | logger := app.Get("logger").(*log.Logger) 77 | 78 | mux := app.Get("mux").(*goji.Mux) 79 | 80 | // disable push, RO repository 81 | gitServer := githttp.New(config.DataDir) 82 | gitServer.ReceivePack = false 83 | gitServer.EventHandler = func(ev githttp.Event) { 84 | entry := logger.WithFields(log.Fields{ 85 | "commit": ev.Commit, 86 | "type": ev.Type.String(), 87 | "dir": ev.Dir, 88 | }) 89 | 90 | if ev.Error != nil { 91 | entry.WithError(ev.Error).Info("Git server error") 92 | } else { 93 | entry.Debug("Git command received") 94 | } 95 | } 96 | 97 | preAction := func(fn http.Handler) func(w http.ResponseWriter, r *http.Request) { 98 | return func(w http.ResponseWriter, r *http.Request) { 99 | 100 | for name, conf := range config.Git { 101 | path := "/git/" + conf.Server 102 | 103 | if !conf.Enabled { // not enable skip 104 | continue 105 | } 106 | 107 | logger.WithFields(log.Fields{ 108 | "request.path": r.URL.Path, 109 | "path": path, 110 | "handler": "git", 111 | "code": name, 112 | }).Debug("Check auto cloning action") 113 | 114 | if len(r.URL.Path) > len(path) && path == r.URL.Path[0:len(path)] { 115 | //found match 116 | s := app.Get(fmt.Sprintf("pkgmirror.git.%s", name)).(*GitService) 117 | 118 | if len(s.Config.Clone) == 0 { 119 | break // not configured, so skip clone 120 | } 121 | 122 | expression := fmt.Sprintf(`/git/%s/((.*)\.git)(|.*)`, conf.Server) 123 | 124 | l := logger.WithFields(log.Fields{ 125 | "path": r.URL.Path, 126 | "handler": "git", 127 | "code": name, 128 | "expression": expression, 129 | }) 130 | 131 | reg := regexp.MustCompile(expression) 132 | 133 | path := "" 134 | if results := reg.FindStringSubmatch(r.URL.Path); len(results) > 0 { 135 | path = results[1] 136 | } else { 137 | l.Error("Unable to find valid git path") 138 | 139 | break // not valid 140 | } 141 | 142 | if s.Has(path) { // repository exists, nothing to do 143 | l.Debug("Skipping cloning, repository exist") 144 | 145 | break 146 | } 147 | 148 | // not available, clone the repository 149 | if err := s.Clone(path); err != nil { 150 | l.WithError(err).Error("Unable to clone the repository") 151 | } 152 | 153 | break 154 | } else { 155 | logger.WithFields(log.Fields{ 156 | "path": r.URL.Path, 157 | "handler": "git", 158 | "code": name, 159 | }).Warn("Does not match auto clone path") 160 | } 161 | } 162 | 163 | fn.ServeHTTP(w, r) 164 | } 165 | } 166 | 167 | mux.HandleFunc(pat.Get("/git/*"), preAction(gitServer)) 168 | mux.HandleFunc(pat.Post("/git/*"), preAction(gitServer)) 169 | 170 | return nil 171 | }) 172 | 173 | for name, conf := range config.Git { 174 | if !conf.Enabled { 175 | continue 176 | } 177 | 178 | l.Run(func(name string) func(app *goapp.App, state *goapp.GoroutineState) error { 179 | return func(app *goapp.App, state *goapp.GoroutineState) error { 180 | s := app.Get(fmt.Sprintf("pkgmirror.git.%s", name)).(pkgmirror.MirrorService) 181 | s.Serve(state) 182 | 183 | return nil 184 | } 185 | }(name)) 186 | } 187 | } 188 | 189 | func ConfigureHttp(name string, conf *pkgmirror.GitConfig, app *goapp.App) { 190 | gitService := app.Get(fmt.Sprintf("pkgmirror.git.%s", name)).(*GitService) 191 | 192 | mux := app.Get("mux").(*goji.Mux) 193 | 194 | mux.HandleFuncC(NewGitPat(conf.Server), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 195 | w.Header().Set("Content-Type", "application/zip") 196 | if err := gitService.WriteArchive(w, fmt.Sprintf("%s.git", pat.Param(ctx, "path")), pat.Param(ctx, "ref")); err != nil { 197 | pkgmirror.SendWithHttpCode(w, 500, err.Error()) 198 | } 199 | }) 200 | } 201 | -------------------------------------------------------------------------------- /mirror/git/git_pat.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package git 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | "regexp" 12 | 13 | "goji.io" 14 | "goji.io/pattern" 15 | "golang.org/x/net/context" 16 | ) 17 | 18 | func NewGitPat(hostname string) goji.Pattern { 19 | return &GitPat{ 20 | Hostname: hostname, 21 | Pattern: regexp.MustCompile(fmt.Sprintf(`\/git\/%s\/(.*)\/([\w\d]{40}|(.*))\.zip`, hostname)), 22 | } 23 | } 24 | 25 | type GitPat struct { 26 | Hostname string 27 | Pattern *regexp.Regexp 28 | } 29 | 30 | func (pp *GitPat) Match(ctx context.Context, r *http.Request) context.Context { 31 | if results := pp.Pattern.FindStringSubmatch(r.URL.Path); len(results) == 0 { 32 | return nil 33 | } else { 34 | return &gitPatMatch{ctx, pp.Hostname, results[1], results[2], "zip"} 35 | } 36 | } 37 | 38 | type gitPatMatch struct { 39 | context.Context 40 | Hostname string 41 | Path string 42 | Ref string 43 | Format string 44 | } 45 | 46 | func (m gitPatMatch) Value(key interface{}) interface{} { 47 | 48 | switch key { 49 | case pattern.AllVariables: 50 | return map[pattern.Variable]string{ 51 | "hostname": m.Hostname, 52 | "path": m.Path, 53 | "ref": m.Ref, 54 | "format": m.Format, 55 | } 56 | case pattern.Variable("hostname"): 57 | return m.Hostname 58 | case pattern.Variable("path"): 59 | return m.Path 60 | case pattern.Variable("ref"): 61 | return m.Ref 62 | case pattern.Variable("format"): 63 | return m.Format 64 | } 65 | 66 | return m.Context.Value(key) 67 | } 68 | -------------------------------------------------------------------------------- /mirror/git/git_pat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package git 7 | 8 | import ( 9 | "net/http" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "goji.io/pattern" 14 | "golang.org/x/net/context" 15 | ) 16 | 17 | func mustReq(method, path string) (context.Context, *http.Request) { 18 | req, err := http.NewRequest(method, path, nil) 19 | if err != nil { 20 | panic(err) 21 | } 22 | ctx := pattern.SetPath(context.Background(), req.URL.EscapedPath()) 23 | 24 | return ctx, req 25 | } 26 | 27 | func Test_Composer_Pat_Archive(t *testing.T) { 28 | p := NewGitPat("github.com") 29 | 30 | c, r := mustReq("GET", "/git/github.com/kevinlebrun/colors.php/cb9b6666a2dfd9b6074b4a5caec7902fe3033578.zip") 31 | 32 | result := p.Match(c, r) 33 | 34 | assert.NotNil(t, result) 35 | assert.Equal(t, "github.com", result.Value(pattern.Variable("hostname"))) 36 | assert.Equal(t, "kevinlebrun/colors.php", result.Value(pattern.Variable("path"))) 37 | assert.Equal(t, "cb9b6666a2dfd9b6074b4a5caec7902fe3033578", result.Value(pattern.Variable("ref"))) 38 | assert.Equal(t, "zip", result.Value(pattern.Variable("format"))) 39 | } 40 | 41 | func Test_Git_Pat_AllVariables(t *testing.T) { 42 | p := NewGitPat("github.com") 43 | 44 | c, r := mustReq("GET", "/git/github.com/kevinlebrun/colors.php/cb9b6666a2dfd9b6074b4a5caec7902fe3033578.zip") 45 | 46 | result := p.Match(c, r) 47 | 48 | assert.NotNil(t, result) 49 | 50 | vars := result.Value(pattern.AllVariables).(map[pattern.Variable]string) 51 | 52 | assert.Equal(t, "github.com", vars["hostname"]) 53 | assert.Equal(t, "kevinlebrun/colors.php", vars["path"]) 54 | assert.Equal(t, "cb9b6666a2dfd9b6074b4a5caec7902fe3033578", vars["ref"]) 55 | assert.Equal(t, "zip", vars["format"]) 56 | } 57 | 58 | func Test_Git_Pat_OtherVariable(t *testing.T) { 59 | p := NewGitPat("github.com") 60 | 61 | c, r := mustReq("GET", "/git/github.com/kevinlebrun/colors.php/cb9b6666a2dfd9b6074b4a5caec7902fe3033578.zip") 62 | 63 | result := p.Match(c, r) 64 | 65 | assert.NotNil(t, result) 66 | 67 | assert.Nil(t, result.Value(pattern.Variable("foo"))) 68 | } 69 | -------------------------------------------------------------------------------- /mirror/git/git_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package git 7 | 8 | import ( 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | type Expectation struct { 15 | Expected string 16 | Value string 17 | } 18 | 19 | func Test_Archive_Rewrite_Github(t *testing.T) { 20 | publicServer := "https://mirrors.localhost" 21 | 22 | values := []*Expectation{ 23 | {"https://mirrors.localhost/git/github.com/sonata-project/exporter/b9098b5007c525a238ddf44d578b8efae7bccc72.zip", "https://api.github.com/repos/sonata-project/exporter/zipball/b9098b5007c525a238ddf44d578b8efae7bccc72"}, 24 | {"https://mirrors.localhost/git/github.com/kevinlebrun/colors.php/6d7140aeedef46c97c2324f09b752c599ef17dac.zip", "https://api.github.com/repos/kevinlebrun/colors.php/zipball/6d7140aeedef46c97c2324f09b752c599ef17dac"}, 25 | } 26 | 27 | for _, v := range values { 28 | assert.Equal(t, v.Expected, GitRewriteArchive(publicServer, v.Value)) 29 | } 30 | 31 | } 32 | 33 | func Test_Archive_Rewrite_Bitbucket(t *testing.T) { 34 | publicServer := "https://mirrors.localhost" 35 | 36 | path := GitRewriteArchive(publicServer, "https://bitbucket.org/sonata-project/exporter/get/b9098b5007c525a238ddf44d578b8efae7bccc72.zip") 37 | assert.Equal(t, "https://mirrors.localhost/git/bitbucket.org/sonata-project/exporter/b9098b5007c525a238ddf44d578b8efae7bccc72.zip", path) 38 | } 39 | 40 | func Test_Archive_Rewrite_Gitlab(t *testing.T) { 41 | publicServer := "https://mirrors.localhost" 42 | 43 | path := GitRewriteArchive(publicServer, "https://gitlab.example.com/sonata-project/exporter/repository/archive.zip?ref=b9098b5007c525a238ddf44d578b8efae7bccc72") 44 | assert.Equal(t, "https://mirrors.localhost/git/gitlab.example.com/sonata-project/exporter/b9098b5007c525a238ddf44d578b8efae7bccc72.zip", path) 45 | } 46 | 47 | func Test_Repository_Rewrite_Git(t *testing.T) { 48 | publicServer := "https://mirrors.localhost" 49 | 50 | values := []*Expectation{ 51 | {"https://mirrors.localhost/git/github.com/DavidForest/ImgBundle.git", "git@github.com:DavidForest/ImgBundle.git"}, 52 | {"https://mirrors.localhost/git/github.com/sonata-project/exporter.git", "https://github.com/sonata-project/exporter.git"}, 53 | {"https://mirrors.localhost/git/bitbucket.org/foo/bar.git", "https://bitbucket.org/foo/bar"}, 54 | {"https://mirrors.localhost/git/github.com/xstudios/flames.git", "git://github.com/xstudios/flames.git"}, 55 | {"https://mirrors.localhost/git/git.kootstradevelopment.nl/r.kootstra/stackinstance-bundles-mailer-bundle.git", "http://git.kootstradevelopment.nl/r.kootstra/stackinstance-bundles-mailer-bundle.git"}, 56 | {"https://mirrors.localhost/git/github.com/zyncro/bower-videogular-themes-default.git", "https://github.com/zyncro/bower-videogular-themes-default.git"}, 57 | } 58 | 59 | for _, v := range values { 60 | assert.Equal(t, v.Expected, GitRewriteRepository(publicServer, v.Value)) 61 | } 62 | } 63 | 64 | func Test_Repository_Rewrite_SVN(t *testing.T) { 65 | publicServer := "https://mirrors.localhost" 66 | 67 | values := []*Expectation{ 68 | {"https://m10s.svn.beanstalkapp.com/m10s-common", "https://m10s.svn.beanstalkapp.com/m10s-common"}, 69 | {"svn://localhost/path/to/project", "svn://localhost/path/to/project"}, 70 | } 71 | 72 | for _, v := range values { 73 | assert.Equal(t, v.Expected, GitRewriteRepository(publicServer, v.Value)) 74 | } 75 | } 76 | 77 | func Test_Drupal_Archive(t *testing.T) { 78 | publicServer := "https://mirrors.localhost" 79 | 80 | path := GitRewriteArchive(publicServer, "https://ftp.drupal.org/files/projects/ctools-8.x-3.0.zip") 81 | assert.Equal(t, "https://mirrors.localhost/static/drupal/ctools-8.x-3.0.zip", path) 82 | 83 | } 84 | -------------------------------------------------------------------------------- /mirror/npm/npm_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package npm 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | 12 | log "github.com/Sirupsen/logrus" 13 | "github.com/rande/goapp" 14 | "github.com/rande/gonode/core/vault" 15 | "github.com/rande/pkgmirror" 16 | "goji.io" 17 | "goji.io/pat" 18 | "golang.org/x/net/context" 19 | ) 20 | 21 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 22 | 23 | l.Register(func(app *goapp.App) error { 24 | logger := app.Get("logger").(*log.Logger) 25 | 26 | for name, conf := range config.Npm { 27 | 28 | if !conf.Enabled { 29 | continue 30 | } 31 | 32 | app.Set(fmt.Sprintf("pkgmirror.npm.%s", name), func(name string, conf *pkgmirror.NpmConfig) func(app *goapp.App) interface{} { 33 | return func(app *goapp.App) interface{} { 34 | s := NewNpmService() 35 | s.Config.Path = fmt.Sprintf("%s/npm", config.DataDir) 36 | s.Config.PublicServer = config.PublicServer 37 | s.Config.SourceServer = conf.Server 38 | s.Config.Code = []byte(name) 39 | s.Logger = logger.WithFields(log.Fields{ 40 | "handler": "npm", 41 | "server": s.Config.SourceServer, 42 | "code": name, 43 | }) 44 | s.Vault = &vault.Vault{ 45 | Algo: "no_op", 46 | Driver: &vault.DriverFs{ 47 | Root: fmt.Sprintf("%s/npm/%s_packages", config.DataDir, name), 48 | }, 49 | } 50 | s.StateChan = pkgmirror.GetStateChannel(fmt.Sprintf("pkgmirror.npm.%s", name), app.Get("pkgmirror.channel.state").(chan pkgmirror.State)) 51 | s.BoltCompacter = app.Get("bolt.compacter").(*pkgmirror.BoltCompacter) 52 | 53 | if err := s.Init(app); err != nil { 54 | panic(err) 55 | } 56 | 57 | return s 58 | } 59 | }(name, conf)) 60 | } 61 | 62 | return nil 63 | }) 64 | 65 | l.Prepare(func(app *goapp.App) error { 66 | //c.Ui.Info(fmt.Sprintf("Start HTTP Server (bind: %s)", config.InternalServer)) 67 | 68 | for name, conf := range config.Npm { 69 | if !conf.Enabled { 70 | continue 71 | } 72 | 73 | ConfigureHttp(name, conf, app) 74 | } 75 | 76 | return nil 77 | }) 78 | 79 | for name, conf := range config.Npm { 80 | if !conf.Enabled { 81 | continue 82 | } 83 | 84 | l.Run(func(name string, conf *pkgmirror.NpmConfig) func(app *goapp.App, state *goapp.GoroutineState) error { 85 | return func(app *goapp.App, state *goapp.GoroutineState) error { 86 | //c.Ui.Info(fmt.Sprintf("Start Npm Sync (server: %s/npm)", config.PublicServer)) 87 | s := app.Get(fmt.Sprintf("pkgmirror.npm.%s", name)).(*NpmService) 88 | s.Serve(state) 89 | 90 | return nil 91 | } 92 | }(name, conf)) 93 | } 94 | } 95 | 96 | func ConfigureHttp(name string, conf *pkgmirror.NpmConfig, app *goapp.App) { 97 | mux := app.Get("mux").(*goji.Mux) 98 | npmService := app.Get(fmt.Sprintf("pkgmirror.npm.%s", name)).(*NpmService) 99 | 100 | mux.HandleFuncC(NewArchivePat(name), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 101 | w.Header().Set("Content-Type", "Content-Type: application/octet-stream") 102 | if err := npmService.WriteArchive(w, pat.Param(ctx, "package"), pat.Param(ctx, "version")); err != nil { 103 | pkgmirror.SendWithHttpCode(w, 500, err.Error()) 104 | } 105 | }) 106 | 107 | mux.HandleFuncC(pat.Get(fmt.Sprintf("/npm/%s/*", name)), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 108 | pkg := r.URL.Path[6+len(name):] 109 | 110 | if refresh := r.FormValue("refresh"); len(refresh) > 0 { 111 | w.Header().Set("Content-Type", "application/json") 112 | 113 | if err := npmService.UpdatePackage(pkg); err != nil { 114 | pkgmirror.SendWithHttpCode(w, 500, err.Error()) 115 | } else { 116 | pkgmirror.SendWithHttpCode(w, 200, "Package updated") 117 | } 118 | 119 | return 120 | } 121 | 122 | if data, err := npmService.Get(pkg); err != nil { 123 | pkgmirror.SendWithHttpCode(w, 404, err.Error()) 124 | } else { 125 | w.Header().Set("Content-Type", "application/json") 126 | w.Header().Set("Content-Encoding", "gzip") 127 | w.Write(data) 128 | } 129 | }) 130 | } 131 | -------------------------------------------------------------------------------- /mirror/npm/npm_pat.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package npm 7 | 8 | import ( 9 | "fmt" 10 | "net/http" 11 | "regexp" 12 | "strings" 13 | 14 | "goji.io" 15 | "goji.io/pattern" 16 | "golang.org/x/net/context" 17 | ) 18 | 19 | func NewArchivePat(code string) goji.Pattern { 20 | return &PackagePat{ 21 | Pattern: regexp.MustCompile(fmt.Sprintf(`\/npm\/%s\/((@([\w\d.-]+)\/|)([@\w\d\.-]+))\/-\/(.*)\.(tgz)`, code)), 22 | } 23 | } 24 | 25 | type PackagePat struct { 26 | Pattern *regexp.Regexp 27 | } 28 | 29 | func (pp *PackagePat) Match(ctx context.Context, r *http.Request) context.Context { 30 | var results []string 31 | 32 | if results = pp.Pattern.FindStringSubmatch(r.URL.Path); len(results) == 0 { 33 | return nil 34 | } 35 | 36 | name := strings.Replace(results[1], "/", "%2f", -1) 37 | version := results[5][(len(results[4]) + 1):] 38 | 39 | return &packagePatMatch{ctx, name, version, "tgz"} 40 | 41 | } 42 | 43 | type packagePatMatch struct { 44 | context.Context 45 | Package string 46 | Version string 47 | Format string 48 | } 49 | 50 | func (m packagePatMatch) Value(key interface{}) interface{} { 51 | 52 | switch key { 53 | case pattern.AllVariables: 54 | return map[pattern.Variable]string{ 55 | "package": m.Package, 56 | "version": m.Version, 57 | "format": m.Format, 58 | } 59 | case pattern.Variable("version"): 60 | return m.Version 61 | case pattern.Variable("package"): 62 | return m.Package 63 | case pattern.Variable("format"): 64 | return m.Format 65 | } 66 | 67 | return m.Context.Value(key) 68 | } 69 | -------------------------------------------------------------------------------- /mirror/npm/npm_pat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package npm 7 | 8 | import ( 9 | "testing" 10 | 11 | "net/http" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "goji.io/pattern" 15 | "golang.org/x/net/context" 16 | ) 17 | 18 | type TestVersion struct { 19 | Url string 20 | Package string 21 | Version string 22 | } 23 | 24 | func mustReq(method, path string) (context.Context, *http.Request) { 25 | req, err := http.NewRequest(method, path, nil) 26 | if err != nil { 27 | panic(err) 28 | } 29 | ctx := pattern.SetPath(context.Background(), req.URL.EscapedPath()) 30 | 31 | return ctx, req 32 | } 33 | 34 | func Test_Npm_Pat(t *testing.T) { 35 | 36 | cases := []struct{ Url, Package, Version string }{ 37 | {"/npm/npm/aspace/-/aspace-0.0.1.tgz", "aspace", "0.0.1"}, 38 | {"/npm/npm/@type%2fnode/-/node-6.0.90.tgz", "@type%2fnode", "6.0.90"}, 39 | {"/npm/npm/dateformat/-/dateformat-1.0.2-1.2.3.tgz", "dateformat", "1.0.2-1.2.3"}, 40 | {"/npm/npm/@storybook%2freact/-/react-3.0.0.tgz", "@storybook%2freact", "3.0.0"}, 41 | } 42 | 43 | matcher := NewArchivePat("npm") 44 | 45 | for _, p := range cases { 46 | c, r := mustReq("GET", p.Url) 47 | 48 | result := matcher.Match(c, r) 49 | 50 | assert.NotNil(t, result) 51 | assert.Equal(t, p.Package, result.Value(pattern.Variable("package"))) 52 | assert.Equal(t, p.Version, result.Value(pattern.Variable("version"))) 53 | assert.Equal(t, "tgz", result.Value(pattern.Variable("format"))) 54 | } 55 | } 56 | 57 | func Test_Npm_Pat_AllVariables(t *testing.T) { 58 | p := NewArchivePat("npm") 59 | 60 | c, r := mustReq("GET", "/npm/npm/aspace/-/aspace-0.0.1.tgz") 61 | 62 | result := p.Match(c, r) 63 | 64 | assert.NotNil(t, result) 65 | 66 | vars := result.Value(pattern.AllVariables).(map[pattern.Variable]string) 67 | 68 | assert.Equal(t, "aspace", vars["package"]) 69 | assert.Equal(t, "0.0.1", vars["version"]) 70 | assert.Equal(t, "tgz", vars["format"]) 71 | } 72 | 73 | func Test_Npm_Pat_OtherVariable(t *testing.T) { 74 | p := NewArchivePat("npm") 75 | 76 | c, r := mustReq("GET", "/npm/npm/aspace/-/aspace-0.0.1.tgz") 77 | 78 | result := p.Match(c, r) 79 | 80 | assert.NotNil(t, result) 81 | 82 | assert.Nil(t, result.Value(pattern.Variable("foo"))) 83 | } 84 | -------------------------------------------------------------------------------- /mirror/npm/npm_structs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package npm 7 | 8 | import ( 9 | "encoding/json" 10 | ) 11 | 12 | type PackageVersionDefinition struct { 13 | Name string `json:"name,omitempty"` 14 | Description *json.RawMessage `json:"description,omitempty"` 15 | Version string `json:"version,omitempty"` 16 | Homepage *json.RawMessage `json:"homepage,omitempty"` 17 | Repository *json.RawMessage `json:"repository,omitempty"` 18 | //Contributors *json.RawMessage `json:"contributors,omitempty"` 19 | Main *json.RawMessage `json:"main,omitempty"` 20 | Licences *json.RawMessage `json:"licenses,omitempty"` 21 | Author *json.RawMessage `json:"author,omitempty"` 22 | Tags *json.RawMessage `json:"tags,omitempty"` 23 | Files *json.RawMessage `json:"files,omitempty"` 24 | Bin *json.RawMessage `json:"bin,omitempty"` 25 | Man *json.RawMessage `json:"man,omitempty"` 26 | Dependencies *json.RawMessage `json:"dependencies,omitempty"` 27 | DevDependencies *json.RawMessage `json:"devDependencies,omitempty"` 28 | PeerDependencies *json.RawMessage `json:"peerDependencies,omitempty"` 29 | OptionalDependencies *json.RawMessage `json:"optionalDependencies,omitempty"` 30 | EngineStrict *json.RawMessage `json:"engineStrict,omitempty"` 31 | Scripts *json.RawMessage `json:"scripts,omitempty"` 32 | Engines *json.RawMessage `json:"engines,omitempty"` 33 | Gypfile *json.RawMessage `json:"gypfile,omitempty"` 34 | License *json.RawMessage `json:"license,omitempty"` 35 | GitHead *json.RawMessage `json:"gitHead,omitempty"` 36 | //Bugs *json.RawMessage `json:"bugs,omitempty"` 37 | Binary *json.RawMessage `json:"binary,omitempty"` 38 | Os *json.RawMessage `json:"os,omitempty"` 39 | Cpu *json.RawMessage `json:"cpu,omitempty"` 40 | PreferGlobal *json.RawMessage `json:"preferGlobal,omitempty"` 41 | PublishConfig *json.RawMessage `json:"publishConfig,omitempty"` 42 | BundleDependencies *json.RawMessage `json:"bundleDependencies,omitempty"` 43 | Keywords *json.RawMessage `json:"keywords,omitempty"` 44 | //ID *json.RawMessage `json:"_id,omitempty"` 45 | //Shasum *json.RawMessage `json:"_shasum,omitempty"` 46 | //From *json.RawMessage `json:"_from,omitempty"` 47 | //NpmVersion *json.RawMessage `json:"_npmVersion,omitempty"` 48 | //NodeVersion *json.RawMessage `json:"_nodeVersion,omitempty"` 49 | //NpmUser *json.RawMessage `json:"_npmUser,omitempty"` 50 | //Maintainers *json.RawMessage `json:"maintainers,omitempty"` 51 | Dist struct { 52 | Shasum string `json:"shasum,omitempty"` 53 | Tarball string `json:"tarball,omitempty"` 54 | } `json:"dist,omitempty"` 55 | //NpmOperationalInternal *json.RawMessage `json:"_npmOperationalInternal,omitempty"` 56 | Directories *json.RawMessage `json:"directories,omitempty"` 57 | } 58 | 59 | type ShortPackageDefinition struct { 60 | ID string `json:"_id,omitempty"` 61 | Rev string `json:"_rev,omitempty"` 62 | Name string `json:"name,omitempty"` 63 | ReleasesAvailable int `json:"releases_available,omitempty"` 64 | } 65 | 66 | type FullPackageDefinition struct { 67 | ID string `json:"_id,omitempty"` 68 | Rev string `json:"_rev,omitempty"` 69 | Name string `json:"name,omitempty"` 70 | Description *json.RawMessage `json:"description,omitempty"` 71 | DistTags *json.RawMessage `json:"dist-tags"` 72 | Versions map[string]*PackageVersionDefinition `json:"versions,omitempty"` 73 | //Readme *json.RawMessage `json:"readme,omitempty"` 74 | //Maintainers *json.RawMessage `json:"maintainers,omitempty"` 75 | Time *json.RawMessage `json:"time,omitempty"` 76 | Author *json.RawMessage `json:"author,omitempty"` 77 | Repository *json.RawMessage `json:"repository,omitempty"` 78 | //Users *json.RawMessage `json:"users,omitempty"` 79 | //ReadmeFilename *json.RawMessage `json:"readmeFilename,omitempty"` 80 | //Homepage *json.RawMessage `json:"homepage,omitempty"` 81 | //Bugs *json.RawMessage `json:"bugs,omitempty"` 82 | License *json.RawMessage `json:"license,omitempty"` 83 | Attachments *json.RawMessage `json:"_attachments,omitempty"` 84 | } 85 | -------------------------------------------------------------------------------- /mirror/npm/npm_structs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package npm 7 | 8 | import ( 9 | "testing" 10 | 11 | "fmt" 12 | 13 | "github.com/rande/pkgmirror" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func Test_Load_Package(t *testing.T) { 18 | p := &FullPackageDefinition{} 19 | 20 | files := []struct{ Name, File string }{ 21 | {"knwl.js", "knwl.js.json"}, 22 | {"math_example_bulbignz", "math_example_bulbignz.json"}, 23 | {"gulp-app-manager", "gulp-app-manager.json"}, 24 | {"jsontocsv", "jsontocsv.json"}, 25 | {"qs", "qs.json"}, 26 | {"repeat", "repeat.json"}, 27 | } 28 | 29 | for _, f := range files { 30 | assert.NoError(t, pkgmirror.LoadStruct(fmt.Sprintf("../../fixtures/npm/%s", f.File), p), fmt.Sprintf("Package %s", f.File)) 31 | 32 | assert.Equal(t, f.Name, p.Name, fmt.Sprintf("Package %s", f.File)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /mirror/static/static.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package static 7 | 8 | import ( 9 | "encoding/json" 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "os" 14 | "path/filepath" 15 | "sync" 16 | "time" 17 | 18 | log "github.com/Sirupsen/logrus" 19 | "github.com/boltdb/bolt" 20 | "github.com/rande/goapp" 21 | "github.com/rande/gonode/core/vault" 22 | "github.com/rande/pkgmirror" 23 | ) 24 | 25 | func NewStaticService() *StaticService { 26 | return &StaticService{ 27 | Config: &StaticConfig{ 28 | Path: "./data/static", 29 | SourceServer: "http://localhost", 30 | }, 31 | Vault: &vault.Vault{ 32 | Algo: "no_op", 33 | Driver: &vault.DriverFs{ 34 | Root: "./cache/git", 35 | }, 36 | }, 37 | } 38 | } 39 | 40 | type StaticConfig struct { 41 | SourceServer string 42 | Path string 43 | Code []byte 44 | } 45 | 46 | type StaticService struct { 47 | DB *bolt.DB 48 | Config *StaticConfig 49 | Logger *log.Entry 50 | Vault *vault.Vault 51 | StateChan chan pkgmirror.State 52 | BoltCompacter *pkgmirror.BoltCompacter 53 | lock bool 54 | } 55 | 56 | func (gs *StaticService) Init(app *goapp.App) (err error) { 57 | os.MkdirAll(string(filepath.Separator)+gs.Config.Path, 0755) 58 | 59 | return gs.openDatabase() 60 | } 61 | 62 | func (gs *StaticService) openDatabase() (err error) { 63 | if gs.DB, err = pkgmirror.OpenDatabaseWithBucket(gs.Config.Path, gs.Config.Code); err != nil { 64 | gs.Logger.WithFields(log.Fields{ 65 | log.ErrorKey: err, 66 | "path": gs.Config.Path, 67 | "bucket": string(gs.Config.Code), 68 | "action": "Init", 69 | }).Error("Unable to open the internal database") 70 | 71 | return err 72 | } 73 | 74 | return nil 75 | } 76 | 77 | func (gs *StaticService) optimize() error { 78 | gs.lock = true 79 | 80 | path := gs.DB.Path() 81 | 82 | gs.DB.Close() 83 | 84 | if err := gs.BoltCompacter.Compact(path); err != nil { 85 | return err 86 | } 87 | 88 | err := gs.openDatabase() 89 | 90 | gs.lock = false 91 | 92 | return err 93 | } 94 | 95 | func (gs *StaticService) Serve(state *goapp.GoroutineState) error { 96 | // nothing to do, do sync feature available 97 | 98 | return nil 99 | } 100 | 101 | func (gs *StaticService) WriteArchive(w io.Writer, path string) (*StaticFile, error) { 102 | vaultKey := fmt.Sprintf("%s", path) 103 | bucketKey := vaultKey 104 | url := fmt.Sprintf("%s/%s", gs.Config.SourceServer, path) 105 | 106 | logger := gs.Logger.WithFields(log.Fields{ 107 | "path": path, 108 | "action": "WriteArchive", 109 | "vaultKey": vaultKey, 110 | "bucketKey": bucketKey, 111 | "url": url, 112 | }) 113 | 114 | file := &StaticFile{} 115 | 116 | err := gs.DB.View(func(tx *bolt.Tx) error { 117 | b := tx.Bucket(gs.Config.Code) 118 | 119 | data := b.Get([]byte(bucketKey)) 120 | 121 | if len(data) == 0 { 122 | return pkgmirror.EmptyDataError 123 | } 124 | 125 | if err := json.Unmarshal(data, file); err != nil { 126 | return err 127 | } 128 | 129 | return nil 130 | }) 131 | 132 | if err == pkgmirror.EmptyDataError { 133 | file.Url = url 134 | } else if err != nil { 135 | return nil, err 136 | } 137 | 138 | if !gs.Vault.Has(vaultKey) { 139 | logger.Info("Create vault entry") 140 | 141 | var wg sync.WaitGroup 142 | var err error 143 | var data []byte 144 | 145 | pr, pw := io.Pipe() 146 | wg.Add(1) 147 | 148 | go func() { 149 | meta := vault.NewVaultMetadata() 150 | meta["path"] = path 151 | 152 | if _, err := gs.Vault.Put(vaultKey, meta, pr); err != nil { 153 | logger.WithError(err).Info("Error while writing into vault") 154 | 155 | gs.Vault.Remove(vaultKey) 156 | } 157 | 158 | wg.Done() 159 | }() 160 | 161 | if err = gs.downloadStatic(pw, file); err != nil { 162 | logger.WithError(err).Info("Error while writing archive") 163 | 164 | pw.Close() 165 | pr.Close() 166 | 167 | gs.Vault.Remove(vaultKey) 168 | 169 | return nil, err 170 | } else { 171 | pw.Close() 172 | } 173 | 174 | wg.Wait() 175 | 176 | pr.Close() 177 | 178 | err = gs.DB.Update(func(tx *bolt.Tx) error { 179 | b := tx.Bucket(gs.Config.Code) 180 | 181 | data, err = json.Marshal(file) 182 | 183 | if err != nil { 184 | return err 185 | } 186 | 187 | if err := b.Put([]byte(bucketKey), data); err != nil { 188 | return err 189 | } 190 | 191 | return nil 192 | }) 193 | } else { 194 | logger.Info("Vault entry exist!") 195 | } 196 | 197 | logger.Info("Read vault entry") 198 | if _, err := gs.Vault.Get(vaultKey, w); err != nil { 199 | return nil, err 200 | } 201 | 202 | return file, nil 203 | } 204 | 205 | func (gs *StaticService) downloadStatic(w io.Writer, file *StaticFile) error { 206 | logger := gs.Logger.WithFields(log.Fields{ 207 | "url": file.Url, 208 | "action": "writeArchive", 209 | }) 210 | 211 | logger.Info("Start downloading the remote static file") 212 | 213 | resp, err := http.Get(file.Url) 214 | 215 | if err != nil { 216 | return err 217 | } 218 | 219 | defer resp.Body.Close() 220 | 221 | if resp.StatusCode == 404 { 222 | return pkgmirror.ResourceNotFoundError 223 | } 224 | 225 | if resp.StatusCode != 200 { 226 | return pkgmirror.HttpError 227 | } 228 | 229 | written, err := io.Copy(w, resp.Body) 230 | 231 | if err != nil { 232 | logger.WithError(err).Error("Error while writing input stream to the target stream") 233 | 234 | return err 235 | } 236 | 237 | file.Size = written 238 | file.Header = resp.Header 239 | file.DownloadAt = time.Now() 240 | 241 | logger.Info("Complete downloading the remote static file") 242 | 243 | return nil 244 | } 245 | -------------------------------------------------------------------------------- /mirror/static/static_app.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package static 7 | 8 | import ( 9 | "bytes" 10 | "fmt" 11 | "net/http" 12 | 13 | log "github.com/Sirupsen/logrus" 14 | "github.com/rande/goapp" 15 | "github.com/rande/gonode/core/vault" 16 | "github.com/rande/pkgmirror" 17 | "goji.io" 18 | "goji.io/pat" 19 | "golang.org/x/net/context" 20 | ) 21 | 22 | func ConfigureApp(config *pkgmirror.Config, l *goapp.Lifecycle) { 23 | 24 | l.Register(func(app *goapp.App) error { 25 | logger := app.Get("logger").(*log.Logger) 26 | 27 | v := &vault.Vault{ 28 | Algo: "no_op", 29 | Driver: &vault.DriverFs{ 30 | Root: fmt.Sprintf("%s/static", config.CacheDir), 31 | }, 32 | } 33 | 34 | for name, conf := range config.Static { 35 | if !conf.Enabled { 36 | continue 37 | } 38 | 39 | app.Set(fmt.Sprintf("pkgmirror.static.%s", name), func(name string, conf *pkgmirror.StaticConfig) func(app *goapp.App) interface{} { 40 | return func(app *goapp.App) interface{} { 41 | s := NewStaticService() 42 | s.Vault = v 43 | s.Config.SourceServer = conf.Server 44 | s.Config.Path = fmt.Sprintf("%s/static", config.DataDir) 45 | s.Config.Code = []byte(name) 46 | s.Logger = logger.WithFields(log.Fields{ 47 | "handler": "static", 48 | "code": name, 49 | }) 50 | s.StateChan = pkgmirror.GetStateChannel(fmt.Sprintf("pkgmirror.static.%s", name), app.Get("pkgmirror.channel.state").(chan pkgmirror.State)) 51 | s.BoltCompacter = app.Get("bolt.compacter").(*pkgmirror.BoltCompacter) 52 | 53 | if err := s.Init(app); err != nil { 54 | panic(err) 55 | } 56 | 57 | return s 58 | } 59 | }(name, conf)) 60 | } 61 | 62 | return nil 63 | }) 64 | 65 | l.Prepare(func(app *goapp.App) error { 66 | for name, conf := range config.Static { 67 | if !conf.Enabled { 68 | continue 69 | } 70 | 71 | ConfigureHttp(name, conf, app) 72 | } 73 | 74 | return nil 75 | }) 76 | 77 | for name, conf := range config.Static { 78 | if !conf.Enabled { 79 | continue 80 | } 81 | 82 | l.Run(func(name string) func(app *goapp.App, state *goapp.GoroutineState) error { 83 | return func(app *goapp.App, state *goapp.GoroutineState) error { 84 | s := app.Get(fmt.Sprintf("pkgmirror.static.%s", name)).(pkgmirror.MirrorService) 85 | s.Serve(state) 86 | 87 | return nil 88 | } 89 | }(name)) 90 | } 91 | } 92 | 93 | func ConfigureHttp(name string, conf *pkgmirror.StaticConfig, app *goapp.App) { 94 | staticService := app.Get(fmt.Sprintf("pkgmirror.static.%s", name)).(*StaticService) 95 | 96 | mux := app.Get("mux").(*goji.Mux) 97 | 98 | mux.HandleFuncC(pat.Get(fmt.Sprintf("/static/%s/*", name)), func(ctx context.Context, w http.ResponseWriter, r *http.Request) { 99 | path := r.URL.Path[9+len(name):] 100 | 101 | // this will consumes too much memory with large files. 102 | buf := bytes.NewBuffer([]byte("")) 103 | 104 | if file, err := staticService.WriteArchive(buf, path); err != nil { 105 | code := 500 106 | if err == pkgmirror.ResourceNotFoundError { 107 | code = 404 108 | } 109 | 110 | pkgmirror.SendWithHttpCode(w, code, err.Error()) 111 | } else { 112 | 113 | // copy header 114 | for name := range file.Header { 115 | if name == "Content-Length" { 116 | continue 117 | } 118 | 119 | w.Header().Set(name, file.Header.Get(name)) 120 | } 121 | 122 | w.WriteHeader(200) 123 | 124 | buf.WriteTo(w) 125 | } 126 | }) 127 | } 128 | -------------------------------------------------------------------------------- /mirror/static/static_structs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package static 7 | 8 | import ( 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | type StaticFile struct { 14 | Header http.Header 15 | Url string 16 | DownloadAt time.Time 17 | Size int64 18 | } 19 | -------------------------------------------------------------------------------- /pkgmirror.toml: -------------------------------------------------------------------------------- 1 | DataDir = "./var/data" 2 | CacheDir = "./var/cache" 3 | PublicServer = "http://localhost:8000" 4 | InternalServer = ":8000" 5 | LogDir = "./var/logs" 6 | LogLevel = "info" 7 | 8 | [Composer] 9 | [Composer.packagist] 10 | Server = "https://packagist.org" 11 | Enabled = true 12 | Icon = "https://getcomposer.org/img/logo-composer-transparent.png" 13 | 14 | [Composer.drupal8] 15 | Server = "https://packages.drupal.org/8" 16 | Enabled = true 17 | Icon = "https://www.drupal.org/files/druplicon-small.png" 18 | 19 | [Composer.drupal7] 20 | Server = "https://packages.drupal.org/7" 21 | Enabled = true 22 | Icon = "https://www.drupal.org/files/druplicon-small.png" 23 | 24 | [Npm] 25 | [Npm.npm] 26 | Server = "https://registry.npmjs.org" 27 | Enabled = true 28 | Icon = "https://cldup.com/Rg6WLgqccB.svg" 29 | 30 | [Bower] 31 | [Bower.bower] 32 | Server = "https://registry.bower.io" 33 | Enabled = false 34 | Icon = "https://bower.io/img/bower-logo.svg" 35 | 36 | [Static] 37 | [Static.drupal] 38 | Enabled = true 39 | Server = "https://ftp.drupal.org/files/projects" 40 | Icon = "https://www.drupal.org/files/druplicon-small.png" 41 | 42 | [Git] 43 | [Git.github] 44 | Server = "github.com" 45 | Clone = "git@github.com:{path}" 46 | Enabled = true 47 | Icon = "https://assets-cdn.github.com/images/modules/logos_page/GitHub-Mark.png" 48 | 49 | [Git.drupal] 50 | Server = "git.drupal.org" 51 | Clone = "https://git.drupal.org/{path}" 52 | Enabled = true 53 | Icon = "https://www.drupal.org/files/druplicon-small.png" -------------------------------------------------------------------------------- /service.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | import ( 9 | "github.com/rande/goapp" 10 | ) 11 | 12 | const ( 13 | STATUS_RUNNING = 1 14 | STATUS_HOLD = 2 15 | STATUS_ERROR = 3 16 | ) 17 | 18 | type MirrorService interface { 19 | Init(app *goapp.App) error 20 | Serve(state *goapp.GoroutineState) error 21 | } 22 | 23 | type State struct { 24 | Id string 25 | Status int 26 | Message string 27 | } 28 | -------------------------------------------------------------------------------- /sse_broker.go: -------------------------------------------------------------------------------- 1 | package pkgmirror 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | // code adapted from https://gist.github.com/ismasan/3fb75381cd2deb6bfa9c 9 | type SseBroker struct { 10 | // Events are pushed to this channel by the main events-gathering routine 11 | Notifier chan []byte 12 | 13 | // New client connections 14 | newClients chan chan []byte 15 | 16 | // Closed client connections 17 | closingClients chan chan []byte 18 | 19 | // Client connections registry 20 | clients map[chan []byte]bool 21 | 22 | onConnect func() 23 | } 24 | 25 | func (broker *SseBroker) Handler(rw http.ResponseWriter, req *http.Request) { 26 | // Make sure that the writer supports flushing. 27 | flusher, ok := rw.(http.Flusher) 28 | 29 | if !ok { 30 | http.Error(rw, "Streaming unsupported!", http.StatusInternalServerError) 31 | return 32 | } 33 | 34 | rw.Header().Set("Content-Type", "text/event-stream") 35 | rw.Header().Set("Cache-Control", "no-cache") 36 | rw.Header().Set("Connection", "keep-alive") 37 | rw.Header().Set("Access-Control-Allow-Origin", "*") 38 | 39 | // Each connection registers its own message channel with the Broker's connections registry 40 | messageChan := make(chan []byte) 41 | 42 | // Signal the broker that we have a new connection 43 | broker.newClients <- messageChan 44 | 45 | // Remove this client from the map of connected clients 46 | // when this handler exits. 47 | defer func() { 48 | broker.closingClients <- messageChan 49 | }() 50 | 51 | // Listen to connection close and un-register messageChan 52 | notify := rw.(http.CloseNotifier).CloseNotify() 53 | 54 | go func() { 55 | <-notify 56 | broker.closingClients <- messageChan 57 | }() 58 | 59 | if broker.onConnect != nil { 60 | go broker.onConnect() 61 | } 62 | 63 | for { 64 | // Write to the ResponseWriter 65 | // Server Sent Events compatible 66 | fmt.Fprintf(rw, "data: %s\n\n", <-messageChan) 67 | 68 | // Flush the data immediatly instead of buffering it for later. 69 | flusher.Flush() 70 | } 71 | } 72 | 73 | func (broker *SseBroker) OnConnect(fn func()) { 74 | broker.onConnect = fn 75 | } 76 | 77 | func (broker *SseBroker) Listen() { 78 | for { 79 | select { 80 | case s := <-broker.newClients: 81 | 82 | // A new client has connected. 83 | // Register their message channel 84 | broker.clients[s] = true 85 | case s := <-broker.closingClients: 86 | 87 | // A client has dettached and we want to 88 | // stop sending them messages. 89 | delete(broker.clients, s) 90 | case event := <-broker.Notifier: 91 | // We got a new event from the outside! 92 | // Send event to all connected clients 93 | for clientMessageChan := range broker.clients { 94 | clientMessageChan <- event 95 | } 96 | } 97 | } 98 | } 99 | 100 | func NewSseBroker() *SseBroker { 101 | // Instantiate a broker 102 | return &SseBroker{ 103 | Notifier: make(chan []byte, 1), 104 | newClients: make(chan chan []byte), 105 | closingClients: make(chan chan []byte), 106 | clients: make(map[chan []byte]bool), 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /test/api/api_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package api 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | 12 | //"encoding/json" 13 | 14 | //"github.com/rande/pkgmirror/api" 15 | "github.com/rande/pkgmirror/test" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func Test_Api_Ping(t *testing.T) { 20 | optin := &test.TestOptin{} 21 | 22 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 23 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/api/ping", args.TestServer.URL)) 24 | 25 | assert.NoError(t, err) 26 | assert.Equal(t, 200, res.StatusCode) 27 | assert.Equal(t, []byte("pong"), res.GetBody()) 28 | }) 29 | } 30 | 31 | //func Test_Api_List(t *testing.T) { 32 | // optin := &test.TestOptin{true, true, true, true} 33 | // 34 | // test.RunHttpTest(t, optin, func(args *test.Arguments) { 35 | // res, err := test.RunRequest("GET", fmt.Sprintf("%s/api/mirrors", args.TestServer.URL)) 36 | // 37 | // assert.NoError(t, err) 38 | // assert.Equal(t, 200, res.StatusCode) 39 | // 40 | // mirrors := []*api.ServiceMirror{} 41 | // 42 | // data := res.GetBody() 43 | // 44 | // err = json.Unmarshal(data, &mirrors) 45 | // assert.NoError(t, err) 46 | // 47 | // assert.Equal(t, 4, len(mirrors)) 48 | // }) 49 | //} 50 | -------------------------------------------------------------------------------- /test/mirror/bower_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package mirror 7 | 8 | import ( 9 | "encoding/json" 10 | "fmt" 11 | "testing" 12 | "time" 13 | 14 | "github.com/rande/pkgmirror/mirror/bower" 15 | "github.com/rande/pkgmirror/test" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func Test_Bower_Get_Package(t *testing.T) { 20 | optin := &test.TestOptin{Bower: true} 21 | 22 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 23 | // wait for the synchro to complete 24 | time.Sleep(1 * time.Second) 25 | 26 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/bower/bower/packages/non-existant-package", args.TestServer.URL)) 27 | assert.NoError(t, err) 28 | assert.Equal(t, 404, res.StatusCode) 29 | 30 | res, err = test.RunRequest("GET", fmt.Sprintf("%s/bower/bower/packages/10digit-legal", args.TestServer.URL)) 31 | assert.NoError(t, err) 32 | assert.Equal(t, 200, res.StatusCode) 33 | 34 | v := &bower.Package{} 35 | err = json.Unmarshal(res.GetBody(), v) 36 | 37 | assert.Equal(t, "http://localhost:8000/git/github.com/10digit/legal.git", v.Url) 38 | assert.Equal(t, "10digit-legal", v.Name) 39 | }) 40 | } 41 | 42 | func Test_Bower_Get_Packages(t *testing.T) { 43 | optin := &test.TestOptin{Bower: true} 44 | 45 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 46 | // wait for the synchro to complete 47 | time.Sleep(1 * time.Second) 48 | 49 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/bower/bower/packages", args.TestServer.URL)) 50 | assert.NoError(t, err) 51 | assert.Equal(t, 200, res.StatusCode) 52 | 53 | data := res.GetBody() 54 | 55 | v := make(bower.Packages, 0) 56 | err = json.Unmarshal(data, &v) 57 | 58 | assert.Equal(t, 3, len(v)) 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /test/mirror/composer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package mirror 7 | 8 | import ( 9 | "encoding/json" 10 | "fmt" 11 | "testing" 12 | "time" 13 | 14 | "github.com/rande/pkgmirror/mirror/composer" 15 | "github.com/rande/pkgmirror/test" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func Test_Composer_Get_PackagesJson(t *testing.T) { 20 | optin := &test.TestOptin{Composer: true} 21 | 22 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 23 | // wait for the synchro to complete 24 | time.Sleep(1 * time.Second) 25 | 26 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/composer/packagist/packages.json", args.TestServer.URL)) 27 | 28 | assert.NoError(t, err) 29 | assert.Equal(t, 200, res.StatusCode) 30 | }) 31 | } 32 | 33 | func Test_Composer_Redirect_Package(t *testing.T) { 34 | optin := &test.TestOptin{Composer: true} 35 | 36 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 37 | // wait for the synchro to complete 38 | time.Sleep(1 * time.Second) 39 | 40 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/composer/packagist/p/symfony/framework-standard-edition", args.TestServer.URL)) 41 | 42 | assert.NoError(t, err) 43 | assert.Equal(t, 200, res.StatusCode) 44 | 45 | v := &composer.PackageResult{} 46 | err = json.Unmarshal(res.GetBody(), v) 47 | 48 | assert.NoError(t, err) 49 | 50 | if err != nil { 51 | return 52 | } 53 | 54 | assert.Equal(t, "symfony/framework-standard-edition", v.Packages["symfony/framework-standard-edition"]["2.1.x-dev"].Name) 55 | }) 56 | } 57 | 58 | func Test_Deprecated_Redirect(t *testing.T) { 59 | optin := &test.TestOptin{Composer: true} 60 | 61 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 62 | time.Sleep(1 * time.Second) 63 | 64 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/packagist/packages.json", args.TestServer.URL)) 65 | 66 | assert.NoError(t, err) 67 | assert.Equal(t, 200, res.StatusCode) 68 | }) 69 | } 70 | 71 | func Test_Update_Package(t *testing.T) { 72 | optin := &test.TestOptin{Composer: true} 73 | 74 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 75 | time.Sleep(1 * time.Second) 76 | 77 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/composer/packagist/p/symfony/framework-standard-edition$sha1.json?refresh=1", args.TestServer.URL)) 78 | 79 | assert.NoError(t, err) 80 | assert.Equal(t, 200, res.StatusCode) 81 | 82 | data := res.GetBody() 83 | 84 | fmt.Println(string(data)) 85 | 86 | v := map[string]string{} 87 | err = json.Unmarshal(data, &v) 88 | 89 | assert.Equal(t, "OK", v["status"]) 90 | assert.Equal(t, "Package updated", v["message"]) 91 | }) 92 | } 93 | 94 | func Test_Update_Unknown_Package(t *testing.T) { 95 | optin := &test.TestOptin{Composer: true} 96 | 97 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 98 | time.Sleep(1 * time.Second) 99 | 100 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/composer/packagist/p/foo/bar$sha1.json?refresh=1", args.TestServer.URL)) 101 | 102 | assert.NoError(t, err) 103 | assert.Equal(t, 500, res.StatusCode) 104 | 105 | data := res.GetBody() 106 | 107 | v := map[string]string{} 108 | err = json.Unmarshal(data, &v) 109 | 110 | assert.Equal(t, "KO", v["status"]) 111 | assert.Equal(t, "No value available", v["message"]) 112 | }) 113 | } 114 | -------------------------------------------------------------------------------- /test/mirror/git_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package mirror 7 | 8 | import ( 9 | "fmt" 10 | "os" 11 | "os/exec" 12 | "testing" 13 | 14 | "github.com/rande/pkgmirror/mirror/git" 15 | "github.com/rande/pkgmirror/test" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func Test_Git_Is_Ref_With_Valid_Reference(t *testing.T) { 20 | references := []string{ 21 | "b5e004cc051bf68838f12b8463ac9ca84432ffce", 22 | "0.0.1", 23 | "1.1.1-beta.1", 24 | } 25 | 26 | for _, v := range references { 27 | assert.True(t, git.IS_REF.Match([]byte(v)), v) 28 | } 29 | } 30 | 31 | func Test_Git_Is_Ref_With_Invalid_Reference(t *testing.T) { 32 | references := []string{ 33 | "asds@next", 34 | "%...", 35 | "/%...", 36 | } 37 | 38 | for _, v := range references { 39 | assert.False(t, git.IS_REF.Match([]byte(v)), v) 40 | } 41 | } 42 | 43 | func Test_Git_Clone_Existing_Repo(t *testing.T) { 44 | optin := &test.TestOptin{Git: true} 45 | 46 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 47 | assert.NoError(t, os.RemoveAll("/tmp/pkgmirror/foo")) 48 | 49 | gitService := args.App.Get("pkgmirror.git.local").(*git.GitService) 50 | 51 | url := fmt.Sprintf("%s/git/local/foo.git", args.TestServer.URL) 52 | 53 | cmd := exec.Command(gitService.Config.Binary, "clone", url, "/tmp/pkgmirror/foo") 54 | 55 | assert.NoError(t, cmd.Start()) 56 | assert.NoError(t, cmd.Wait()) 57 | }) 58 | } 59 | 60 | func Test_Git_Clone_Non_Existing_Repo(t *testing.T) { 61 | optin := &test.TestOptin{Git: true} 62 | 63 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 64 | assert.NoError(t, os.RemoveAll("/tmp/pkgmirror/foo")) 65 | 66 | gitService := args.App.Get("pkgmirror.git.local").(*git.GitService) 67 | 68 | assert.False(t, gitService.Has("foobar.git")) 69 | url := fmt.Sprintf("%s/git/local/foobar.git", args.TestServer.URL) 70 | 71 | cmd := exec.Command(gitService.Config.Binary, "clone", url, "/tmp/pkgmirror/foo") 72 | 73 | assert.NoError(t, cmd.Start()) 74 | assert.NoError(t, cmd.Wait()) 75 | 76 | assert.True(t, gitService.Has("foobar.git")) 77 | }) 78 | } 79 | 80 | func Test_Git_Has(t *testing.T) { 81 | optin := &test.TestOptin{Git: true} 82 | 83 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 84 | assert.NoError(t, os.RemoveAll("/tmp/pkgmirror/foo")) 85 | 86 | gitService := args.App.Get("pkgmirror.git.local").(*git.GitService) 87 | 88 | assert.True(t, gitService.Has("foo.git")) 89 | }) 90 | } 91 | 92 | func Test_Git_Download_Master_Archive(t *testing.T) { 93 | optin := &test.TestOptin{Git: true} 94 | 95 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 96 | res, _ := test.RunRequest("GET", fmt.Sprintf("%s/git/local/foo/master.zip", args.TestServer.URL)) 97 | 98 | assert.Equal(t, 200, res.StatusCode) 99 | assert.Equal(t, "application/zip", res.Header.Get("Content-Type")) 100 | }) 101 | } 102 | 103 | func Test_Git_Download_Tag_Archive(t *testing.T) { 104 | 105 | optin := &test.TestOptin{Git: true} 106 | 107 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 108 | res, _ := test.RunRequest("GET", fmt.Sprintf("%s/git/local/foo/0.0.1.zip", args.TestServer.URL)) 109 | 110 | assert.Equal(t, 200, res.StatusCode) 111 | assert.Equal(t, "application/zip", res.Header.Get("Content-Type")) 112 | }) 113 | } 114 | 115 | func Test_Git_Download_Sha1_Archive(t *testing.T) { 116 | optin := &test.TestOptin{Git: true} 117 | 118 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 119 | res, _ := test.RunRequest("GET", fmt.Sprintf("%s/git/local/foo/9b9cc9573693611badb397b5d01a1e6645704da7.zip", args.TestServer.URL)) 120 | 121 | assert.Equal(t, 200, res.StatusCode) 122 | assert.Equal(t, "application/zip", res.Header.Get("Content-Type")) 123 | }) 124 | } 125 | 126 | func Test_Git_Download_Non_Existant_Archive(t *testing.T) { 127 | optin := &test.TestOptin{Git: true} 128 | 129 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 130 | res, _ := test.RunRequest("GET", fmt.Sprintf("%s/git/local/bar/master.zip", args.TestServer.URL)) 131 | 132 | assert.Equal(t, 500, res.StatusCode) 133 | assert.Equal(t, "application/json", res.Header.Get("Content-Type")) 134 | }) 135 | } 136 | -------------------------------------------------------------------------------- /test/mirror/npm_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package mirror 7 | 8 | import ( 9 | "encoding/json" 10 | "fmt" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/rande/pkgmirror/mirror/npm" 15 | "github.com/rande/pkgmirror/test" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func Test_Npm_Get_Standard_Package(t *testing.T) { 20 | 21 | optin := &test.TestOptin{Npm: true} 22 | 23 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 24 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/npm/angular-oauth", args.MockedServer.URL)) // Mocked server 25 | 26 | assert.NoError(t, err) 27 | assert.Equal(t, 200, res.StatusCode) 28 | 29 | res, err = test.RunRequest("GET", fmt.Sprintf("%s/npm/npm/non-existant-package", args.TestServer.URL)) 30 | assert.NoError(t, err) 31 | assert.Equal(t, 404, res.StatusCode) 32 | 33 | res, err = test.RunRequest("GET", fmt.Sprintf("%s/npm/npm/angular-nvd3-nb", args.TestServer.URL)) 34 | assert.NoError(t, err) 35 | assert.Equal(t, 200, res.StatusCode) 36 | 37 | v := &npm.FullPackageDefinition{} 38 | err = json.Unmarshal(res.GetBody(), v) 39 | 40 | assert.Equal(t, "angular-nvd3-nb", v.Name) 41 | assert.Equal(t, "http://localhost:8000/npm/npm/angular-nvd3-nb/-/angular-nvd3-nb-1.0.5-nb.tgz", v.Versions["1.0.5-nb"].Dist.Tarball) 42 | 43 | // download tar file 44 | url := strings.Replace(v.Versions["1.0.5-nb"].Dist.Tarball, "http://localhost:8000", args.TestServer.URL, -1) 45 | 46 | res, err = test.RunRequest("GET", url) 47 | 48 | assert.NoError(t, err) 49 | assert.Equal(t, 200, res.StatusCode) 50 | assert.Equal(t, 19497, len(res.GetBody())) 51 | }) 52 | } 53 | func Test_Npm_Download_Scoped_Package_Archive(t *testing.T) { 54 | 55 | optin := &test.TestOptin{Npm: true} 56 | 57 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 58 | url := strings.Replace("http://localhost:8000/npm/npm/@types/react/-/react-0.0.0.tgz", "http://localhost:8000", args.TestServer.URL, -1) 59 | 60 | res, err := test.RunRequest("GET", url) 61 | 62 | assert.NoError(t, err) 63 | assert.Equal(t, 200, res.StatusCode) 64 | assert.Equal(t, 25276, len(res.GetBody())) 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /test/mirror/static_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package mirror 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | 12 | "github.com/rande/pkgmirror/test" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func Test_Static_Get_Valid_File(t *testing.T) { 17 | optin := &test.TestOptin{Static: true} 18 | 19 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 20 | // check the original file exist on the remote server 21 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/static/file.txt", args.MockedServer.URL)) 22 | 23 | assert.NoError(t, err) 24 | assert.Equal(t, 200, res.StatusCode) 25 | assert.Equal(t, "This is a sample test file.", string(res.GetBody())) 26 | 27 | // get the proxied file 28 | res, err = test.RunRequest("GET", fmt.Sprintf("%s/static/static/file.txt", args.TestServer.URL)) 29 | 30 | assert.NoError(t, err) 31 | assert.Equal(t, 200, res.StatusCode) 32 | assert.Equal(t, "This is a sample test file.", string(res.GetBody())) 33 | 34 | // test if the second time the code work fine (using the cache) 35 | res, err = test.RunRequest("GET", fmt.Sprintf("%s/static/static/file.txt", args.TestServer.URL)) 36 | 37 | assert.NoError(t, err) 38 | assert.Equal(t, 200, res.StatusCode) 39 | assert.Equal(t, "This is a sample test file.", string(res.GetBody())) 40 | }) 41 | } 42 | 43 | func Test_Static_Get_Invalid_File(t *testing.T) { 44 | optin := &test.TestOptin{Static: true} 45 | 46 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 47 | // check the original file exist on the remote server 48 | res, err := test.RunRequest("GET", fmt.Sprintf("%s/static/static/non-existant-file.txt", args.TestServer.URL)) 49 | 50 | assert.NoError(t, err) 51 | assert.Equal(t, 404, res.StatusCode) 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /test/test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package test 7 | 8 | import ( 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "math/rand" 13 | "net/http" 14 | "net/http/httptest" 15 | "net/url" 16 | "os" 17 | "os/exec" 18 | godebug "runtime/debug" 19 | "strings" 20 | "testing" 21 | "time" 22 | 23 | log "github.com/Sirupsen/logrus" 24 | "github.com/rande/goapp" 25 | "github.com/rande/pkgmirror" 26 | "github.com/rande/pkgmirror/api" 27 | "github.com/rande/pkgmirror/mirror/bower" 28 | "github.com/rande/pkgmirror/mirror/composer" 29 | "github.com/rande/pkgmirror/mirror/git" 30 | "github.com/rande/pkgmirror/mirror/npm" 31 | "github.com/rande/pkgmirror/mirror/static" 32 | "github.com/stretchr/testify/assert" 33 | "goji.io" 34 | ) 35 | 36 | var src = rand.NewSource(time.Now().UnixNano()) 37 | 38 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 39 | const ( 40 | letterIdxBits = 6 // 6 bits to represent a letter index 41 | letterIdxMask = 1<= 0; { 49 | if remain == 0 { 50 | cache, remain = src.Int63(), letterIdxMax 51 | } 52 | if idx := int(cache & letterIdxMask); idx < len(letterBytes) { 53 | b[i] = letterBytes[idx] 54 | i-- 55 | } 56 | cache >>= letterIdxBits 57 | remain-- 58 | } 59 | 60 | return string(b) 61 | } 62 | 63 | type Response struct { 64 | *http.Response 65 | RawBody []byte 66 | bodyRead bool 67 | } 68 | 69 | type Arguments struct { 70 | TestServer *httptest.Server 71 | MockedServer *httptest.Server 72 | App *goapp.App 73 | T *testing.T 74 | } 75 | 76 | func (r Response) GetBody() []byte { 77 | var err error 78 | 79 | if !r.bodyRead { 80 | r.RawBody, err = ioutil.ReadAll(r.Body) 81 | r.Body.Close() 82 | if err != nil { 83 | log.Fatalf("Fail to GetBody of with error: %s", err) 84 | } 85 | 86 | r.bodyRead = true 87 | } 88 | 89 | return r.RawBody 90 | } 91 | 92 | func RunRequest(method string, path string, options ...interface{}) (*Response, error) { 93 | var body interface{} 94 | var headers map[string]string 95 | 96 | if len(options) > 0 { 97 | body = options[0] 98 | } 99 | 100 | if len(options) > 1 { 101 | headers = options[1].(map[string]string) 102 | } 103 | 104 | client := &http.Client{} 105 | var req *http.Request 106 | var err error 107 | 108 | switch v := body.(type) { 109 | case nil: 110 | req, err = http.NewRequest(method, path, nil) 111 | case *strings.Reader: 112 | req, err = http.NewRequest(method, path, v) 113 | case io.Reader: 114 | req, err = http.NewRequest(method, path, v) 115 | 116 | case url.Values: 117 | req, err = http.NewRequest(method, path, strings.NewReader(v.Encode())) 118 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 119 | 120 | default: 121 | panic(fmt.Sprintf("please add a new test case for %T", body)) 122 | } 123 | 124 | if headers != nil { 125 | for name, value := range headers { 126 | req.Header.Set(name, value) 127 | } 128 | } 129 | 130 | if err != nil { 131 | panic(err) 132 | } 133 | 134 | resp, err := client.Do(req) 135 | 136 | return &Response{Response: resp}, err 137 | } 138 | 139 | type TestOptin struct { 140 | Composer bool 141 | Npm bool 142 | Git bool 143 | Bower bool 144 | Static bool 145 | } 146 | 147 | func RunHttpTest(t *testing.T, optin *TestOptin, f func(args *Arguments)) { 148 | 149 | baseFolder := fmt.Sprintf("%s/pkgmirror/%s", os.TempDir(), RandStringBytesMaskImprSrc(10)) 150 | 151 | folders := []string{ 152 | "%s/data/npm", 153 | "%s/data/composer", 154 | "%s/data/bower", 155 | "%s/data/git", 156 | "%s/data/static", 157 | "%s/cache/git", 158 | } 159 | 160 | for _, f := range folders { 161 | if err := os.MkdirAll(fmt.Sprintf(f, baseFolder), 0755); err != nil { 162 | assert.NoError(t, err) 163 | return 164 | } 165 | } 166 | 167 | if optin.Git { 168 | targets := []string{ 169 | "data/git/local/foo.git", // use as already available repository 170 | "data/git/source/foobar.git", // use as a source for cloning missing repository 171 | } 172 | 173 | for _, target := range targets { 174 | cmd := exec.Command("git", strings.Split(fmt.Sprintf("clone --mirror ../../fixtures/git/foo.bare %s/%s", baseFolder, target), " ")...) 175 | 176 | if err := cmd.Start(); err != nil { 177 | assert.NoError(t, err) 178 | 179 | return 180 | } 181 | 182 | if err := cmd.Wait(); err != nil { 183 | assert.NoError(t, err) 184 | 185 | return 186 | } 187 | } 188 | } 189 | 190 | l := goapp.NewLifecycle() 191 | 192 | fs := http.FileServer(http.Dir("../../fixtures/mock")) 193 | 194 | ms := httptest.NewServer(fs) 195 | 196 | config := &pkgmirror.Config{ 197 | DataDir: fmt.Sprintf("%s/data", baseFolder), 198 | CacheDir: fmt.Sprintf("%s/cache", baseFolder), 199 | PublicServer: "http://localhost:8000", 200 | InternalServer: "127.0.0.1:8000", 201 | LogLevel: "debug", 202 | Git: map[string]*pkgmirror.GitConfig{ 203 | "local": { 204 | Server: "local", 205 | Enabled: optin.Git, 206 | Icon: "https://assets-cdn.github.com/images/modules/logos_page/GitHub-Mark.png", 207 | Clone: fmt.Sprintf("file://%s/data/git/source/{path}", baseFolder), 208 | }, 209 | }, 210 | Npm: map[string]*pkgmirror.NpmConfig{ 211 | "npm": { 212 | Server: ms.URL + "/npm", 213 | Enabled: optin.Npm, 214 | Icon: "https://cldup.com/Rg6WLgqccB.svg", 215 | }, 216 | }, 217 | Composer: map[string]*pkgmirror.ComposerConfig{ 218 | "packagist": { 219 | Server: ms.URL + "/composer", 220 | Enabled: optin.Composer, 221 | Icon: "https://getcomposer.org/img/logo-composer-transparent.png", 222 | }, 223 | }, 224 | Bower: map[string]*pkgmirror.BowerConfig{ 225 | "bower": { 226 | Server: ms.URL + "/bower", 227 | Enabled: optin.Bower, 228 | Icon: "https://bower.io/img/bower-logo.svg", 229 | }, 230 | }, 231 | Static: map[string]*pkgmirror.StaticConfig{ 232 | "static": { 233 | Server: ms.URL + "/static", 234 | Enabled: optin.Static, 235 | Icon: "", 236 | }, 237 | }, 238 | } 239 | 240 | app, err := pkgmirror.GetApp(config, l) 241 | assert.NoError(t, err) 242 | assert.NotNil(t, app) 243 | 244 | api.ConfigureApp(config, l) 245 | git.ConfigureApp(config, l) 246 | npm.ConfigureApp(config, l) 247 | composer.ConfigureApp(config, l) 248 | bower.ConfigureApp(config, l) 249 | static.ConfigureApp(config, l) 250 | 251 | l.Run(func(app *goapp.App, state *goapp.GoroutineState) error { 252 | mux := app.Get("mux").(*goji.Mux) 253 | 254 | ts := httptest.NewServer(mux) 255 | 256 | defer func() { 257 | state.Out <- goapp.Control_Stop 258 | 259 | ts.Close() 260 | 261 | if r := recover(); r != nil { 262 | assert.Equal(t, false, true, fmt.Sprintf("RunHttpTest: Panic recovered, message=%s\n\n%s", r, string(godebug.Stack()[:]))) 263 | } 264 | }() 265 | 266 | f(&Arguments{ 267 | TestServer: ts, 268 | MockedServer: ms, 269 | T: t, 270 | App: app, 271 | }) 272 | 273 | return nil 274 | }) 275 | 276 | l.Go(app) 277 | } 278 | -------------------------------------------------------------------------------- /test/tools/invalid_struct_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package tools 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | 12 | "github.com/rande/pkgmirror" 13 | "github.com/rande/pkgmirror/test" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func Test_Invalid_Struct(t *testing.T) { 18 | optin := &test.TestOptin{} 19 | 20 | test.RunHttpTest(t, optin, func(args *test.Arguments) { 21 | 22 | fake := &struct { 23 | Foo string 24 | }{} 25 | 26 | err := pkgmirror.LoadRemoteStruct(fmt.Sprintf("%s/invalid.json", args.MockedServer.URL), fake) 27 | 28 | assert.Error(t, err) 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | import ( 9 | "bytes" 10 | "compress/gzip" 11 | "encoding/json" 12 | "io" 13 | "io/ioutil" 14 | "net/http" 15 | "os" 16 | "sync" 17 | ) 18 | 19 | func LoadStruct(file string, v interface{}) error { 20 | r, err := os.Open(file) 21 | 22 | if err != nil { 23 | return err 24 | } 25 | 26 | buf := bytes.NewBuffer([]byte("")) 27 | buf.ReadFrom(r) 28 | 29 | err = json.Unmarshal(buf.Bytes(), v) 30 | 31 | if err != nil { 32 | return err 33 | } 34 | 35 | return nil 36 | } 37 | 38 | func LoadRemoteStruct(url string, v interface{}) error { 39 | cpt := 0 40 | for { 41 | if err := loadRemoteStruct(url, v); err != nil { 42 | cpt++ 43 | 44 | if cpt > 5 { 45 | return err 46 | } 47 | } else { 48 | return nil 49 | } 50 | } 51 | } 52 | 53 | func loadRemoteStruct(url string, v interface{}) error { 54 | resp, err := http.Get(url) 55 | 56 | if err != nil { 57 | return err 58 | } 59 | 60 | defer resp.Body.Close() 61 | 62 | buf := bytes.NewBuffer([]byte("")) 63 | 64 | _, err = io.Copy(buf, resp.Body) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | err = json.Unmarshal(buf.Bytes(), v) 70 | 71 | if err != nil { 72 | return err 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func SendWithHttpCode(res http.ResponseWriter, code int, message string) { 79 | res.Header().Set("Content-Type", "application/json") 80 | 81 | res.WriteHeader(code) 82 | 83 | status := "KO" 84 | if code >= 200 && code < 300 { 85 | status = "OK" 86 | } 87 | 88 | data, _ := json.Marshal(map[string]string{ 89 | "status": status, 90 | "message": message, 91 | }) 92 | 93 | res.Write(data) 94 | } 95 | 96 | func Compress(data []byte) ([]byte, error) { 97 | var buf bytes.Buffer 98 | 99 | if writer, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed); err != nil { 100 | return nil, err 101 | } else if _, err := writer.Write(data); err != nil { 102 | return nil, err 103 | } else { 104 | writer.Close() 105 | 106 | return buf.Bytes(), nil 107 | } 108 | } 109 | 110 | func Decompress(data []byte) ([]byte, error) { 111 | if reader, err := gzip.NewReader(bytes.NewBuffer(data)); err != nil { 112 | return nil, err 113 | } else if data, err := ioutil.ReadAll(reader); err != nil { 114 | return nil, err 115 | } else if err := reader.Close(); err != nil { 116 | return nil, err 117 | } else { 118 | return data, nil 119 | } 120 | } 121 | 122 | func Unmarshal(data []byte, v interface{}) error { 123 | if data, err := Decompress(data); err != nil { 124 | return err 125 | } else if err := json.Unmarshal(data, v); err != nil { 126 | return err 127 | } else { 128 | return nil 129 | } 130 | } 131 | 132 | func Marshal(v interface{}) ([]byte, error) { 133 | if data, err := json.Marshal(v); err != nil { 134 | return nil, err 135 | } else if data, err := Compress(data); err != nil { 136 | return nil, err 137 | } else { 138 | return data, nil 139 | } 140 | } 141 | 142 | func NewWorkerManager(process int, processCallback FuncProcess) *workerManager { 143 | return &workerManager{ 144 | count: process, 145 | processCallback: processCallback, 146 | add: make(chan interface{}), 147 | result: make(chan interface{}), 148 | wg: sync.WaitGroup{}, 149 | resultDone: make(chan bool), 150 | } 151 | } 152 | 153 | type FuncProcess func(id int, data <-chan interface{}, result chan interface{}) 154 | type FuncResult func(raw interface{}) 155 | 156 | type workerManager struct { 157 | add chan interface{} 158 | result chan interface{} 159 | count int 160 | lock bool 161 | processCallback FuncProcess 162 | resultCallback FuncResult 163 | wg sync.WaitGroup 164 | resultDone chan bool 165 | } 166 | 167 | func (dm *workerManager) Start() { 168 | // force count to the number of worker 169 | dm.wg.Add(dm.count) 170 | 171 | for i := 0; i < dm.count; i++ { 172 | go func(id int) { 173 | dm.processCallback(id, dm.add, dm.result) 174 | dm.wg.Done() 175 | }(i) 176 | } 177 | 178 | if dm.resultCallback != nil { 179 | // if we get result increment wg by one 180 | go func() { 181 | for raw := range dm.result { 182 | dm.resultCallback(raw) 183 | } 184 | 185 | dm.resultDone <- true 186 | }() 187 | } 188 | } 189 | 190 | func (dm *workerManager) Add(raw interface{}) { 191 | dm.add <- raw 192 | } 193 | 194 | func (dm *workerManager) Wait() { 195 | // close task related actions 196 | close(dm.add) // close for range loop 197 | dm.wg.Wait() // wait for other to 198 | 199 | // close result related actions 200 | close(dm.result) 201 | 202 | if dm.resultCallback != nil { 203 | <-dm.resultDone 204 | } 205 | } 206 | 207 | func (dm *workerManager) ResultCallback(fn FuncResult) { 208 | dm.resultCallback = fn 209 | } 210 | 211 | func Serialize(w io.Writer, data interface{}) error { 212 | encoder := json.NewEncoder(w) 213 | err := encoder.Encode(data) 214 | 215 | return err 216 | } 217 | 218 | func GetStateChannel(id string, primary chan State) chan State { 219 | ch := make(chan State) 220 | 221 | go func() { 222 | for { 223 | select { 224 | case s := <-ch: 225 | s.Id = id 226 | primary <- s 227 | } 228 | 229 | } 230 | }() 231 | 232 | return ch 233 | } 234 | -------------------------------------------------------------------------------- /tools_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2016-present Thomas Rabaix . 2 | // 3 | // Use of this source code is governed by an MIT-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package pkgmirror 7 | 8 | import ( 9 | "sync/atomic" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test_Compress(t *testing.T) { 16 | d := []byte("Hello") 17 | 18 | c, err := Compress(d) 19 | 20 | assert.NoError(t, err) 21 | assert.True(t, len(c) > 0) 22 | } 23 | 24 | func Test_Compress_EmptyData(t *testing.T) { 25 | d := []byte("") 26 | 27 | c, err := Compress(d) 28 | 29 | assert.NoError(t, err) 30 | assert.True(t, len(c) > 0) 31 | } 32 | 33 | func Test_Decompress(t *testing.T) { 34 | c, err := Compress([]byte("Hello")) 35 | assert.NoError(t, err) 36 | 37 | d, err := Decompress(c) 38 | assert.NoError(t, err) 39 | assert.Equal(t, []byte("Hello"), d) 40 | } 41 | 42 | func Test_WorkerManager_WorkerNumber(t *testing.T) { 43 | // should be called 10 times 44 | var cpt int32 45 | 46 | m := NewWorkerManager(10, func(id int, data <-chan interface{}, result chan interface{}) { 47 | atomic.AddInt32(&cpt, 1) 48 | }) 49 | 50 | m.Start() 51 | 52 | m.Wait() 53 | 54 | assert.Equal(t, cpt, int32(10)) 55 | } 56 | 57 | type chnStruct struct { 58 | v int32 59 | } 60 | 61 | func Test_WorkerManager_DataIn(t *testing.T) { 62 | // should be called 10 times 63 | var cpt int32 64 | 65 | m := NewWorkerManager(5, func(id int, data <-chan interface{}, result chan interface{}) { 66 | for raw := range data { 67 | atomic.AddInt32(&cpt, raw.(chnStruct).v) 68 | } 69 | }) 70 | 71 | m.Start() 72 | 73 | m.Add(chnStruct{v: 5}) 74 | m.Add(chnStruct{v: 5}) 75 | m.Add(chnStruct{v: 5}) 76 | 77 | m.Wait() 78 | 79 | assert.Equal(t, cpt, int32(15)) 80 | } 81 | 82 | func Test_WorkerManager_Result(t *testing.T) { 83 | var cpt int32 84 | 85 | m := NewWorkerManager(5, func(id int, data <-chan interface{}, result chan interface{}) { 86 | for raw := range data { 87 | result <- raw 88 | } 89 | }) 90 | 91 | m.ResultCallback(func(raw interface{}) { 92 | cpt += raw.(chnStruct).v 93 | }) 94 | 95 | m.Start() 96 | 97 | m.Add(chnStruct{v: 5}) 98 | m.Add(chnStruct{v: 5}) 99 | m.Add(chnStruct{v: 5}) 100 | 101 | m.Wait() 102 | 103 | assert.Equal(t, int32(15), cpt) 104 | } 105 | --------------------------------------------------------------------------------