├── OSSMETADATA ├── pkg ├── metrics │ └── stats │ │ ├── sender_test.go │ │ ├── config.go │ │ └── sender.go ├── history │ ├── hist.go │ └── history.txt ├── repository │ ├── redirect.go │ ├── revinfo.go │ ├── blob.go │ ├── blob_test.go │ ├── parse_test.go │ └── parse.go ├── netservice │ ├── hostname.go │ ├── address.go │ └── address_test.go ├── configutil │ ├── format.go │ ├── format_test.go │ ├── load.go │ └── load_test.go ├── coordinates │ ├── ranges.go │ ├── modules_test.go │ └── modules.go ├── since │ └── since.go ├── webutil │ ├── utils.go │ ├── middleware.go │ └── middleware_test.go ├── clients │ ├── payloads │ │ ├── heartbeat.go │ │ └── configuration.go │ ├── registry │ │ └── modreq.go │ └── zips │ │ ├── client-upstream.go │ │ ├── client-proxy_test.go │ │ ├── rewrite_test.go │ │ └── http.go ├── database │ └── db.go ├── setup │ └── dsn.go └── upstream │ ├── request.go │ ├── request_test.go │ └── go-get.go ├── registry ├── internal │ ├── service │ │ ├── init_test.go │ │ ├── registry.go │ │ └── init.go │ ├── web │ │ ├── common.go │ │ ├── history_test.go │ │ ├── history.go │ │ ├── router_test.go │ │ ├── about_test.go │ │ ├── about.go │ │ ├── blocks.go │ │ ├── redirects.go │ │ ├── v1_registry.go │ │ ├── v1_registry_list_test.go │ │ ├── mods_list.go │ │ ├── v1_startup.go │ │ ├── v1_heartbeat.go │ │ ├── home.go │ │ ├── router.go │ │ ├── mods_show.go │ │ ├── mods_find.go │ │ ├── v1_registry_list.go │ │ └── mods_add.go │ ├── proxies │ │ ├── prune.go │ │ └── prune_test.go │ ├── data │ │ ├── store.go │ │ ├── sql.go │ │ └── pokes.go │ └── tools │ │ └── finder │ │ ├── finder_test.go │ │ ├── github_test.go │ │ ├── finder.go │ │ └── github.go ├── static │ ├── img │ │ └── favicon.ico │ ├── html │ │ ├── blocks.html │ │ ├── redirects.html │ │ ├── mods_list.html │ │ ├── mods_show.html │ │ ├── layout.html │ │ ├── about.html │ │ ├── mods_add.html │ │ ├── navbar.html │ │ └── home.html │ └── css │ │ └── registry.css ├── server.go └── config │ ├── config.go │ └── config_test.go ├── hack ├── sql │ ├── mysql-reg │ │ ├── passwords.sql │ │ └── modproxdb.sql │ └── mysql-prox │ │ └── modproxdb.sql ├── connect-mysql-proxy.sh ├── connect-mysql-registry.sh ├── configs │ ├── registry-local.mysql.json │ └── proxy-local.json └── docker-compose.yaml ├── tools.go ├── .gitignore ├── cmd ├── modprox-proxy │ ├── run-dev.sh │ └── main.go └── modprox-registry │ ├── run-dev.sh │ └── main.go ├── .travis.yml ├── proxy ├── server.go ├── internal │ ├── web │ │ ├── common_test.go │ │ ├── history.go │ │ ├── api_problems.go │ │ ├── mod_file.go │ │ ├── mod_info.go │ │ ├── mod_zip.go │ │ ├── mod_list.go │ │ ├── output │ │ │ └── write.go │ │ ├── common.go │ │ ├── mod_rm.go │ │ └── router.go │ ├── modules │ │ ├── store │ │ │ ├── zipstore.go │ │ │ └── fs.go │ │ ├── get │ │ │ ├── request_test.go │ │ │ ├── request.go │ │ │ └── downloader.go │ │ └── bg │ │ │ └── worker.go │ ├── status │ │ ├── heartbeat │ │ │ ├── loop.go │ │ │ └── sender.go │ │ └── startup │ │ │ ├── sender.go │ │ │ └── sender_test.go │ ├── service │ │ └── proxy.go │ └── problems │ │ ├── tracker_test.go │ │ └── tracker.go └── config │ └── config.go ├── go.mod ├── LICENSE ├── CODE_OF_CONDUCT.md └── README.md /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active -------------------------------------------------------------------------------- /pkg/metrics/stats/sender_test.go: -------------------------------------------------------------------------------- 1 | package stats 2 | -------------------------------------------------------------------------------- /registry/internal/service/init_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // todo: write tests 4 | -------------------------------------------------------------------------------- /registry/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modprox/mp/HEAD/registry/static/img/favicon.ico -------------------------------------------------------------------------------- /hack/sql/mysql-reg/passwords.sql: -------------------------------------------------------------------------------- 1 | grant all privileges on *.* to 'docker'@'%' with grant option; 2 | flush privileges; 3 | -------------------------------------------------------------------------------- /pkg/history/hist.go: -------------------------------------------------------------------------------- 1 | package history 2 | 3 | //go:generate go run gophers.dev/cmds/petrify/v5/cmd/petrify -pkg history -o generated.go . 4 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | //+build tools 2 | 3 | package modprox 4 | 5 | import ( 6 | _ "github.com/gojuno/minimock/v3" 7 | 8 | _ "gophers.dev/cmds/petrify/v5" 9 | ) 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmd/modprox-registry/modprox-registry 2 | cmd/modprox-proxy/modprox-proxy 3 | 4 | **/cover.out 5 | **/generated.go 6 | hack/configs/*indeed.json 7 | /.idea 8 | -------------------------------------------------------------------------------- /pkg/repository/redirect.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | type Redirect struct { 4 | Original string `json:"original"` 5 | Substitution string `json:"substitution"` 6 | } 7 | -------------------------------------------------------------------------------- /pkg/metrics/stats/config.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import "oss.indeed.com/go/modprox/pkg/netservice" 4 | 5 | type Statsd struct { 6 | Agent netservice.Instance `json:"agent"` 7 | } 8 | -------------------------------------------------------------------------------- /hack/connect-mysql-proxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | mysql \ 6 | --protocol=tcp \ 7 | --host=localhost \ 8 | --port=3307 \ 9 | --user=docker \ 10 | --password=docker \ 11 | --database=modproxdb-prox 12 | -------------------------------------------------------------------------------- /hack/connect-mysql-registry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | mysql \ 6 | --protocol=tcp \ 7 | --host=localhost \ 8 | --port=3306 \ 9 | --user=docker \ 10 | --password=docker \ 11 | --database=modproxdb-reg 12 | -------------------------------------------------------------------------------- /cmd/modprox-proxy/run-dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | set -x 5 | 6 | go generate 7 | go build 8 | 9 | if [ ${#} -eq 0 ]; then 10 | ./modprox-proxy ../../hack/configs/proxy-local.json 11 | else 12 | ./modprox-proxy "${1}" 13 | fi 14 | -------------------------------------------------------------------------------- /registry/server.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "oss.indeed.com/go/modprox/registry/config" 5 | "oss.indeed.com/go/modprox/registry/internal/service" 6 | ) 7 | 8 | func Start(config config.Configuration) { 9 | service.NewRegistry(config).Run() 10 | } 11 | -------------------------------------------------------------------------------- /pkg/netservice/hostname.go: -------------------------------------------------------------------------------- 1 | package netservice 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // Hostname returns the hostname or panics. 8 | func Hostname() string { 9 | hostname, err := os.Hostname() 10 | if err != nil { 11 | panic(err) 12 | } 13 | return hostname 14 | } 15 | -------------------------------------------------------------------------------- /registry/static/html/blocks.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

manage allow / block lists

6 |
7 |
8 | not implemented yet 9 |
10 |
11 | {{end}} -------------------------------------------------------------------------------- /pkg/netservice/address.go: -------------------------------------------------------------------------------- 1 | package netservice 2 | 3 | import "fmt" 4 | 5 | type Instance struct { 6 | Address string `json:"address"` 7 | Port int `json:"port"` 8 | } 9 | 10 | func (s Instance) String() string { 11 | return fmt.Sprintf("[%s:%d]", s.Address, s.Port) 12 | } 13 | -------------------------------------------------------------------------------- /cmd/modprox-registry/run-dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | go clean 6 | go generate 7 | go build 8 | 9 | if [[ ${#} -eq 1 ]]; then 10 | configfile="${1}" 11 | else 12 | configfile="../../hack/configs/registry-local.mysql.json" 13 | fi 14 | 15 | ./modprox-registry ${configfile} 16 | -------------------------------------------------------------------------------- /registry/static/html/redirects.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

Manage Upstream Redirects

6 |
7 |
8 |
9 |

blah blah blah

10 |
11 |
12 | {{end}} 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.12.x 5 | 6 | install: 7 | - echo "do nothing for install step" 8 | 9 | script: 10 | - go generate ./... 11 | - go build ./... 12 | - go vet ./... 13 | - go test -race ./... 14 | 15 | services: 16 | - mysql 17 | 18 | before_script: 19 | - mysql -e 'CREATE DATABASE modproxdb;' 20 | -------------------------------------------------------------------------------- /proxy/server.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "oss.indeed.com/go/modprox/proxy/config" 5 | "oss.indeed.com/go/modprox/proxy/internal/service" 6 | ) 7 | 8 | // Start a proxy service instance parameterized by the given Configuration. 9 | func Start(configuration config.Configuration) { 10 | service.NewProxy(configuration).Run() 11 | } 12 | -------------------------------------------------------------------------------- /pkg/configutil/format.go: -------------------------------------------------------------------------------- 1 | package configutil 2 | 3 | import "encoding/json" 4 | 5 | // Format the given configuration c into formatted JSON 6 | // with 2-space indentation level. 7 | func Format(c interface{}) string { 8 | bs, err := json.MarshalIndent(c, "", " ") 9 | if err != nil { 10 | panic(err) 11 | } 12 | return string(bs) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/history/history.txt: -------------------------------------------------------------------------------- 1 | Change history 2 | ============== 3 | 4 | Version 0.1.2 5 | ------------- 6 | 7 | https://github.com/modprox/mp/issues/136: Add /history endpoint 8 | https://github.com/modprox/mp/issues/137: fix .rm suffix 9 | 10 | 11 | Version 0.1.1 12 | ------------- 13 | 14 | https://github.com/modprox/mp/issues/122: proxy does not handle modules using v2 directory convention 15 | 16 | -------------------------------------------------------------------------------- /pkg/netservice/address_test.go: -------------------------------------------------------------------------------- 1 | package netservice 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func Test_String(t *testing.T) { 10 | try := func(input Instance, exp string) { 11 | output := input.String() 12 | require.Equal(t, exp, output) 13 | } 14 | 15 | try(Instance{ 16 | Address: "1.1.1.1", 17 | Port: 1111, 18 | }, "[1.1.1.1:1111]") 19 | } 20 | -------------------------------------------------------------------------------- /pkg/configutil/format_test.go: -------------------------------------------------------------------------------- 1 | package configutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func Test_Format(t *testing.T) { 10 | config := struct { 11 | Foo string `json:"foo"` 12 | Bar int `json:"bar"` 13 | }{ 14 | Foo: "red", 15 | Bar: 8, 16 | } 17 | 18 | formatted := Format(config) 19 | require.JSONEq(t, `{"foo":"red", "bar":8}`, formatted) 20 | } 21 | -------------------------------------------------------------------------------- /registry/internal/web/common.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "bufio" 5 | "strings" 6 | ) 7 | 8 | func linesOfText(text string) []string { 9 | lines := make([]string, 0, 1) 10 | scanner := bufio.NewScanner(strings.NewReader(text)) 11 | for scanner.Scan() { 12 | line := strings.TrimSpace(scanner.Text()) 13 | if line != "" { 14 | lines = append(lines, scanner.Text()) 15 | } 16 | } 17 | return lines 18 | } 19 | -------------------------------------------------------------------------------- /pkg/coordinates/ranges.go: -------------------------------------------------------------------------------- 1 | package coordinates 2 | 3 | // A RangeID represents a sequential list of IDs. The two 4 | // boundary numbers are inclusive. A RangeID of [3, 7] implies 5 | // the list of IDs: {3, 4, 5, 6, 7} (in incrementing order). 6 | type RangeID [2]int64 7 | 8 | // A RangeIDs represents an increasing list of RangeID. A 9 | // RangeIDs of [[2, 4], [8, 8], [13, 14]] implies the list of IDs: 10 | // {2, 3, 4, 8, 13, 14} (in increasing order). 11 | type RangeIDs [][2]int64 12 | -------------------------------------------------------------------------------- /proxy/internal/web/common_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func Test_demangle(t *testing.T) { 10 | try := func(input, exp string) { 11 | output := demangle(input) 12 | require.Equal(t, exp, output) 13 | } 14 | 15 | try("", "") 16 | try("foo/bar", "foo/bar") 17 | try("github.com/!burnt!sushi/toml", "github.com/BurntSushi/toml") 18 | try("foo/bar!", "foo/bar!") 19 | try("foo!A/bar!B", "foo!A/bar!B") 20 | } 21 | -------------------------------------------------------------------------------- /pkg/since/since.go: -------------------------------------------------------------------------------- 1 | // Package since provides convenience functions for computing elapsed time. 2 | package since 3 | 4 | import ( 5 | "time" 6 | ) 7 | 8 | // MS returns the number of milliseconds that have passed since t as int64. 9 | func MS(t time.Time) int64 { 10 | return int64(1000 * time.Since(t).Seconds()) 11 | } 12 | 13 | // MSFrom returns the number of milliseconds that have passed between t and now 14 | // as an int64. 15 | func MSFrom(t, now time.Time) int64 { 16 | return int64(1000 * now.Sub(t).Seconds()) 17 | } 18 | -------------------------------------------------------------------------------- /registry/static/html/mods_list.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

modules in registry

6 |
7 |
8 | 9 | {{range $k, $v := .Mods}} 10 | 11 | 16 | 17 | {{end}} 18 |
12 | {{$k}} 13 | ({{len $v}} versions) 15 |
19 |
20 |
21 | {{end}} 22 | -------------------------------------------------------------------------------- /registry/internal/web/history_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func Test_history_ok(t *testing.T) { 13 | h, mocks := makeRouter(t) 14 | defer mocks.assertions() 15 | 16 | request, err := http.NewRequest(http.MethodGet, "/history", nil) 17 | require.NoError(t, err) 18 | 19 | recorder := httptest.NewRecorder() 20 | h.ServeHTTP(recorder, request) 21 | 22 | require.Equal(t, http.StatusOK, recorder.Code) 23 | 24 | bytes, err := ioutil.ReadAll(recorder.Result().Body) 25 | require.NoError(t, err) 26 | require.Equal(t, "this is some fake history", string(bytes)) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/configutil/load.go: -------------------------------------------------------------------------------- 1 | package configutil 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | func GetConfigFilename(args []string) (string, error) { 11 | if len(args) != 2 { 12 | return "", errors.Errorf("expected 1 argument, got %d", len(args)-1) 13 | } 14 | 15 | return args[1], nil 16 | } 17 | 18 | func LoadConfig(filename string, destination interface{}) error { 19 | bs, err := ioutil.ReadFile(filename) 20 | if err != nil { 21 | return errors.Wrap(err, "could not read config file") 22 | } 23 | 24 | if err := json.Unmarshal(bs, &destination); err != nil { 25 | return errors.Wrap(err, "could not parse config file") 26 | } 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /registry/internal/web/history.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 9 | ) 10 | 11 | type historyHandler struct { 12 | emitter stats.Sender 13 | log loggy.Logger 14 | history string 15 | } 16 | 17 | func newHistoryHandler(emitter stats.Sender, history string) http.Handler { 18 | return &historyHandler{ 19 | emitter: emitter, 20 | log: loggy.New("history-handler"), 21 | history: history, 22 | } 23 | } 24 | 25 | func (h *historyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 26 | w.Header().Set("Content-Type", "text/plain") 27 | w.Write([]byte(h.history)) 28 | 29 | h.emitter.Count("ui-history-ok", 1) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/repository/revinfo.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | ) 7 | 8 | type RevInfo struct { 9 | Version string `json:"version,omitempty"` // version string 10 | Name string `json:"name,omitempty"` // complete ID in underlying repository 11 | Short string `json:"short,omitempty"` // shortened ID, for use in pseudo-version 12 | Time time.Time `json:"time,omitempty"` // commit time 13 | } 14 | 15 | func (ri RevInfo) String() string { 16 | bs, err := json.MarshalIndent(ri, "", " ") 17 | if err != nil { 18 | panic(err) 19 | } 20 | return string(bs) 21 | } 22 | 23 | func (ri RevInfo) Bytes() []byte { 24 | bs, err := json.Marshal(ri) 25 | if err != nil { 26 | panic(err) 27 | } 28 | return bs 29 | } 30 | -------------------------------------------------------------------------------- /registry/internal/web/router_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 8 | "oss.indeed.com/go/modprox/registry/internal/data" 9 | ) 10 | 11 | type mocks struct { 12 | store *data.StoreMock 13 | } 14 | 15 | func newMocks(t *testing.T) mocks { 16 | return mocks{ 17 | store: data.NewStoreMock(t), 18 | } 19 | } 20 | 21 | func (m mocks) assertions() { 22 | m.store.MinimockFinish() 23 | } 24 | 25 | func makeRouter(t *testing.T) (http.Handler, mocks) { 26 | // emitter := &statstest.Sender{} no testing this? 27 | 28 | emitter := stats.Discard() 29 | 30 | mocks := newMocks(t) 31 | 32 | router := NewRouter(nil, nil, mocks.store, emitter, "this is some fake history", nil) 33 | return router, mocks 34 | } 35 | -------------------------------------------------------------------------------- /proxy/internal/web/history.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 9 | "oss.indeed.com/go/modprox/proxy/internal/web/output" 10 | ) 11 | 12 | type applicationHistory struct { 13 | log loggy.Logger 14 | history string 15 | emitter stats.Sender 16 | } 17 | 18 | func appHistory(emitter stats.Sender, history string) http.Handler { 19 | return &applicationHistory{ 20 | emitter: emitter, 21 | history: history, 22 | log: loggy.New("app-history"), 23 | } 24 | } 25 | 26 | // e.g. GET http://localhost:9000/history 27 | 28 | func (h *applicationHistory) ServeHTTP(w http.ResponseWriter, r *http.Request) { 29 | output.Write(w, output.Text, h.history) 30 | h.emitter.Count("app-history-ok", 1) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/webutil/utils.go: -------------------------------------------------------------------------------- 1 | package webutil 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "strconv" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // Write the given struct as JSON into w. 15 | func WriteJSON(w http.ResponseWriter, i interface{}) { 16 | w.Header().Set("Content-Type", "application/json") 17 | if err := json.NewEncoder(w).Encode(i); err != nil { 18 | log.Println("failed to write json response: " + err.Error()) 19 | } 20 | } 21 | 22 | // ParseURL parses tsURL, triggering a failure on t if it is not 23 | // possible to do so. 24 | func ParseURL(t *testing.T, tsURL string) (string, int) { 25 | tsURL = strings.TrimPrefix(tsURL, "http://") 26 | tokens := strings.Split(tsURL, ":") 27 | port, err := strconv.Atoi(tokens[1]) 28 | require.NoError(t, err) 29 | return tokens[0], port 30 | } 31 | -------------------------------------------------------------------------------- /cmd/modprox-proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/configutil" 9 | "oss.indeed.com/go/modprox/proxy" 10 | "oss.indeed.com/go/modprox/proxy/config" 11 | ) 12 | 13 | func main() { 14 | log := loggy.New("modprox-proxy") 15 | log.Infof("--- starting up ---") 16 | 17 | configFilename, err := configutil.GetConfigFilename(os.Args) 18 | if err != nil { 19 | log.Errorf("failed to startup: %v", err) 20 | os.Exit(1) 21 | } 22 | log.Infof("loading configuration from: %s", configFilename) 23 | 24 | var configuration config.Configuration 25 | if err := configutil.LoadConfig(configFilename, &configuration); err != nil { 26 | log.Errorf("failed to startup: %v", err) 27 | os.Exit(1) 28 | } 29 | log.Tracef("starting with configuration: %s", configuration) 30 | 31 | proxy.Start(configuration) 32 | } 33 | -------------------------------------------------------------------------------- /hack/sql/mysql-prox/modproxdb.sql: -------------------------------------------------------------------------------- 1 | create table proxy_module_zips ( 2 | id int(5) unsigned not null auto_increment, 3 | s_at_v varchar(1024) not null, -- unique module identifier source@version 4 | zip longblob not null, -- binary blob of the well formed zip archive 5 | primary key(id), 6 | unique (s_at_v) 7 | ) engine=InnoDB default charset=utf8; 8 | 9 | create table proxy_modules_index ( 10 | id int(5) unsigned not null auto_increment, 11 | source varchar(256) not null, -- module package, e.g. github.com/pkg/errors 12 | version varchar(256) not null, -- module version, e.g. v1.0.0-alpha1 13 | registry_mod_id int(5) unsigned not null, -- registry serial number of the module 14 | go_mod_file text not null, -- text of the go.mod file of the module 15 | version_info text not null, -- JSON of .info pseudo file 16 | primary key(id), 17 | unique (source, version) 18 | ) engine=InnoDB default charset=utf8; 19 | -------------------------------------------------------------------------------- /hack/configs/registry-local.mysql.json: -------------------------------------------------------------------------------- 1 | { 2 | "web_server": { 3 | "tls": { 4 | "enabled": false, 5 | "certificate": "", 6 | "key": "" 7 | }, 8 | "bind_address": "127.0.0.1", 9 | "port": 12500, 10 | "api_keys": ["abc123"] 11 | }, 12 | "csrf": { 13 | "development_mode": true, 14 | "authentication_key": "this is a 32 byte long string ok" 15 | }, 16 | "database_storage": { 17 | "mysql": { 18 | "user": "docker", 19 | "password": "docker", 20 | "address": "localhost:3306", 21 | "database": "modproxdb-reg", 22 | "allow_native_passwords": true 23 | } 24 | }, 25 | "statsd": { 26 | "agent": { 27 | "address": "127.0.0.1", 28 | "port": 8125 29 | } 30 | }, 31 | "proxies": { 32 | "prune_after_s": 60 33 | }, 34 | "proxy_client": { 35 | "protocol": "https", 36 | "base_url": "proxy.golang.org" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /registry/internal/web/about_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func Test_about_ok(t *testing.T) { 12 | h, mocks := makeRouter(t) 13 | defer mocks.assertions() 14 | 15 | request, err := http.NewRequest(http.MethodGet, "/configure/about", nil) 16 | require.NoError(t, err) 17 | 18 | recorder := httptest.NewRecorder() 19 | h.ServeHTTP(recorder, request) 20 | 21 | require.Equal(t, http.StatusOK, recorder.Code) 22 | } 23 | 24 | func Test_about_bad_method(t *testing.T) { 25 | h, mocks := makeRouter(t) 26 | defer mocks.assertions() 27 | 28 | request, err := http.NewRequest(http.MethodPost, "/configure/about", nil) 29 | require.NoError(t, err) 30 | 31 | recorder := httptest.NewRecorder() 32 | h.ServeHTTP(recorder, request) 33 | 34 | require.Equal(t, http.StatusMethodNotAllowed, recorder.Code) 35 | } 36 | -------------------------------------------------------------------------------- /proxy/internal/web/api_problems.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 9 | "oss.indeed.com/go/modprox/proxy/internal/problems" 10 | "oss.indeed.com/go/modprox/proxy/internal/web/output" 11 | ) 12 | 13 | type downloadProblems struct { 14 | dlTracker problems.Tracker 15 | emitter stats.Sender 16 | log loggy.Logger 17 | } 18 | 19 | func newDownloadProblems(dlTracker problems.Tracker, emitter stats.Sender) http.Handler { 20 | return &downloadProblems{ 21 | dlTracker: dlTracker, 22 | emitter: emitter, 23 | log: loggy.New("download-problems"), 24 | } 25 | } 26 | 27 | func (h *downloadProblems) ServeHTTP(w http.ResponseWriter, r *http.Request) { 28 | h.emitter.Count("api-download-problems", 1) 29 | 30 | dlProblems := h.dlTracker.Problems() 31 | h.log.Tracef("reporting %d download problems", len(dlProblems)) 32 | 33 | output.WriteJSON(w, dlProblems) 34 | } 35 | -------------------------------------------------------------------------------- /registry/internal/web/about.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 10 | "oss.indeed.com/go/modprox/registry/static" 11 | ) 12 | 13 | type aboutHandler struct { 14 | html *template.Template 15 | emitter stats.Sender 16 | log loggy.Logger 17 | } 18 | 19 | func newAboutHandler(emitter stats.Sender) http.Handler { 20 | html := static.MustParseTemplates( 21 | "static/html/layout.html", 22 | "static/html/navbar.html", 23 | "static/html/about.html", 24 | ) 25 | return &aboutHandler{ 26 | html: html, 27 | emitter: emitter, 28 | log: loggy.New("about-handler"), 29 | } 30 | } 31 | 32 | func (h *aboutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 33 | if err := h.html.Execute(w, nil); err != nil { 34 | h.log.Errorf("failed to execute about template: %v", err) 35 | return 36 | } 37 | 38 | h.emitter.Count("ui-about-ok", 1) 39 | } 40 | -------------------------------------------------------------------------------- /registry/internal/web/blocks.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 10 | "oss.indeed.com/go/modprox/registry/static" 11 | ) 12 | 13 | type blocksHandler struct { 14 | html *template.Template 15 | emitter stats.Sender 16 | log loggy.Logger 17 | } 18 | 19 | func newBlocksHandler(emitter stats.Sender) http.Handler { 20 | html := static.MustParseTemplates( 21 | "static/html/layout.html", 22 | "static/html/navbar.html", 23 | "static/html/blocks.html", 24 | ) 25 | 26 | return &blocksHandler{ 27 | html: html, 28 | emitter: emitter, 29 | log: loggy.New("blocks-handler"), 30 | } 31 | } 32 | 33 | func (h *blocksHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 34 | if err := h.html.Execute(w, nil); err != nil { 35 | h.log.Errorf("failed to execute blocks template: %v", err) 36 | return 37 | } 38 | 39 | h.emitter.Count("ui-blocks-ok", 1) 40 | } 41 | -------------------------------------------------------------------------------- /registry/internal/web/redirects.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/registry/internal/data" 10 | "oss.indeed.com/go/modprox/registry/static" 11 | ) 12 | 13 | type redirectsHandler struct { 14 | html *template.Template 15 | store data.Store 16 | log loggy.Logger 17 | } 18 | 19 | func newRedirectsHandler(store data.Store) http.Handler { 20 | html := static.MustParseTemplates( 21 | "static/html/layout.html", 22 | "static/html/navbar.html", 23 | "static/html/redirects.html", 24 | ) 25 | 26 | return &redirectsHandler{ 27 | html: html, 28 | store: store, 29 | log: loggy.New("redirects-handler"), 30 | } 31 | } 32 | 33 | func (h *redirectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 34 | h.log.Tracef("loaded page %v", r.Method) 35 | 36 | if err := h.html.Execute(w, nil); err != nil { 37 | h.log.Errorf("failed to execute redirects template: %v", err) 38 | return 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module oss.indeed.com/go/modprox 2 | 3 | require ( 4 | github.com/boltdb/bolt v1.3.1 5 | github.com/cactus/go-statsd-client v3.1.1+incompatible 6 | github.com/go-sql-driver/mysql v1.4.1 7 | github.com/gojuno/minimock/v3 v3.0.4 8 | github.com/gorilla/csrf v1.6.1 9 | github.com/gorilla/mux v1.7.3 10 | github.com/hashicorp/go-cleanhttp v0.5.1 11 | github.com/jinzhu/copier v0.0.0-20190625015134-976e0346caa8 12 | github.com/lib/pq v1.2.0 13 | github.com/pkg/errors v0.8.1 14 | github.com/stretchr/testify v1.4.0 15 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac // indirect 16 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 // indirect 17 | golang.org/x/tools v0.0.0-20190917032747-2dc213d980bc // indirect 18 | google.golang.org/appengine v1.6.2 // indirect 19 | gophers.dev/cmds/petrify/v5 v5.2.1 20 | gophers.dev/pkgs/atomicfs v0.3.2 21 | gophers.dev/pkgs/ignore v0.2.0 22 | gophers.dev/pkgs/loggy v0.2.0 23 | gophers.dev/pkgs/repeat v0.1.1 24 | gophers.dev/pkgs/semantic v0.1.0 25 | ) 26 | 27 | go 1.12 28 | -------------------------------------------------------------------------------- /pkg/clients/payloads/heartbeat.go: -------------------------------------------------------------------------------- 1 | package payloads 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "oss.indeed.com/go/modprox/pkg/netservice" 8 | ) 9 | 10 | type instance = netservice.Instance 11 | 12 | // Heartbeat of a proxy that is sent periodically to the registry. 13 | type Heartbeat struct { 14 | Self instance `json:"self"` 15 | NumModules int `json:"num_modules"` 16 | NumVersions int `json:"num_versions"` 17 | Timestamp int `json:"send_time"` // unix timestamp seconds 18 | } 19 | 20 | func (hb Heartbeat) String() string { 21 | return fmt.Sprintf("[%s:%d %d %d]", 22 | hb.Self.Address, 23 | hb.Self.Port, 24 | hb.NumModules, 25 | hb.NumVersions, 26 | ) 27 | } 28 | 29 | func (hb Heartbeat) TimeSince() string { 30 | t1 := time.Unix(int64(hb.Timestamp), 0) 31 | dur := time.Since(t1) 32 | d := dur.Round(time.Second) 33 | h := d / time.Hour 34 | d -= h * time.Hour 35 | m := d / time.Minute 36 | d -= m * time.Minute 37 | s := d / time.Second 38 | return fmt.Sprintf("%dh%dm%ds", h, m, s) 39 | } 40 | -------------------------------------------------------------------------------- /cmd/modprox-registry/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/configutil" 9 | "oss.indeed.com/go/modprox/registry" 10 | "oss.indeed.com/go/modprox/registry/config" 11 | ) 12 | 13 | // generate webpage statics 14 | //go:generate go run gophers.dev/cmds/petrify/v5/cmd/petrify -prefix ../../registry -o ../../registry/static/generated.go -pkg static ../../registry/static/... 15 | 16 | func main() { 17 | log := loggy.New("modprox-registry") 18 | log.Infof("--- starting up ---") 19 | 20 | configFilename, err := configutil.GetConfigFilename(os.Args) 21 | if err != nil { 22 | log.Errorf("failed to startup: %v", err) 23 | os.Exit(1) 24 | } 25 | log.Infof("loading configuration from: %s", configFilename) 26 | 27 | var configuration config.Configuration 28 | if err := configutil.LoadConfig(configFilename, &configuration); err != nil { 29 | log.Errorf("failed to startup: %v", err) 30 | os.Exit(1) 31 | } 32 | log.Tracef("starting with configuration: %s", configuration) 33 | 34 | registry.Start(configuration) 35 | } 36 | -------------------------------------------------------------------------------- /hack/sql/mysql-reg/modproxdb.sql: -------------------------------------------------------------------------------- 1 | create table modules ( 2 | id int(3) unsigned not null auto_increment, 3 | source varchar(896) not null, 4 | version varchar(128) not null, 5 | created timestamp not null default current_timestamp, 6 | primary key(id), 7 | unique (source, version) 8 | ) engine=InnoDB default charset=utf8; 9 | 10 | create table proxy_configurations ( 11 | id int(3) unsigned not null auto_increment, 12 | hostname varchar(128) not null, 13 | port int(6) not null, 14 | storage text not null, 15 | registry text not null, 16 | transforms text not null, 17 | ts timestamp not null default current_timestamp, 18 | primary key(id), 19 | unique (hostname, port) 20 | ) engine=InnoDB default charset=utf8; 21 | 22 | create table proxy_heartbeats ( 23 | id int(3) unsigned not null auto_increment, 24 | hostname varchar(128) not null, 25 | port int(6) not null, 26 | num_modules int(10) not null, 27 | num_versions int(10) not null, 28 | ts timestamp not null default current_timestamp, 29 | primary key(id), 30 | unique (hostname, port) 31 | ) engine=InnoDB default charset=utf8; 32 | -------------------------------------------------------------------------------- /registry/static/html/mods_show.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

versions of {{$.Source}}

6 |
7 |
8 | {{if not .Mods}} 9 |

there are none

10 | {{end}} 11 | 12 | {{range $v := .Mods}} 13 | 14 | 15 | 16 | 24 | 25 | {{end}} 26 |
{{.Source}}{{.Version}} 17 |
18 | {{$.CSRF}} 19 | 20 | 21 | 22 |
23 |
27 |
28 |
29 | {{end}} 30 | -------------------------------------------------------------------------------- /pkg/repository/blob.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "io/ioutil" 7 | "path/filepath" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // A Blob is an in-memory zip archive, representative of 13 | // a module that was extracted from a repository that was downloaded from upstream. 14 | // 15 | // There might not be a go.mod file, but there should not be more than one. 16 | type Blob []byte 17 | 18 | func (b Blob) ModFile() (string, bool, error) { 19 | r := bytes.NewReader(b) 20 | unzip, err := zip.NewReader(r, int64(len(b))) 21 | if err != nil { 22 | return "", false, errors.Wrap(err, "failed to open blob") 23 | } 24 | 25 | for _, f := range unzip.File { 26 | filename := filepath.Base(f.Name) 27 | if filename == "go.mod" { 28 | rc, err := f.Open() 29 | if err != nil { 30 | return "", false, err 31 | } 32 | 33 | bs, err := ioutil.ReadAll(rc) 34 | if err != nil { 35 | return "", false, errors.Wrap(err, "failed to read go.mod file from blob") 36 | } 37 | 38 | return string(bs), true, rc.Close() 39 | } 40 | } 41 | 42 | return "", false, nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/database/db.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "database/sql" 5 | "time" 6 | 7 | "github.com/go-sql-driver/mysql" 8 | 9 | "github.com/pkg/errors" 10 | 11 | "oss.indeed.com/go/modprox/pkg/setup" 12 | ) 13 | 14 | func Connect(kind string, dsn setup.DSN) (*sql.DB, error) { 15 | var err error 16 | var db *sql.DB 17 | 18 | switch kind { 19 | case "mysql": 20 | 21 | db, err = connectMySQL(mysql.Config{ 22 | Net: "tcp", 23 | User: dsn.User, 24 | Passwd: dsn.Password, 25 | Addr: dsn.Address, 26 | DBName: dsn.Database, 27 | AllowNativePasswords: dsn.AllowNativePasswords, 28 | ReadTimeout: 1 * time.Minute, // todo 29 | WriteTimeout: 1 * time.Minute, // todo 30 | }) 31 | if err != nil { 32 | return nil, errors.Wrap(err, "failed to connect to mysql") 33 | } 34 | default: 35 | return nil, errors.Errorf("%s is not a supported database", kind) 36 | } 37 | 38 | return db, nil 39 | } 40 | 41 | func connectMySQL(config mysql.Config) (*sql.DB, error) { 42 | dsn := config.FormatDSN() 43 | return sql.Open("mysql", dsn) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/clients/registry/modreq.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import "oss.indeed.com/go/modprox/pkg/coordinates" 4 | 5 | // ReqMods is the data sent from the proxy to the registry when 6 | // requesting from the registry a list of modules that the proxy 7 | // is in need of downloading to its local data-store. When making 8 | // the request, the proxy sends a list of ranges of serial IDs of 9 | // the modules it already has contained in its data-store. That way 10 | // the registry can compare that list of ranges with the set of all 11 | // modules that are registered, and reply with a list of modules that 12 | // only contains modules the proxy needs to download. 13 | type ReqMods struct { 14 | IDs coordinates.RangeIDs `json:"ids"` 15 | } 16 | 17 | // ReqModsResp is the response sent from the registry to the proxy 18 | // when the proxy requests a list of which modules it needs to download. 19 | // There is no guarantee the proxy will not have already downloaded 20 | // some of the modules, but given the implementation it should be pretty 21 | // well optimized to not include duplicates of what is in the proxy store. 22 | type ReqModsResp struct { 23 | Mods []coordinates.SerialModule `json:"serials"` 24 | } 25 | -------------------------------------------------------------------------------- /registry/internal/service/registry.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "os" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/clients/zips" 9 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 10 | "oss.indeed.com/go/modprox/registry/config" 11 | "oss.indeed.com/go/modprox/registry/internal/data" 12 | ) 13 | 14 | type Registry struct { 15 | config config.Configuration 16 | store data.Store 17 | emitter stats.Sender 18 | log loggy.Logger 19 | history string 20 | proxyClient zips.ProxyClient 21 | } 22 | 23 | func NewRegistry(config config.Configuration) *Registry { 24 | r := &Registry{ 25 | config: config, 26 | log: loggy.New("registry-service"), 27 | } 28 | 29 | for _, f := range []initer{ 30 | initSender, 31 | initStore, 32 | initProxyPrune, 33 | initHistory, 34 | initProxyClient, 35 | initWebServer, 36 | } { 37 | if err := f(r); err != nil { 38 | r.log.Errorf("cannot startup: failed to initialize registry") 39 | r.log.Errorf("caused by: %v", err) 40 | os.Exit(1) 41 | } 42 | } 43 | 44 | return r 45 | } 46 | 47 | func (r *Registry) Run() { 48 | select { 49 | //intentionally left blank 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /proxy/internal/modules/store/zipstore.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/coordinates" 9 | "oss.indeed.com/go/modprox/pkg/database" 10 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 11 | "oss.indeed.com/go/modprox/pkg/repository" 12 | "oss.indeed.com/go/modprox/pkg/setup" 13 | ) 14 | 15 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i ZipStore -s _mock.go 16 | 17 | type ZipStore interface { 18 | PutZip(coordinates.Module, repository.Blob) error 19 | GetZip(coordinates.Module) (repository.Blob, error) 20 | DelZip(coordinates.Module) error 21 | } 22 | 23 | func Connect(dsn setup.DSN, emitter stats.Sender) (*mysqlStore, error) { 24 | db, err := database.Connect("mysql", dsn) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | return New(db, emitter) 30 | } 31 | 32 | func New(db *sql.DB, emitter stats.Sender) (*mysqlStore, error) { 33 | statements, err := load(db) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &mysqlStore{ 38 | emitter: emitter, 39 | db: db, 40 | statements: statements, 41 | log: loggy.New("store"), 42 | }, nil 43 | } 44 | -------------------------------------------------------------------------------- /registry/internal/web/v1_registry.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/webutil" 13 | "oss.indeed.com/go/modprox/registry/internal/data" 14 | ) 15 | 16 | func registryAdd(store data.Store, emitter stats.Sender) http.HandlerFunc { 17 | log := loggy.New("registry-add-api") 18 | 19 | return func(w http.ResponseWriter, r *http.Request) { 20 | log.Tracef("adding to the registry") 21 | 22 | var wantToAdd []coordinates.Module 23 | 24 | if err := json.NewDecoder(r.Body).Decode(&wantToAdd); err != nil { 25 | http.Error(w, err.Error(), http.StatusBadRequest) 26 | emitter.Count("api-addmod-bad-request", 1) 27 | return 28 | } 29 | 30 | modulesAdded, err := store.InsertModules(wantToAdd) 31 | if err != nil { 32 | http.Error(w, err.Error(), http.StatusInternalServerError) 33 | emitter.Count("api-addmod-error", 1) 34 | return 35 | } 36 | 37 | msg := fmt.Sprintf("added %d new modules", modulesAdded) 38 | webutil.WriteJSON(w, msg) 39 | emitter.Count("api-addmod-ok", 1) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /registry/internal/web/v1_registry_list_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "oss.indeed.com/go/modprox/pkg/coordinates" 9 | ) 10 | 11 | type Range = coordinates.RangeID 12 | 13 | type Ranges = coordinates.RangeIDs 14 | 15 | func Test_inRange(t *testing.T) { 16 | try := func(i int64, rangeID Range, exp bool) { 17 | output := inRange(i, rangeID) 18 | require.Equal(t, exp, output) 19 | } 20 | 21 | try(1, Range{1, 1}, true) 22 | try(1, Range{1, 5}, true) 23 | try(1, Range{2, 5}, false) 24 | try(2, Range{1, 3}, true) 25 | try(2, Range{2, 3}, true) 26 | try(2, Range{1, 1}, false) 27 | try(2, Range{3, 5}, false) 28 | try(10, Range{3, 9}, false) 29 | try(10, Range{3, 13}, true) 30 | try(10, Range{11, 30}, false) 31 | } 32 | 33 | func Test_inListButNotRange(t *testing.T) { 34 | try := func(ids []int64, ranges Ranges, exp []int64) { 35 | output := inListButNotRange(ids, ranges) 36 | require.Equal(t, exp, output) 37 | } 38 | 39 | try( 40 | []int64{1, 2, 3}, 41 | Ranges{{1, 2}, {5, 6}}, 42 | []int64{3}, 43 | ) 44 | 45 | try( 46 | []int64{4, 5, 6, 10, 11, 12}, 47 | Ranges{{4, 4}, {11, 12}}, 48 | []int64{5, 6, 10}, 49 | ) 50 | } 51 | -------------------------------------------------------------------------------- /proxy/internal/web/mod_file.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 9 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 10 | "oss.indeed.com/go/modprox/proxy/internal/web/output" 11 | ) 12 | 13 | type moduleFile struct { 14 | index store.Index 15 | emitter stats.Sender 16 | log loggy.Logger 17 | } 18 | 19 | func modFile(index store.Index, emitter stats.Sender) http.Handler { 20 | return &moduleFile{ 21 | index: index, 22 | emitter: emitter, 23 | log: loggy.New("mod-file"), 24 | } 25 | } 26 | 27 | func (h *moduleFile) ServeHTTP(w http.ResponseWriter, r *http.Request) { 28 | mod, err := modInfoFromPath(r.URL.Path) 29 | if err != nil { 30 | http.Error(w, err.Error(), http.StatusBadRequest) 31 | h.emitter.Count("mod-file-bad-request", 1) 32 | return 33 | } 34 | h.log.Infof("serving request for go.mod file of %s", mod) 35 | 36 | modFile, err := h.index.Mod(mod) 37 | if err != nil { 38 | http.Error(w, err.Error(), http.StatusNotFound) 39 | h.emitter.Count("mod-file-not-found", 1) 40 | return 41 | } 42 | 43 | output.Write(w, output.Text, modFile) 44 | h.emitter.Count("mod-file-ok", 1) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/coordinates/modules_test.go: -------------------------------------------------------------------------------- 1 | package coordinates 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func sMod(id int64, source, version string) SerialModule { 11 | return SerialModule{ 12 | SerialID: id, 13 | Module: Module{ 14 | Source: source, 15 | Version: version, 16 | }, 17 | } 18 | } 19 | 20 | func Test_ModsByVersion(t *testing.T) { 21 | mods := []SerialModule{ 22 | sMod(3, "foo", "v1.2.13"), 23 | sMod(3, "bar", "v0.0.3"), 24 | sMod(3, "bar", "v1.0.10"), 25 | sMod(3, "bar", "v1.2.1"), 26 | sMod(3, "baz", "v1.2.3"), 27 | sMod(3, "bar", "v11.3.3"), 28 | sMod(3, "foo", "v2.0.14"), 29 | sMod(3, "bar", "v11.2.3"), 30 | sMod(3, "baz", "v3.2.3"), 31 | sMod(3, "baz", "v3.12.1"), 32 | sMod(3, "bar", "v1.20.3"), 33 | } 34 | 35 | sort.Sort(ModsByVersion(mods)) 36 | 37 | require.Equal(t, []SerialModule{ 38 | sMod(3, "bar", "v0.0.3"), 39 | sMod(3, "bar", "v1.0.10"), 40 | sMod(3, "bar", "v1.2.1"), 41 | sMod(3, "bar", "v1.20.3"), 42 | sMod(3, "bar", "v11.2.3"), 43 | sMod(3, "bar", "v11.3.3"), 44 | sMod(3, "baz", "v1.2.3"), 45 | sMod(3, "baz", "v3.2.3"), 46 | sMod(3, "baz", "v3.12.1"), 47 | sMod(3, "foo", "v1.2.13"), 48 | sMod(3, "foo", "v2.0.14"), 49 | }, mods) 50 | } 51 | -------------------------------------------------------------------------------- /registry/static/html/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 13 | 15 | 16 | 17 | Go Module Proxy 18 | 19 | 20 |
21 | {{template "navbar" .}} 22 |
23 |
24 | {{template "body" .}} 25 |
26 | 27 | 28 | -------------------------------------------------------------------------------- /hack/configs/proxy-local.json: -------------------------------------------------------------------------------- 1 | { 2 | "api_server": { 3 | "tls": { 4 | "enabled": false, 5 | "certificate": "", 6 | "key": "" 7 | }, 8 | "bind_address": "127.0.0.1", 9 | "port": 12505 10 | }, 11 | "statsd": { 12 | "agent": { 13 | "address": "127.0.0.1", 14 | "port": 8125 15 | } 16 | }, 17 | "registry": { 18 | "instances": [ 19 | { 20 | "address": "localhost", 21 | "port": 12500 22 | } 23 | ], 24 | "poll_frequency_s": 20, 25 | "request_timeout_s": 10, 26 | "api_key": "abc123" 27 | }, 28 | "module_db_storage": { 29 | "mysql": { 30 | "user": "docker", 31 | "password": "docker", 32 | "address": "localhost:3307", 33 | "database": "modproxdb-prox", 34 | "allow_native_passwords": true 35 | } 36 | }, 37 | "zip_proxy": { 38 | "protocol": "https", 39 | "base_url": "proxy.golang.org" 40 | }, 41 | "transforms": { 42 | "auto_redirect": true, 43 | "domain_paths": [{ 44 | "domain": "code.internal.company.net", 45 | "path": "ELEM1/ELEM2/-/archive/VERSION/ELEM2-VERSION.zip" 46 | }], 47 | "domain_headers": [{ 48 | "domain": "code.internal.company.net", 49 | "headers": {"Private-Token": "mysecrettoken"} 50 | }] 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/repository/blob_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func createFakeZip(t *testing.T, hasModFile bool) []byte { 12 | buf := new(bytes.Buffer) 13 | w := zip.NewWriter(buf) 14 | var files = []struct { 15 | Name, Body string 16 | }{ 17 | {"readme.txt", "This archive contains some text files."}, 18 | {"go.mod", "module github.com/modprox/libmodprox"}, 19 | {"todo.txt", "Get animal handling licence.\nWrite more examples."}, 20 | } 21 | for _, file := range files { 22 | if (file.Name != "go.mod") || hasModFile { 23 | f, err := w.Create(file.Name) 24 | require.NoError(t, err) 25 | _, err = f.Write([]byte(file.Body)) 26 | require.NoError(t, err) 27 | } 28 | } 29 | 30 | err := w.Close() 31 | require.NoError(t, err) 32 | return buf.Bytes() 33 | } 34 | 35 | func Test_Blob_ModFile(t *testing.T) { 36 | b := createFakeZip(t, true) 37 | blob := Blob(b) 38 | 39 | content, exists, err := blob.ModFile() 40 | require.NoError(t, err) 41 | require.True(t, exists) 42 | require.Equal(t, "module github.com/modprox/libmodprox", content) 43 | } 44 | 45 | func Test_Blob_ModFile_none(t *testing.T) { 46 | b := createFakeZip(t, false) 47 | blob := Blob(b) 48 | 49 | _, exists, err := blob.ModFile() 50 | require.NoError(t, err) 51 | require.False(t, exists) 52 | } 53 | -------------------------------------------------------------------------------- /proxy/internal/web/mod_info.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 9 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 10 | "oss.indeed.com/go/modprox/proxy/internal/web/output" 11 | ) 12 | 13 | type moduleInfo struct { 14 | log loggy.Logger 15 | emitter stats.Sender 16 | index store.Index 17 | } 18 | 19 | func modInfo(index store.Index, emitter stats.Sender) http.Handler { 20 | return &moduleInfo{ 21 | index: index, 22 | emitter: emitter, 23 | log: loggy.New("mod-info"), 24 | } 25 | } 26 | 27 | // e.g. GET http://localhost:9000/github.com/example/toolkit/@v/v1.0.0.info 28 | 29 | func (h *moduleInfo) ServeHTTP(w http.ResponseWriter, r *http.Request) { 30 | mod, err := modInfoFromPath(r.URL.Path) 31 | if err != nil { 32 | h.log.Warnf("bad request for info: %v", err) 33 | http.Error(w, err.Error(), http.StatusBadRequest) 34 | h.emitter.Count("mod-info-bad-request", 1) 35 | return 36 | } 37 | 38 | h.log.Infof("serving request for .info of: %s", mod) 39 | 40 | revInfo, err := h.index.Info(mod) 41 | if err != nil { 42 | http.Error(w, err.Error(), http.StatusNotFound) 43 | h.emitter.Count("mod-info-not-found", 1) 44 | return 45 | } 46 | 47 | content := revInfo.String() 48 | output.Write(w, output.JSON, content) 49 | h.emitter.Count("mod-info-ok", 1) 50 | } 51 | -------------------------------------------------------------------------------- /pkg/repository/parse_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "oss.indeed.com/go/modprox/pkg/coordinates" 9 | ) 10 | 11 | func Test_Parse(t *testing.T) { 12 | try := func(input string, expMod coordinates.Module, expErr bool) { 13 | output, err := Parse(input) 14 | require.True(t, expErr == (err != nil), "err was: %v", err) 15 | require.Equal(t, expMod, output) 16 | } 17 | 18 | try("github.com/foo/bar v2.0.0", coordinates.Module{ 19 | Source: "github.com/foo/bar", 20 | Version: "v2.0.0", 21 | }, false) 22 | 23 | try("github.com/tdewolff/parse v2.3.3+incompatible // indirect", coordinates.Module{ 24 | Source: "github.com/tdewolff/parse", 25 | Version: "v2.3.3+incompatible", 26 | }, false) 27 | 28 | try("golang.org/x/tools v0.0.0-20180111040409-fbec762f837d", coordinates.Module{ 29 | Source: "golang.org/x/tools", 30 | Version: "v0.0.0-20180111040409-fbec762f837d", 31 | }, false) 32 | 33 | try("/github.com/cpuguy83/go-md2man/@v/v1.0.6.info", coordinates.Module{ 34 | Source: "github.com/cpuguy83/go-md2man", 35 | Version: "v1.0.6", 36 | }, false) 37 | 38 | try("/github.com/cpuguy83/go-md2man/@v/v1.0.6.rm", coordinates.Module{ 39 | Source: "github.com/cpuguy83/go-md2man", 40 | Version: "v1.0.6", 41 | }, false) 42 | } 43 | 44 | // http://localhost:9000/gopkg.in/check.v1/@v/v0.0.0-20161208181325-20d25e280405.info 45 | -------------------------------------------------------------------------------- /registry/internal/proxies/prune.go: -------------------------------------------------------------------------------- 1 | package proxies 2 | 3 | import ( 4 | "time" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/registry/internal/data" 9 | ) 10 | 11 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Pruner -s _mock.go 12 | 13 | type Pruner interface { 14 | Prune(time.Time) error 15 | } 16 | 17 | type pruner struct { 18 | maxAge time.Duration 19 | store data.Store 20 | log loggy.Logger 21 | } 22 | 23 | func NewPruner(maxAge time.Duration, store data.Store) Pruner { 24 | return &pruner{ 25 | maxAge: maxAge, 26 | store: store, 27 | log: loggy.New("proxy-prune"), 28 | } 29 | } 30 | 31 | func (p *pruner) Prune(now time.Time) error { 32 | heartbeats, err := p.store.ListHeartbeats() 33 | if err != nil { 34 | return err 35 | } 36 | 37 | p.log.Tracef("looking through proxy heartbeats for removable instances") 38 | for _, heartbeat := range heartbeats { 39 | then := time.Unix(int64(heartbeat.Timestamp), 0) 40 | elapsed := now.Sub(then) 41 | if elapsed > p.maxAge { 42 | p.log.Warnf("purging M.I.A. proxy %s", heartbeat.Self) 43 | if err := p.store.PurgeProxy(heartbeat.Self); err != nil { 44 | p.log.Errorf("failed to purge proxy: %s: %v", heartbeat.Self, err) 45 | return err 46 | } 47 | } else { 48 | p.log.Tracef("not purging proxy of age %v (max %v)", elapsed, p.maxAge) 49 | } 50 | } 51 | 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /proxy/internal/web/mod_zip.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 9 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 10 | "oss.indeed.com/go/modprox/proxy/internal/web/output" 11 | ) 12 | 13 | type moduleZip struct { 14 | store store.ZipStore 15 | emitter stats.Sender 16 | log loggy.Logger 17 | } 18 | 19 | func modZip(store store.ZipStore, emitter stats.Sender) http.Handler { 20 | return &moduleZip{ 21 | store: store, 22 | emitter: emitter, 23 | log: loggy.New("mod-zip"), 24 | } 25 | } 26 | 27 | // e.g. GET http://localhost:9000/github.com/example/toolkit/@v/v1.0.0.zip 28 | 29 | func (h *moduleZip) ServeHTTP(w http.ResponseWriter, r *http.Request) { 30 | mod, err := modInfoFromPath(r.URL.Path) 31 | if err != nil { 32 | http.Error(w, err.Error(), http.StatusBadRequest) 33 | h.emitter.Count("mod-zip-bad-request", 1) 34 | return 35 | } 36 | 37 | h.log.Infof("serving request for .zip file of %s", mod) 38 | 39 | zipBlob, err := h.store.GetZip(mod) 40 | if err != nil { 41 | h.log.Warnf("failed to get zip file of %s, %v", mod, err) 42 | http.Error(w, err.Error(), http.StatusNotFound) 43 | h.emitter.Count("mod-zip-not-found", 1) 44 | return 45 | } 46 | 47 | h.log.Infof("sending zip which is %d bytes", len(zipBlob)) 48 | output.WriteZip(w, zipBlob) 49 | h.emitter.Count("mod-zip-ok", 1) 50 | } 51 | -------------------------------------------------------------------------------- /proxy/internal/web/mod_list.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 10 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 11 | "oss.indeed.com/go/modprox/proxy/internal/web/output" 12 | ) 13 | 14 | type moduleList struct { 15 | index store.Index 16 | emitter stats.Sender 17 | log loggy.Logger 18 | } 19 | 20 | func modList(index store.Index, emitter stats.Sender) http.Handler { 21 | return &moduleList{ 22 | index: index, 23 | emitter: emitter, 24 | log: loggy.New("mod-list"), 25 | } 26 | } 27 | 28 | func (h *moduleList) ServeHTTP(w http.ResponseWriter, r *http.Request) { 29 | 30 | module, err := moduleFromPath(r.URL.Path) 31 | if err != nil { 32 | http.Error(w, err.Error(), http.StatusBadRequest) 33 | return 34 | } 35 | 36 | h.log.Infof("serving request for list module: %s", module) 37 | 38 | listing, err := h.index.Versions(module) 39 | if err != nil { 40 | http.Error(w, err.Error(), http.StatusNotFound) 41 | h.emitter.Count("mod-list-not-found", 1) 42 | return 43 | } 44 | 45 | output.Write(w, output.Text, formatList(listing)) 46 | h.emitter.Count("mod-list-ok", 1) 47 | } 48 | 49 | func formatList(list []string) string { 50 | var sb strings.Builder 51 | for _, version := range list { 52 | sb.WriteString(version) 53 | sb.WriteString("\n") 54 | } 55 | return sb.String() 56 | } 57 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 The modprox authors, All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 7 | Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 8 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 9 | 10 | -------------------------------------------------------------------------------- /pkg/repository/parse.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/pkg/errors" 7 | 8 | "oss.indeed.com/go/modprox/pkg/coordinates" 9 | ) 10 | 11 | var ( 12 | // examples 13 | // mod file style 14 | // github.com/foo/bar v2.0.0 15 | // github.com/tdewolff/parse v2.3.3+incompatible // indirect 16 | // golang.org/x/tools v0.0.0-20180111040409-fbec762f837d 17 | // gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 18 | // sum file style 19 | // github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= 20 | // proxy request style 21 | // /github.com/cpuguy83/go-md2man/@v/v1.0.6.info 22 | // zip style 23 | // github.com/kr/pty@v1.1.1 24 | ) 25 | 26 | // Parse will parse s as a module in string form. 27 | func Parse(s string) (coordinates.Module, error) { 28 | orig := s 29 | s = strings.Trim(s, "/") 30 | s = strings.TrimSuffix(s, ".info") 31 | s = strings.TrimSuffix(s, ".zip") 32 | s = strings.TrimSuffix(s, ".mod") 33 | s = strings.TrimSuffix(s, ".rm") 34 | s = strings.Replace(s, "/@v/", " ", -1) // in web handlers 35 | s = strings.Replace(s, "@v", " v", -1) // pasted from logs 36 | 37 | var mod coordinates.Module 38 | split := strings.Fields(s) 39 | if len(split) < 2 { 40 | return mod, errors.Errorf("malformed module line: %q", orig) 41 | } 42 | 43 | source := strings.TrimSuffix(split[0], "/") 44 | version := strings.TrimSuffix(split[1], "/go.mod") 45 | 46 | mod.Source = source 47 | mod.Version = version 48 | return mod, nil 49 | } 50 | -------------------------------------------------------------------------------- /proxy/internal/web/output/write.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strconv" 7 | 8 | "oss.indeed.com/go/modprox/pkg/repository" 9 | ) 10 | 11 | const ( 12 | headerContentType = "Content-Type" 13 | headerContentDescription = "Content-Description" 14 | headerContentTransferEncoding = "Content-Transfer-Encoding" 15 | headerContentLength = "Content-Length" 16 | ) 17 | 18 | const ( 19 | JSON = "application/json" 20 | Text = "text/plain" 21 | Zip = "application/zip" 22 | OctetStream = "application/octet-stream" 23 | FileTransfer = "File Transfer" 24 | Binary = "binary" 25 | ) 26 | 27 | func Write(w http.ResponseWriter, mime, content string) { 28 | w.Header().Set(headerContentType, mime) 29 | w.WriteHeader(http.StatusOK) 30 | _, _ = w.Write([]byte(content)) 31 | } 32 | 33 | func WriteJSON(w http.ResponseWriter, i interface{}) { 34 | w.Header().Set(headerContentType, JSON) 35 | if err := json.NewEncoder(w).Encode(i); err != nil { 36 | panic("failure to encode json response: " + err.Error()) 37 | } 38 | } 39 | 40 | func WriteZip(w http.ResponseWriter, blob repository.Blob) { 41 | w.Header().Set(headerContentType, Zip) 42 | w.Header().Add(headerContentType, OctetStream) 43 | w.Header().Set(headerContentDescription, FileTransfer) 44 | w.Header().Set(headerContentTransferEncoding, Binary) 45 | w.Header().Set(headerContentLength, strconv.Itoa(len(blob))) 46 | 47 | w.WriteHeader(http.StatusOK) 48 | _, _ = w.Write(blob) 49 | } 50 | -------------------------------------------------------------------------------- /hack/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | mysql-registry: 5 | image: mysql/mysql-server:5.7 6 | container_name: modprox-mysql-registry 7 | ports: 8 | - "3306:3306" 9 | tmpfs: 10 | - /var/lib/mysql 11 | - /tmp 12 | volumes: 13 | - ./sql/mysql-reg:/docker-entrypoint-initdb.d 14 | environment: 15 | - MYSQL_ROOT_PASSWORD=passw0rd 16 | - MYSQL_DATABASE=modproxdb-reg 17 | - MYSQL_USER=docker 18 | - MYSQL_PASSWORD=docker 19 | entrypoint: 20 | - /entrypoint.sh 21 | - mysqld 22 | 23 | mysql-proxy: 24 | image: mysql/mysql-server:5.7 25 | container_name: modprox-mysql-proxy 26 | ports: 27 | - "3307:3306" 28 | tmpfs: 29 | - /var/lib/mysql 30 | - /tmp 31 | volumes: 32 | - ./sql/mysql-prox:/docker-entrypoint-initdb.d 33 | # allow up to 128 MiB blobs over the connection 34 | command: --max_allowed_packet=134217728 35 | environment: 36 | - MYSQL_ROOT_PASSWORD=passw0rd 37 | - MYSQL_DATABASE=modproxdb-prox 38 | - MYSQL_USER=docker 39 | - MYSQL_PASSWORD=docker 40 | entrypoint: 41 | - /entrypoint.sh 42 | - mysqld 43 | 44 | # A fake datadog/statsd collector which prints metrics 45 | # to standard out in your docker container 46 | fakeadog: 47 | image: johnstcn/fakeadog:latest 48 | container_name: modprox-fakeadog 49 | ports: 50 | - target: 8125 51 | published: 8125 52 | protocol: udp 53 | mode: host 54 | tmpfs: 55 | - /tmp 56 | environment: 57 | - HOST=0.0.0.0 58 | - PORT=8125 59 | -------------------------------------------------------------------------------- /pkg/clients/payloads/configuration.go: -------------------------------------------------------------------------------- 1 | package payloads 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/jinzhu/copier" 7 | 8 | "oss.indeed.com/go/modprox/pkg/netservice" 9 | "oss.indeed.com/go/modprox/pkg/setup" 10 | "oss.indeed.com/go/modprox/proxy/config" 11 | ) 12 | 13 | // Configuration of a proxy instance when it starts up that is sent 14 | // to the registry. 15 | type Configuration struct { 16 | Self netservice.Instance `json:"self"` 17 | DiskStorage config.Storage `json:"disk_storage,omitempty"` 18 | DatabaseStorage setup.PersistentStore `json:"database_storage,omitempty"` 19 | Registry config.Registry `json:"registry"` 20 | Transforms config.Transforms `json:"transforms"` 21 | } 22 | 23 | func (c Configuration) Texts() (string, string, string, error) { 24 | 25 | storageText, err := json.Marshal(c.DiskStorage) 26 | if err != nil { 27 | return "", "", "", err 28 | } 29 | 30 | registriesText, err := json.Marshal(c.Registry) 31 | if err != nil { 32 | return "", "", "", err 33 | } 34 | 35 | // hide the values of the headers, which may contain secrets 36 | var t2 config.Transforms 37 | 38 | if err := copier.Copy(&t2, &c.Transforms); err != nil { 39 | return "", "", "", err 40 | } 41 | 42 | for _, transform := range t2.DomainHeaders { 43 | for key := range transform.Headers { 44 | transform.Headers[key] = "********" 45 | } 46 | } 47 | 48 | transformsText, err := json.Marshal(t2) 49 | if err != nil { 50 | return "", "", "", err 51 | } 52 | 53 | return string(storageText), string(registriesText), string(transformsText), nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/configutil/load_test.go: -------------------------------------------------------------------------------- 1 | package configutil 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func Test_GetConfigFilename_ok(t *testing.T) { 12 | args := []string{"executable", "foo.json"} 13 | filename, err := GetConfigFilename(args) 14 | require.NoError(t, err) 15 | require.Equal(t, "foo.json", filename) 16 | } 17 | 18 | func Test_GetConfigFilename_too_few(t *testing.T) { 19 | args := []string{"executable"} 20 | _, err := GetConfigFilename(args) 21 | require.Error(t, err) 22 | } 23 | 24 | func Test_GetConfigFilename_too_many(t *testing.T) { 25 | args := []string{"executable", "bar.json", "baz.json"} 26 | _, err := GetConfigFilename(args) 27 | require.Error(t, err) 28 | } 29 | 30 | type c struct { 31 | Foo string `json:"foo"` 32 | } 33 | 34 | func setup(t *testing.T, content string) { 35 | err := ioutil.WriteFile("foo.txt", []byte(content), 0600) 36 | require.NoError(t, err) 37 | } 38 | 39 | func cleanup(t *testing.T) { 40 | err := os.Remove("foo.txt") 41 | require.NoError(t, err) 42 | } 43 | 44 | func Test_LoadConfig_ok(t *testing.T) { 45 | setup(t, `{"foo":"bar"}`) 46 | defer cleanup(t) 47 | 48 | var config c 49 | err := LoadConfig("foo.txt", &config) 50 | require.NoError(t, err) 51 | require.Equal(t, "bar", config.Foo) 52 | } 53 | 54 | func Test_LoadConfig_unparsable(t *testing.T) { 55 | setup(t, "{{{{") 56 | defer cleanup(t) 57 | 58 | var config c 59 | err := LoadConfig("", &config) 60 | require.Error(t, err) 61 | } 62 | 63 | func Test_LoadConfig_no_file(t *testing.T) { 64 | var config c 65 | err := LoadConfig("/does/not/exist", &config) 66 | require.Error(t, err) 67 | } 68 | -------------------------------------------------------------------------------- /pkg/coordinates/modules.go: -------------------------------------------------------------------------------- 1 | package coordinates 2 | 3 | import ( 4 | "fmt" 5 | 6 | "gophers.dev/pkgs/semantic" 7 | ) 8 | 9 | type Module struct { 10 | Source string `json:"source"` 11 | Version string `json:"version"` 12 | } 13 | 14 | // String representation intended for human consumption. 15 | // 16 | // Includes surrounding parenthesis and some whitespace to pop in logs. 17 | func (m Module) String() string { 18 | return fmt.Sprintf("(%s @ %s)", m.Source, m.Version) 19 | } 20 | 21 | // AtVersion representation intended for machine consumption. 22 | // 23 | // Format is source@version. 24 | func (m Module) AtVersion() string { 25 | return fmt.Sprintf( 26 | "%s@%s", 27 | m.Source, 28 | m.Version, 29 | ) 30 | } 31 | 32 | // Bytes is AtVersion but converted to Bytes for use in a data-store which 33 | // stores bytes. 34 | func (m Module) Bytes() []byte { 35 | return []byte(m.AtVersion()) 36 | } 37 | 38 | type SerialModule struct { 39 | Module 40 | SerialID int64 `json:"id"` 41 | } 42 | 43 | type ModsByVersion []SerialModule 44 | 45 | func (mods ModsByVersion) Len() int { return len(mods) } 46 | func (mods ModsByVersion) Swap(x, y int) { mods[x], mods[y] = mods[y], mods[x] } 47 | func (mods ModsByVersion) Less(x, y int) bool { 48 | modX := mods[x] 49 | modY := mods[y] 50 | 51 | if modX.Source < modY.Source { 52 | return true 53 | } else if modX.Source > modY.Source { 54 | return false 55 | } 56 | 57 | tagX, okX := semantic.Parse(modX.Version) 58 | tagY, okY := semantic.Parse(modY.Version) 59 | 60 | if !okX && !okY { 61 | return false 62 | } else if okX && !okY { 63 | return false 64 | } else if !okX && okY { 65 | return true 66 | } 67 | 68 | return tagX.Less(tagY) 69 | } 70 | -------------------------------------------------------------------------------- /proxy/internal/modules/get/request_test.go: -------------------------------------------------------------------------------- 1 | package get 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "oss.indeed.com/go/modprox/pkg/clients/registry" 12 | "oss.indeed.com/go/modprox/pkg/coordinates" 13 | "oss.indeed.com/go/modprox/pkg/netservice" 14 | "oss.indeed.com/go/modprox/pkg/webutil" 15 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 16 | ) 17 | 18 | const modsReply = ` {"serials": [{ 19 | "id": 2, 20 | "source": "github.com/pkg/errors", 21 | "version": "v0.8.0" 22 | }]}` 23 | 24 | func Test_ModulesNeeded(t *testing.T) { 25 | index := store.NewIndexMock(t) 26 | defer index.MinimockFinish() 27 | 28 | ids := coordinates.RangeIDs{ 29 | coordinates.RangeID{1, 3}, 30 | coordinates.RangeID{6, 6}, 31 | coordinates.RangeID{10, 20}, 32 | } 33 | 34 | index.IDsMock.Return(ids, nil) 35 | 36 | ts := httptest.NewServer( 37 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 38 | _, _ = w.Write([]byte(modsReply)) 39 | }), 40 | ) 41 | defer ts.Close() 42 | 43 | address, port := webutil.ParseURL(t, ts.URL) 44 | client := registry.NewClient(registry.Options{ 45 | Timeout: 10 * time.Second, 46 | Instances: []netservice.Instance{{ 47 | Address: address, 48 | Port: port, 49 | }}, 50 | }) 51 | 52 | apiClient := NewRegistryAPI(client, index) 53 | 54 | serialModules, err := apiClient.ModulesNeeded(ids) 55 | require.NoError(t, err) 56 | 57 | require.Equal(t, []coordinates.SerialModule{ 58 | { 59 | SerialID: 2, 60 | Module: coordinates.Module{ 61 | Source: "github.com/pkg/errors", 62 | Version: "v0.8.0", 63 | }, 64 | }, 65 | }, serialModules) 66 | } 67 | -------------------------------------------------------------------------------- /proxy/internal/web/common.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "strings" 5 | 6 | "oss.indeed.com/go/modprox/pkg/coordinates" 7 | "oss.indeed.com/go/modprox/pkg/repository" 8 | ) 9 | 10 | // GET baseURL/module/@v/list fetches a list of all known versions, one per line. 11 | 12 | func moduleFromPath(p string) (string, error) { 13 | p = demangle(p) 14 | mod, err := repository.Parse(p) 15 | return mod.Source, err 16 | } 17 | 18 | func modInfoFromPath(p string) (coordinates.Module, error) { 19 | p = demangle(p) 20 | return repository.Parse(p) 21 | } 22 | 23 | // from the Go documentation: https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol 24 | // 25 | // To avoid problems when serving from case-sensitive file systems, the and 26 | // elements are case-encoded, replacing every uppercase letter with an exclamation mark followed 27 | // by the corresponding lower-case letter: github.com/Azure encodes as github.com/!azure. 28 | // 29 | // modprox currently store modules under their correct names, so we must rewrite the go 30 | // commands download requests from the mangled name to the correct name. 31 | func demangle(s string) string { 32 | var correct strings.Builder 33 | 34 | // copy s into correct, using lookahead to rewrite letters 35 | var i int 36 | for i = 0; i < len(s)-1; i++ { 37 | c := s[i] 38 | if c == '!' { 39 | next := s[i+1] 40 | if next >= 'a' && next <= 'z' { 41 | correct.WriteByte(next - ('a' - 'A')) 42 | i++ 43 | continue 44 | } 45 | } 46 | correct.WriteByte(c) 47 | } 48 | 49 | // if the last 2 letters were not an encoding, copy the last letter 50 | if i == len(s)-1 { 51 | correct.WriteByte(s[i]) 52 | } 53 | 54 | return correct.String() 55 | } 56 | -------------------------------------------------------------------------------- /proxy/internal/status/heartbeat/loop.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "time" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | "gophers.dev/pkgs/repeat/x" 8 | 9 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 10 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 11 | ) 12 | 13 | type PokeLooper interface { 14 | Loop() 15 | } 16 | 17 | func NewLooper( 18 | interval time.Duration, 19 | index store.Index, 20 | emitter stats.Sender, 21 | sender Sender, 22 | ) PokeLooper { 23 | return &looper{ 24 | interval: interval, 25 | index: index, 26 | emitter: emitter, 27 | sender: sender, 28 | log: loggy.New("heartbeat-looper"), 29 | } 30 | } 31 | 32 | type looper struct { 33 | interval time.Duration 34 | index store.Index 35 | sender Sender 36 | emitter stats.Sender 37 | log loggy.Logger 38 | } 39 | 40 | // Loop will block and run forever, sending heartbeats 41 | // at the configured interval, to whichever of the specified 42 | // registry instances accepts the heartbeat first. 43 | func (l *looper) Loop() { 44 | _ = x.Interval(l.interval, l.loop) 45 | } 46 | 47 | func (l *looper) loop() error { 48 | modules, versions, err := l.index.Summary() 49 | if err != nil { 50 | return err 51 | } 52 | 53 | l.emitter.Gauge("index-num-modules", modules) // really packages 54 | l.emitter.Gauge("index-num-versions", versions) // really modules 55 | 56 | if err := l.sender.Send( 57 | modules, 58 | versions, 59 | ); err != nil { 60 | l.emitter.Count("heartbeat-send-failure", 1) 61 | 62 | l.log.Warnf("could not send heartbeat, will try again later: %v", err) 63 | return nil // always nil, never stop 64 | } 65 | 66 | l.emitter.Count("heartbeat-send-ok", 1) 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/setup/dsn.go: -------------------------------------------------------------------------------- 1 | package setup 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | type PersistentStore struct { 9 | MySQL DSN `json:"mysql,omitempty"` 10 | } 11 | 12 | // DSN returns the one DSN that is configured, or returns 13 | // an error if both or no DSN is configured. 14 | func (ps PersistentStore) DSN() (string, DSN, error) { 15 | emptyDSN := DSN{} 16 | 17 | // check if DSN is empty 18 | if ps.MySQL.equal(emptyDSN) { 19 | return "", emptyDSN, errors.New("mysql was not configured") 20 | } 21 | 22 | return "mysql", ps.MySQL, nil 23 | } 24 | 25 | // DSN represents the "data source name" for a database. 26 | type DSN struct { 27 | User string `json:"user,omitempty"` 28 | Password string `json:"password,omitempty"` 29 | Address string `json:"address,omitempty"` 30 | Database string `json:"database,omitempty"` 31 | Parameters map[string]string `json:"parameters,omitempty"` 32 | ServerPublicKey string `json:"server_public_key,omitempty"` 33 | AllowNativePasswords bool `json:"allow_native_passwords,omitempty"` 34 | } 35 | 36 | func (dsn DSN) equal(other DSN) bool { 37 | switch { 38 | case dsn.User != other.User: 39 | return false 40 | case dsn.Password != other.Password: 41 | return false 42 | case dsn.Database != other.Database: 43 | return false 44 | case dsn.Address != other.Address: 45 | return false 46 | } 47 | return true 48 | } 49 | 50 | func (dsn DSN) String() string { 51 | return fmt.Sprintf( 52 | "dsn:[user: %s, address: %s, database: %s, allownative: %t", 53 | dsn.User, 54 | dsn.Address, 55 | dsn.Database, 56 | dsn.AllowNativePasswords, 57 | ) 58 | } 59 | -------------------------------------------------------------------------------- /proxy/internal/modules/get/request.go: -------------------------------------------------------------------------------- 1 | package get 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/clients/registry" 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 12 | ) 13 | 14 | // Range is an alias of coordinates.RangeIDs for brevity. 15 | type Ranges = coordinates.RangeIDs 16 | 17 | // RegistryAPI is used to issue API request from the registry 18 | type RegistryAPI interface { 19 | ModulesNeeded(Ranges) ([]coordinates.SerialModule, error) 20 | } 21 | 22 | type registryAPI struct { 23 | registryClient registry.Client 24 | index store.Index 25 | log loggy.Logger 26 | } 27 | 28 | func NewRegistryAPI( 29 | registryClient registry.Client, 30 | index store.Index, 31 | ) RegistryAPI { 32 | return ®istryAPI{ 33 | registryClient: registryClient, 34 | index: index, 35 | log: loggy.New("registryAPI"), 36 | } 37 | } 38 | 39 | func (r *registryAPI) ModulesNeeded(excludeIDs Ranges) ([]coordinates.SerialModule, error) { 40 | ids, err := r.index.IDs() 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | rm := registry.ReqMods{ 46 | IDs: ids, 47 | } 48 | 49 | bs, err := json.Marshal(rm) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | reader := bytes.NewReader(bs) 55 | 56 | var buf bytes.Buffer 57 | if err := r.registryClient.Post("/v1/registry/sources/list", reader, &buf); err != nil { 58 | return nil, err 59 | } 60 | 61 | r2 := bytes.NewReader(buf.Bytes()) 62 | 63 | var response registry.ReqModsResp 64 | if err := json.NewDecoder(r2).Decode(&response); err != nil { 65 | return nil, err 66 | } 67 | 68 | return response.Mods, nil 69 | } 70 | -------------------------------------------------------------------------------- /proxy/internal/web/mod_rm.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 11 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 12 | ) 13 | 14 | type removeModule struct { 15 | index store.Index 16 | store store.ZipStore 17 | emitter stats.Sender 18 | log loggy.Logger 19 | } 20 | 21 | func modRM(index store.Index, store store.ZipStore, emitter stats.Sender) http.Handler { 22 | return &removeModule{ 23 | store: store, 24 | emitter: emitter, 25 | index: index, 26 | log: loggy.New("mod-rm"), 27 | } 28 | } 29 | 30 | // e.g. POST http://localhost:9000/github.com/example/toolkit/@v/v1.0.0.rm 31 | 32 | func (h *removeModule) ServeHTTP(w http.ResponseWriter, r *http.Request) { 33 | mod, err := modInfoFromPath(r.URL.Path) 34 | if err != nil { 35 | http.Error(w, err.Error(), http.StatusBadRequest) 36 | h.emitter.Count("mod-rm-bad-request", 1) 37 | return 38 | } 39 | 40 | h.log.Infof("serving request for removal of %s", mod) 41 | 42 | if err := h.index.Remove(mod); err != nil { 43 | h.log.Errorf("failed to remove module from index %s: %v", mod, err) 44 | http.Error(w, err.Error(), http.StatusInternalServerError) 45 | return 46 | } 47 | 48 | // and remove from the store itself 49 | if err := h.store.DelZip(mod); err != nil { 50 | h.log.Errorf("failed to remove module from store %s: %v", mod, err) 51 | http.Error(w, err.Error(), http.StatusInternalServerError) 52 | return 53 | } 54 | 55 | msg := fmt.Sprintf("module %s removed", mod) 56 | w.WriteHeader(http.StatusOK) 57 | if _, err := io.WriteString(w, msg); err != nil { 58 | h.log.Errorf("failed to write response: %v", err) 59 | return 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /proxy/internal/service/proxy.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "os" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/clients/registry" 9 | "oss.indeed.com/go/modprox/pkg/clients/zips" 10 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 11 | "oss.indeed.com/go/modprox/pkg/webutil" 12 | "oss.indeed.com/go/modprox/proxy/config" 13 | "oss.indeed.com/go/modprox/proxy/internal/modules/bg" 14 | "oss.indeed.com/go/modprox/proxy/internal/modules/get" 15 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 16 | "oss.indeed.com/go/modprox/proxy/internal/problems" 17 | ) 18 | 19 | type Proxy struct { 20 | config config.Configuration 21 | middles []webutil.Middleware 22 | emitter stats.Sender 23 | index store.Index 24 | store store.ZipStore 25 | registryClient registry.Client 26 | proxyClient zips.ProxyClient 27 | upstreamClient zips.UpstreamClient 28 | downloader get.Downloader 29 | bgWorker bg.Worker 30 | dlTracker problems.Tracker 31 | log loggy.Logger 32 | history string 33 | } 34 | 35 | func NewProxy(configuration config.Configuration) *Proxy { 36 | p := &Proxy{ 37 | config: configuration, 38 | log: loggy.New("proxy-service"), 39 | } 40 | 41 | for _, f := range []initer{ 42 | initSender, 43 | initTrackers, 44 | initIndex, 45 | initStore, 46 | initRegistryClient, 47 | initZipClients, 48 | initBGWorker, 49 | initHeartbeatSender, 50 | initStartupConfigSender, 51 | initHistory, 52 | initWebServer, 53 | } { 54 | if err := f(p); err != nil { 55 | p.log.Errorf("failed to initialize proxy: %v", err) 56 | os.Exit(1) 57 | } 58 | } 59 | 60 | return p 61 | } 62 | 63 | func (p *Proxy) Run() { 64 | select { 65 | // intentionally left blank 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /registry/static/html/about.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

about modprox

6 |
7 |
8 |

Go Module Proxy

9 | 10 |

11 | The modprox project was created out of a need for a Go Module Proxy 12 | which has first-class support for enterprise Gitlab that has strong 13 | requirements for authenticated access. Existing proxies have/had no 14 | support for this use case, and so this project was started with an 15 | emphasis configurability and adaptability to the more "specialized" 16 | proxy use-cases. 17 |

18 |

19 | We think this proxy could be useful for other organizations, 20 | and so the development work has been open-source from the very beginning. 21 | Together, we hope to build a great open source Go Module 22 | Proxy focused on the internal hosting use case. 23 | Documentation for installing, configuring, and managing the 24 | modprox components can be found on 25 | modprox.org. 26 |

27 | 28 |
29 |

Open Source

30 |

31 | The source code for all of the modprox components is available 32 | on Github. 33 | Contributions are welcome! Particularly if you'd like to add some 34 | significant feature in support of additional "enterprise-y" use cases, 35 | we'd like to work towards getting those features merged! 36 |

37 | 38 |
39 |
40 | {{end}} 41 | -------------------------------------------------------------------------------- /pkg/clients/zips/client-upstream.go: -------------------------------------------------------------------------------- 1 | package zips 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | 6 | "oss.indeed.com/go/modprox/pkg/repository" 7 | "oss.indeed.com/go/modprox/pkg/upstream" 8 | ) 9 | 10 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i UpstreamClient -s _mock.go 11 | 12 | // UpstreamClient is used to download .zip files from an upstream origin 13 | // (e.g. github.com). The returned Blob is in a git archive format 14 | // that must be unpacked and repacked in the way that Go modules are 15 | // expected to be. This is done using Rewrite. 16 | type UpstreamClient interface { 17 | Get(*upstream.Request) (repository.Blob, error) 18 | Protocols() []string 19 | } 20 | 21 | func NewUpstreamClient(clients ...UpstreamClient) UpstreamClient { 22 | clientForProto := make(map[string]UpstreamClient, 1) 23 | for _, clientImpl := range clients { 24 | for _, protocol := range clientImpl.Protocols() { 25 | clientForProto[protocol] = clientImpl 26 | } 27 | } 28 | return &client{ 29 | clients: clientForProto, 30 | } 31 | } 32 | 33 | type client struct { 34 | clients map[string]UpstreamClient 35 | } 36 | 37 | func (c *client) Get(r *upstream.Request) (repository.Blob, error) { 38 | impl, err := c.getClientFor(r.Transport) 39 | if err != nil { 40 | return nil, err 41 | } 42 | return impl.Get(r) 43 | } 44 | 45 | func (c *client) Protocols() []string { 46 | protocols := make([]string, 0, len(c.clients)) 47 | for proto := range c.clients { 48 | protocols = append(protocols, proto) 49 | } 50 | return protocols 51 | } 52 | 53 | func (c *client) getClientFor(transport string) (UpstreamClient, error) { 54 | impl, exists := c.clients[transport] 55 | if !exists { 56 | return nil, errors.Errorf("no client that handles %q", transport) 57 | } 58 | return impl, nil 59 | } 60 | -------------------------------------------------------------------------------- /proxy/internal/status/heartbeat/sender.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 10 | "oss.indeed.com/go/modprox/pkg/clients/registry" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/netservice" 13 | ) 14 | 15 | const ( 16 | heartbeatPath = "/v1/proxy/heartbeat" 17 | ) 18 | 19 | // A Sender is used to send heartbeat status updates to the registry. 20 | type Sender interface { 21 | Send(int, int) error 22 | } 23 | 24 | type sender struct { 25 | registryClient registry.Client 26 | self netservice.Instance 27 | emitter stats.Sender 28 | log loggy.Logger 29 | } 30 | 31 | func NewSender( 32 | self netservice.Instance, 33 | registryClient registry.Client, 34 | emitter stats.Sender, 35 | ) Sender { 36 | 37 | return &sender{ 38 | registryClient: registryClient, 39 | self: self, 40 | emitter: emitter, 41 | log: loggy.New("heartbeat-sender"), 42 | } 43 | } 44 | 45 | func (s *sender) Send(numPackages, numModules int) error { 46 | heartbeat := payloads.Heartbeat{ 47 | Self: s.self, 48 | NumModules: numPackages, 49 | NumVersions: numModules, 50 | } 51 | 52 | s.log.Infof("sending a heartbeat: %s", heartbeat) 53 | 54 | bs, err := json.Marshal(heartbeat) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | reader := bytes.NewReader(bs) 60 | response := bytes.NewBuffer(nil) 61 | 62 | if err := s.registryClient.Post(heartbeatPath, reader, response); err != nil { 63 | s.emitter.Count("heartbeat-send-failure", 1) 64 | return err 65 | } 66 | 67 | s.log.Infof("heartbeat was successfully sent!") 68 | s.emitter.Count("heartbeat-send-ok", 1) 69 | 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/webutil/middleware.go: -------------------------------------------------------------------------------- 1 | package webutil 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | // A Middleware is used to execute intermediate Handlers 9 | // in response to a request. 10 | type Middleware func(http.Handler) http.Handler 11 | 12 | // Chain recursively chains middleware together. 13 | func Chain(h http.Handler, m ...Middleware) http.Handler { 14 | if len(m) == 0 { 15 | return h 16 | } 17 | return m[0](Chain(h, m[1:cap(m)]...)) 18 | } 19 | 20 | const ( 21 | HeaderAPIKey = "X-modprox-api-key" 22 | ) 23 | 24 | // KeyGuard creates a Middleware which protects access to a handler 25 | // by first checking for the X-modprox-api-key header being set. If 26 | // at least one of the values for the header matches one of the keys 27 | // configured for the KeyGuard, the handler is executed for the request. 28 | // Otherwise, a StatusForbidden response is returned. 29 | func KeyGuard(keys []string) Middleware { 30 | allowedKeys := make(map[string]bool) 31 | for _, key := range keys { 32 | allowedKeys[key] = true 33 | } 34 | 35 | return func(h http.Handler) http.Handler { 36 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 37 | // first check that the header is set 38 | key := r.Header.Get(HeaderAPIKey) 39 | if key == "" { 40 | msg := fmt.Sprintf("header %s is not set in request", HeaderAPIKey) 41 | http.Error(w, msg, http.StatusForbidden) 42 | return 43 | } 44 | 45 | // check if the given key is allowable 46 | if allowedKeys[key] { 47 | // found a good key, execute the 48 | // protected handler for the request 49 | h.ServeHTTP(w, r) 50 | return 51 | } 52 | 53 | // no good key was provided, respond with an error 54 | msg := fmt.Sprintf("header %s contains no valid keys", HeaderAPIKey) 55 | http.Error(w, msg, http.StatusForbidden) 56 | return 57 | }) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/clients/zips/client-proxy_test.go: -------------------------------------------------------------------------------- 1 | package zips 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "gophers.dev/pkgs/loggy" 12 | "gophers.dev/pkgs/semantic" 13 | ) 14 | 15 | func Test_mangle(t *testing.T) { 16 | try := func(source, exp string) { 17 | result := mangle(source) 18 | require.Equal(t, exp, result) 19 | } 20 | 21 | try("a.com", "a.com") 22 | try("A.com", "!a.com") 23 | try("a.COM", "a.!c!o!m") 24 | try("alpha.com/foo/bar", "alpha.com/foo/bar") 25 | try("alpha.com/Foo/BAr", "alpha.com/!foo/!b!ar") 26 | try("github.com/Azure/azure-sdk-for-go", "github.com/!azure/azure-sdk-for-go") 27 | try("github.com/GoogleCloudPlatform/cloudsql-proxy", "github.com/!google!cloud!platform/cloudsql-proxy") 28 | try("github.com/Sirupsen/logrus", "github.com/!sirupsen/logrus") 29 | } 30 | 31 | func TestProxyClient_List(t *testing.T) { 32 | httpClient := NewIHTTPClientMock(t) 33 | defer httpClient.MinimockFinish() 34 | 35 | const responseBody = `v0.1.0 36 | v0.2.0 37 | v0.3.0 38 | v0.3.1 39 | ` 40 | 41 | httpClient.DoMock.Set(func(req *http.Request) (rp1 *http.Response, err error) { 42 | require.Equal(t, "https://proxy.golang.org/github.com/foo/bar/@v/list", req.URL.String()) 43 | return &http.Response{Body: ioutil.NopCloser(strings.NewReader(responseBody)), StatusCode: http.StatusOK}, nil 44 | }) 45 | 46 | subject := &proxyClient{ 47 | httpClient: httpClient, 48 | baseURL: "proxy.golang.org", 49 | protocol: "https", 50 | log: loggy.New(""), 51 | } 52 | 53 | versions, err := subject.List("github.com/foo/bar") 54 | require.NoError(t, err) 55 | 56 | require.Equal(t, []semantic.Tag{ 57 | {Major: 0, Minor: 3, Patch: 1}, 58 | {Major: 0, Minor: 3, Patch: 0}, 59 | {Major: 0, Minor: 2, Patch: 0}, 60 | {Major: 0, Minor: 1, Patch: 0}, 61 | }, versions) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/metrics/stats/sender.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/cactus/go-statsd-client/statsd" 8 | 9 | "oss.indeed.com/go/modprox/pkg/since" 10 | ) 11 | 12 | type Service string 13 | 14 | func (s Service) String() string { 15 | return string(s) 16 | } 17 | 18 | const ( 19 | Proxy Service = "modprox-proxy" 20 | Registry Service = "modprox-registry" 21 | ) 22 | 23 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Sender -s _mock.go 24 | 25 | // A Sender is used to emit statsd type metrics. 26 | type Sender interface { 27 | Count(metric string, i int) 28 | Gauge(metric string, n int) 29 | GaugeMS(metric string, t time.Time) 30 | } 31 | 32 | // New creates a new Sender which will send metrics to the receiver described 33 | // by the cfg.Agent configuration. All metrics will be emittited under the 34 | // application named by Service s. 35 | func New(s Service, cfg Statsd) (Sender, error) { 36 | address := fmt.Sprintf("%s:%d", cfg.Agent.Address, cfg.Agent.Port) 37 | emitter, err := statsd.NewClient(address, s.String()) 38 | return &sender{ 39 | emitter: emitter, 40 | }, err 41 | } 42 | 43 | type discard struct{} 44 | 45 | func (d *discard) Count(string, int) {} 46 | func (d *discard) Gauge(string, int) {} 47 | func (d *discard) GaugeMS(string, time.Time) {} 48 | 49 | func Discard() Sender { 50 | return &discard{} 51 | } 52 | 53 | type sender struct { 54 | emitter statsd.Statter 55 | } 56 | 57 | func (s *sender) Count(metric string, n int) { 58 | _ = s.emitter.Inc(metric, 1, 1) 59 | } 60 | 61 | func (s *sender) Gauge(metric string, n int) { 62 | _ = s.emitter.Gauge(metric, int64(n), 1) 63 | } 64 | 65 | // GaugeMS gauges the amount of time that has elapsed since t in milliseconds. 66 | func (s *sender) GaugeMS(metric string, t time.Time) { 67 | _ = s.emitter.Gauge(metric, since.MS(t), 1) 68 | } 69 | -------------------------------------------------------------------------------- /registry/internal/data/store.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | 8 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 9 | "oss.indeed.com/go/modprox/pkg/coordinates" 10 | "oss.indeed.com/go/modprox/pkg/database" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/netservice" 13 | "oss.indeed.com/go/modprox/pkg/setup" 14 | ) 15 | 16 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Store -s _mock.go 17 | 18 | type Store interface { 19 | // modules 20 | ListModuleIDs() ([]int64, error) 21 | ListModulesByIDs(ids []int64) ([]coordinates.SerialModule, error) 22 | ListModulesBySource(source string) ([]coordinates.SerialModule, error) 23 | ListModules() ([]coordinates.SerialModule, error) 24 | InsertModules([]coordinates.Module) (int, error) 25 | DeleteModuleByID(id int) error 26 | 27 | // startup configs and payloads 28 | SetStartConfig(payloads.Configuration) error 29 | ListStartConfigs() ([]payloads.Configuration, error) 30 | SetHeartbeat(payloads.Heartbeat) error 31 | ListHeartbeats() ([]payloads.Heartbeat, error) 32 | PurgeProxy(instance netservice.Instance) error 33 | } 34 | 35 | func Connect(kind string, dsn setup.DSN, emitter stats.Sender) (Store, error) { 36 | db, err := database.Connect(kind, dsn) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | return New(kind, db, emitter) 42 | } 43 | 44 | func New(kind string, db *sql.DB, emitter stats.Sender) (Store, error) { 45 | statements, err := load(kind, db) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return &store{ 50 | emitter: emitter, 51 | db: db, 52 | statements: statements, 53 | log: loggy.New("store"), 54 | }, nil 55 | } 56 | 57 | type store struct { 58 | emitter stats.Sender 59 | db *sql.DB 60 | statements statements 61 | log loggy.Logger 62 | } 63 | -------------------------------------------------------------------------------- /registry/static/html/mods_add.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

register new modules

6 |
7 |
8 |

9 | paste content of go.sum file
10 | paste the require section of go.mod file
11 | common module formats accepted 12 |

13 |
14 |
15 |
16 | {{.CSRF}} 17 | 23 |

24 | 25 |
26 |
27 | 28 |
29 | {{if .Mods}} 30 |
31 | 32 | {{range .Mods}} 33 | 34 | {{if not .Err}} 35 | 40 | 41 | 44 | {{else}} 45 | 50 | 51 | 56 | {{end}} 57 | 58 | {{end}} 59 |
36 | 37 | {{.Module.Source}} {{.Module.Version}} 38 | 39 | => 42 | OK 43 | 46 | 47 | {{.Text}} 48 | 49 | => 52 | 53 | {{.Err.Error}} 54 | 55 |
60 | {{end}} 61 |
62 |
63 | {{end}} 64 | -------------------------------------------------------------------------------- /registry/internal/web/mods_list.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | "sort" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/registry/internal/data" 13 | "oss.indeed.com/go/modprox/registry/static" 14 | ) 15 | 16 | type modsListPage struct { 17 | Mods map[string][]string // pkg => []version 18 | } 19 | 20 | type modsListHandler struct { 21 | html *template.Template 22 | store data.Store 23 | emitter stats.Sender 24 | log loggy.Logger 25 | } 26 | 27 | func newModsListHandler(store data.Store, emitter stats.Sender) http.Handler { 28 | html := static.MustParseTemplates( 29 | "static/html/layout.html", 30 | "static/html/navbar.html", 31 | "static/html/mods_list.html", 32 | ) 33 | 34 | return &modsListHandler{ 35 | html: html, 36 | store: store, 37 | emitter: emitter, 38 | log: loggy.New("list-modules-handler"), 39 | } 40 | } 41 | 42 | func (h *modsListHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 43 | code, page, err := h.get(r) 44 | if err != nil { 45 | h.log.Errorf("failed to serve modules list page") 46 | http.Error(w, err.Error(), code) 47 | h.emitter.Count("ui-list-mods-error", 1) 48 | return 49 | } 50 | 51 | if err := h.html.Execute(w, page); err != nil { 52 | h.log.Errorf("failed to execute modules list page") 53 | return 54 | } 55 | 56 | h.emitter.Count("ui-list-mods-ok", 1) 57 | } 58 | 59 | func (h *modsListHandler) get(r *http.Request) (int, *modsListPage, error) { 60 | mods, err := h.store.ListModules() 61 | if err != nil { 62 | return http.StatusInternalServerError, nil, err 63 | } 64 | 65 | tree := treeOfMods(mods) 66 | page := &modsListPage{ 67 | Mods: tree, 68 | } 69 | 70 | return http.StatusOK, page, nil 71 | } 72 | 73 | func treeOfMods(mods []coordinates.SerialModule) map[string][]string { 74 | m := make(map[string][]string) 75 | for _, mod := range mods { 76 | m[mod.Source] = append(m[mod.Source], mod.Version) 77 | } 78 | 79 | for _, mod := range mods { 80 | sort.Strings(m[mod.Source]) 81 | } 82 | 83 | return m 84 | } 85 | -------------------------------------------------------------------------------- /registry/internal/web/v1_startup.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io" 7 | "net/http" 8 | 9 | "gophers.dev/pkgs/loggy" 10 | 11 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 12 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 13 | "oss.indeed.com/go/modprox/registry/internal/data" 14 | ) 15 | 16 | type startupHandler struct { 17 | store data.Store 18 | emitter stats.Sender 19 | log loggy.Logger 20 | } 21 | 22 | func newStartupHandler(store data.Store, emitter stats.Sender) http.Handler { 23 | return &startupHandler{ 24 | store: store, 25 | emitter: emitter, 26 | log: loggy.New("startup-config-handler"), 27 | } 28 | } 29 | 30 | func (h *startupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 31 | h.log.Tracef("receiving startup configuration from proxy") 32 | 33 | code, msg, err := h.post(r) 34 | if err != nil { 35 | h.log.Errorf("failed to accept startup configuration from %s, %v", r.RemoteAddr, err) 36 | http.Error(w, msg, code) 37 | h.emitter.Count("api-proxy-start-config-error", 1) 38 | return 39 | } 40 | 41 | h.log.Tracef("accepted startup configuration from %s", r.RemoteAddr) 42 | _, _ = io.WriteString(w, "ok") 43 | h.emitter.Count("api-proxy-start-config-ok", 1) 44 | } 45 | 46 | func (h *startupHandler) post(r *http.Request) (int, string, error) { 47 | // proxy should probably send an Instance to identify itself 48 | var configuration payloads.Configuration 49 | 50 | if err := json.NewDecoder(r.Body).Decode(&configuration); err != nil { 51 | return http.StatusBadRequest, "failed to decode request", err 52 | } 53 | 54 | if err := checkConfiguration(configuration); err != nil { 55 | return http.StatusBadRequest, "configuration is not valid", err 56 | } 57 | 58 | if err := h.store.SetStartConfig(configuration); err != nil { 59 | return http.StatusInternalServerError, "failed to save configuration", err 60 | } 61 | 62 | return http.StatusOK, "ok", nil 63 | } 64 | 65 | func checkConfiguration(configuration payloads.Configuration) error { 66 | switch { 67 | case len(configuration.Registry.Instances) == 0: 68 | return errors.New("registries configuration cannot be empty") 69 | } 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /proxy/internal/status/startup/sender.go: -------------------------------------------------------------------------------- 1 | package startup 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "time" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 11 | "oss.indeed.com/go/modprox/pkg/clients/registry" 12 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 13 | ) 14 | 15 | const ( 16 | configurationPath = "/v1/proxy/configuration" 17 | ) 18 | 19 | // A Sender is used to send startup configuration state to the registry. 20 | type Sender interface { 21 | Send(configuration payloads.Configuration) error 22 | } 23 | 24 | type sender struct { 25 | registryClient registry.Client 26 | retryInterval time.Duration 27 | emitter stats.Sender 28 | log loggy.Logger 29 | } 30 | 31 | func NewSender( 32 | registryClient registry.Client, 33 | retryInterval time.Duration, 34 | emitter stats.Sender, 35 | ) Sender { 36 | return &sender{ 37 | registryClient: registryClient, 38 | retryInterval: retryInterval, 39 | emitter: emitter, 40 | log: loggy.New("startup-config-sender"), 41 | } 42 | } 43 | 44 | func (s *sender) Send(configuration payloads.Configuration) error { 45 | 46 | // optimistically try immediately to start with 47 | if err := s.trySend(configuration); err == nil { 48 | return nil 49 | } 50 | 51 | // didn't work; keep trying every 30 seconds until it works 52 | ticker := time.NewTicker(s.retryInterval) 53 | defer ticker.Stop() 54 | 55 | for range ticker.C { 56 | if err := s.trySend(configuration); err == nil { 57 | break 58 | } else { 59 | s.log.Warnf("failed to contact registry; will try again in 30s") 60 | } 61 | } 62 | return nil 63 | } 64 | 65 | func (s *sender) trySend(configuration payloads.Configuration) error { 66 | bs, err := json.Marshal(configuration) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | reader := bytes.NewReader(bs) 72 | response := bytes.NewBuffer(nil) 73 | if err := s.registryClient.Post(configurationPath, reader, response); err != nil { 74 | s.emitter.Count("startup-config-send-failure", 1) 75 | return err 76 | } 77 | 78 | s.log.Infof("startup configuration successfully sent!") 79 | s.emitter.Count("startup-config-send-ok", 1) 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /registry/static/html/navbar.html: -------------------------------------------------------------------------------- 1 | {{define "navbar"}} 2 | 54 | {{end}} 55 | -------------------------------------------------------------------------------- /pkg/upstream/request.go: -------------------------------------------------------------------------------- 1 | package upstream 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // Namespace is the path elements leading up to the package name of a module. 9 | type Namespace []string 10 | 11 | // A Request for a git-archive of a module. 12 | // 13 | // A Request should only be used in the context of acquiring a module archive 14 | // from an upstream source (e.g. github, gitlab, etc.) as opposed to acquiring 15 | // a module from a proxy. 16 | type Request struct { 17 | Transport string 18 | Domain string 19 | Namespace Namespace 20 | Version string 21 | Path string 22 | GoGetRedirect bool 23 | Headers map[string]string 24 | } 25 | 26 | func (r *Request) String() string { 27 | return fmt.Sprintf( 28 | "[%q %q %v %q %q %t]", 29 | r.Transport, 30 | r.Domain, 31 | r.Namespace, 32 | r.Version, 33 | r.Path, 34 | r.GoGetRedirect, 35 | ) 36 | } 37 | 38 | // An explicit implementation of equality between two Request objects. 39 | func (r *Request) Equals(o *Request) bool { 40 | if r.Transport != o.Transport { 41 | return false 42 | } 43 | 44 | if r.Domain != o.Domain { 45 | return false 46 | } 47 | 48 | if len(r.Namespace) != len(o.Namespace) { 49 | return false 50 | } 51 | 52 | for i := 0; i < len(r.Namespace); i++ { 53 | if r.Namespace[i] != o.Namespace[i] { 54 | return false 55 | } 56 | } 57 | 58 | if r.Version != o.Version { 59 | return false 60 | } 61 | 62 | if r.Path != o.Path { 63 | return false 64 | } 65 | 66 | if r.GoGetRedirect != o.GoGetRedirect { 67 | return false 68 | } 69 | 70 | if len(r.Headers) != len(o.Headers) { 71 | return false 72 | } 73 | 74 | for key := range r.Headers { 75 | if r.Headers[key] != o.Headers[key] { 76 | return false 77 | } 78 | } 79 | 80 | return true 81 | } 82 | 83 | // The URI is only valid AFTER a Request has passed through 84 | // all of the Transform functors. 85 | // 86 | // The URI should represent the way to get some git-archive, which must 87 | // later be transformed into a proper module archive before the Go tooling 88 | // will be able to work with it. 89 | func (r *Request) URI() string { 90 | rPath := strings.TrimPrefix(r.Path, "/") 91 | return fmt.Sprintf("%s://%s/%s", r.Transport, r.Domain, rPath) 92 | } 93 | -------------------------------------------------------------------------------- /proxy/internal/problems/tracker_test.go: -------------------------------------------------------------------------------- 1 | package problems 2 | 3 | import ( 4 | "errors" 5 | "sort" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "oss.indeed.com/go/modprox/pkg/coordinates" 12 | ) 13 | 14 | func Test_Tracker_empty_default(t *testing.T) { 15 | pt := New("foo") 16 | 17 | problems := pt.Problems() 18 | require.Equal(t, 0, len(problems)) 19 | 20 | _, exists := pt.Problem(coordinates.Module{ 21 | Source: "github.com/foo/bar", 22 | Version: "1.2.3", 23 | }) 24 | require.False(t, exists) 25 | } 26 | 27 | func Test_Tracker_Set_one(t *testing.T) { 28 | pt := New("foo") 29 | 30 | pt.Set(Problem{ 31 | Module: coordinates.Module{ 32 | Source: "github.com/foo/bar", 33 | Version: "1.2.3", 34 | }, 35 | Time: time.Date(2018, 12, 2, 20, 0, 0, 0, time.UTC), 36 | Message: "foobar is broken", 37 | }) 38 | 39 | problems := pt.Problems() 40 | require.Equal(t, 1, len(problems)) 41 | 42 | problem, exists := pt.Problem(coordinates.Module{ 43 | Source: "github.com/foo/bar", 44 | Version: "1.2.3", 45 | }) 46 | require.True(t, exists) 47 | require.Equal(t, "foobar is broken", problem.Message) 48 | } 49 | 50 | func Test_byName(t *testing.T) { 51 | mod1 := coordinates.Module{ 52 | Source: "github.com/zzz/bar", 53 | Version: "1.0.0", 54 | } 55 | 56 | mod2 := coordinates.Module{ 57 | Source: "github.com/aaa/bar", 58 | Version: "2.2.3", 59 | } 60 | 61 | mod3 := coordinates.Module{ 62 | Source: "github.com/foo/bar", 63 | Version: "1.2.3", 64 | } 65 | 66 | mod4 := coordinates.Module{ 67 | Source: "github.com/aaa/bar", 68 | Version: "1.2.3", 69 | } 70 | 71 | mod5 := coordinates.Module{ 72 | Source: "github.com/bbb/bar", 73 | Version: "1.2.3", 74 | } 75 | 76 | mod6 := coordinates.Module{ 77 | Source: "github.com/foo/ccc", 78 | Version: "1.2.3", 79 | } 80 | 81 | problems := []Problem{ 82 | Create(mod1, errors.New("m1")), 83 | Create(mod2, errors.New("m2")), 84 | Create(mod3, errors.New("m3")), 85 | Create(mod4, errors.New("m4")), 86 | Create(mod5, errors.New("m5")), 87 | Create(mod6, errors.New("m6")), 88 | } 89 | 90 | sort.Sort(byName(problems)) 91 | 92 | // mod2 comes before mod4 in time 93 | require.Equal(t, mod2, problems[0].Module) 94 | 95 | // mod1 is zzz 96 | require.Equal(t, mod1, problems[5].Module) 97 | } 98 | -------------------------------------------------------------------------------- /proxy/internal/problems/tracker.go: -------------------------------------------------------------------------------- 1 | package problems 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | "time" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | ) 12 | 13 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Tracker -s _mock.go 14 | 15 | type Tracker interface { 16 | Set(Problem) 17 | Problem(module coordinates.Module) (Problem, bool) 18 | Problems() []Problem 19 | } 20 | 21 | type Problem struct { 22 | Module coordinates.Module `json:"module"` 23 | Time time.Time `json:"time"` 24 | Message string `json:"message"` 25 | } 26 | 27 | func Create(mod coordinates.Module, err error) Problem { 28 | return Problem{ 29 | Module: mod, 30 | Time: time.Now(), 31 | Message: err.Error(), 32 | } 33 | } 34 | 35 | type tracker struct { 36 | log loggy.Logger 37 | lock sync.RWMutex 38 | problems map[coordinates.Module]Problem 39 | } 40 | 41 | func New(name string) Tracker { 42 | return &tracker{ 43 | log: loggy.New("problems-" + name), 44 | problems: make(map[coordinates.Module]Problem), 45 | } 46 | } 47 | 48 | func (t *tracker) Set(problem Problem) { 49 | t.log.Tracef("setting problem for module %s", problem.Module) 50 | 51 | t.lock.Lock() 52 | defer t.lock.Unlock() 53 | 54 | t.problems[problem.Module] = problem 55 | } 56 | 57 | func (t *tracker) Problem(mod coordinates.Module) (Problem, bool) { 58 | t.lock.RLock() 59 | defer t.lock.RUnlock() 60 | 61 | p, exists := t.problems[mod] 62 | return p, exists 63 | } 64 | 65 | func (t *tracker) Problems() []Problem { 66 | t.lock.RLock() 67 | defer t.lock.RUnlock() 68 | 69 | problems := make([]Problem, 0, len(t.problems)) 70 | for _, p := range t.problems { 71 | problems = append(problems, p) 72 | } 73 | 74 | sort.Sort(byName(problems)) 75 | return problems 76 | } 77 | 78 | type byName []Problem 79 | 80 | func (p byName) Len() int { return len(p) } 81 | func (p byName) Swap(x, y int) { p[x], p[y] = p[y], p[x] } 82 | func (p byName) Less(x, y int) bool { 83 | modX, modY := p[x], p[y] 84 | 85 | if modX.Module.Source < modY.Module.Source { 86 | return true 87 | } else if modX.Module.Source > modY.Module.Source { 88 | return false 89 | } 90 | 91 | // don't bother parsing the tags, nobody cares 92 | // we just need something deterministic 93 | if modX.Time.Before(modY.Time) { 94 | return true 95 | } 96 | return false 97 | } 98 | -------------------------------------------------------------------------------- /registry/internal/data/sql.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | const ( 10 | insertModuleSQL = iota 11 | selectModuleIDSQL 12 | selectModulesBySource 13 | selectModuleIDScanSQL 14 | selectModulesByIDsSQL 15 | selectSourcesScanSQL 16 | insertHeartbeatSQL 17 | insertStartupConfigSQL 18 | selectStartupConfigsSQL 19 | selectHeartbeatsSQL 20 | deleteHeartbeatSQL 21 | deleteStartupConfigSQL 22 | deleteModuleByIDSQL 23 | ) 24 | 25 | type statements map[int]*sql.Stmt 26 | 27 | func load(kind string, db *sql.DB) (statements, error) { 28 | loaded := make(statements, len(mySQLTexts)) 29 | 30 | stmtTexts := mySQLTexts 31 | for id, text := range stmtTexts { 32 | if text != "" { // avoid loading selectModulesByIDsSQL for mysql; must be generated 33 | stmt, err := db.Prepare(text) 34 | if err != nil { 35 | return nil, errors.Wrapf(err, "bad sql statement: %q", text) 36 | } 37 | loaded[id] = stmt 38 | } 39 | } 40 | 41 | return loaded, nil 42 | } 43 | 44 | var ( 45 | mySQLTexts = map[int]string{ 46 | insertModuleSQL: `insert into modules(source, version) values (?, ?)`, 47 | selectModuleIDSQL: `select id from modules where source=? and version=?`, 48 | selectModulesBySource: `select id, source, version from modules where source=?`, 49 | selectModuleIDScanSQL: `select id from modules order by id asc`, 50 | selectModulesByIDsSQL: ``, // select id, source, version from modules where id in(?) order by id asc`, 51 | selectSourcesScanSQL: `select id, source, version from modules`, 52 | insertHeartbeatSQL: `insert into proxy_heartbeats (hostname, port, num_modules, num_versions) values (?, ?, ?, ?) on duplicate key update num_modules=?, num_versions=?, ts=current_timestamp;`, 53 | insertStartupConfigSQL: `insert into proxy_configurations (hostname, port, storage, registry, transforms) values (?, ?, ?, ?, ?) on duplicate key update storage=?, registry=?, transforms=?`, 54 | selectStartupConfigsSQL: `select hostname, port, storage, registry, transforms from proxy_configurations`, 55 | selectHeartbeatsSQL: `select hostname, port, num_modules, num_versions, unix_timestamp(ts) from proxy_heartbeats`, 56 | deleteHeartbeatSQL: `delete from proxy_heartbeats where hostname=? and port=? limit 1`, 57 | deleteStartupConfigSQL: `delete from proxy_configurations where hostname=? and port=? limit 1`, 58 | deleteModuleByIDSQL: `delete from modules where id=?`, 59 | } 60 | ) 61 | -------------------------------------------------------------------------------- /registry/internal/web/v1_heartbeat.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "net/http" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/netservice" 13 | "oss.indeed.com/go/modprox/pkg/webutil" 14 | "oss.indeed.com/go/modprox/registry/internal/data" 15 | ) 16 | 17 | type heartbeatHandler struct { 18 | store data.Store 19 | emitter stats.Sender 20 | log loggy.Logger 21 | } 22 | 23 | func newHeartbeatHandler(store data.Store, emitter stats.Sender) http.Handler { 24 | return &heartbeatHandler{ 25 | store: store, 26 | emitter: emitter, 27 | log: loggy.New("heartbeat-update-handler"), 28 | } 29 | } 30 | 31 | func (h *heartbeatHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 32 | code, response, from, err := h.post(r) 33 | 34 | if err != nil { 35 | h.log.Errorf("failed to accept heartbeat from %s, %v", from, err) 36 | http.Error(w, response, code) 37 | h.emitter.Count("api-heartbeat-unaccepted", 1) 38 | return 39 | } 40 | 41 | h.log.Tracef("accepted heartbeat from %s", from) 42 | webutil.WriteJSON(w, response) 43 | h.emitter.Count("api-heartbeat-accepted", 1) 44 | } 45 | 46 | func (h *heartbeatHandler) post(r *http.Request) (int, string, netservice.Instance, error) { 47 | var from netservice.Instance 48 | var heartbeat payloads.Heartbeat 49 | if err := json.NewDecoder(r.Body).Decode(&heartbeat); err != nil { 50 | return http.StatusBadRequest, "failed to decode request", from, err 51 | } 52 | 53 | if err := checkHeartbeat(heartbeat); err != nil { 54 | return http.StatusBadRequest, "heartbeat is not valid", from, err 55 | } 56 | 57 | from = heartbeat.Self 58 | 59 | if err := h.store.SetHeartbeat(heartbeat); err != nil { 60 | return http.StatusInternalServerError, "failed to save heartbeat", from, err 61 | } 62 | 63 | return http.StatusOK, "ok", from, nil 64 | } 65 | 66 | func checkHeartbeat(heartbeat payloads.Heartbeat) error { 67 | switch { 68 | case heartbeat.Self.Address == "": 69 | return errors.New("heartbeat address cannot be empty") 70 | case heartbeat.Self.Port <= 0: 71 | return errors.New("heartbeat port must be positive") 72 | case heartbeat.NumModules < 0: 73 | return errors.New("heartbeat num_packages must be non-negative") 74 | case heartbeat.NumVersions < 0: 75 | return errors.New("heartbeat num_modules must be non-negative") 76 | } 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /pkg/clients/zips/rewrite_test.go: -------------------------------------------------------------------------------- 1 | package zips 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func Test_majorVersion(t *testing.T) { 10 | tryMajorVersion(t, "v2.0.4", "v2", false) 11 | tryMajorVersion(t, "v1.0.0", "", false) 12 | tryMajorVersion(t, "v0.0.1", "", false) 13 | tryMajorVersion(t, "blah", "", true) 14 | tryMajorVersion(t, "v123", "", true) 15 | } 16 | 17 | func tryMajorVersion(t *testing.T, version, expectedMajor string, expectError bool) { 18 | m, err := majorVersion(version) 19 | if expectError { 20 | require.Error(t, err) 21 | } else { 22 | require.NoError(t, err) 23 | require.Equal(t, expectedMajor, m) 24 | } 25 | } 26 | 27 | func Test_ModulePath(t *testing.T) { 28 | gomod := `module github.com/modprox/mp 29 | 30 | require ( 31 | github.com/googleapis/gax-go/v2 v2.0.4 32 | google.golang.org/grpc 1.19.0 33 | ) 34 | ` 35 | expected := "github.com/modprox/mp" 36 | 37 | s := ModulePath([]byte(gomod)) 38 | 39 | require.Equal(t, expected, s) 40 | } 41 | 42 | func Test_ModulePath_none(t *testing.T) { 43 | gomod := `// I absent-mindedly commented out the module line 44 | //module github.com/modprox/mp` 45 | 46 | s := ModulePath([]byte(gomod)) 47 | require.Equal(t, "", s) 48 | } 49 | 50 | func Test_moduleOf(t *testing.T) { 51 | goModPath := map[string]string{ 52 | "github.com/billsmith/module1-1.1.4": "github.com/billsmith/module1", 53 | "github.com/billsmith/module1/v2": "github.com/billsmith/module1/v2", 54 | "github.com/billsmith/module1/v3": "github.com/billsmith/module1/v3", 55 | } 56 | 57 | require.Equal(t, "github.com/billsmith/module1", moduleOf(goModPath, "github.com/billsmith/module1-1.1.4/main.go")) 58 | } 59 | 60 | func Test_moduleOf_v2(t *testing.T) { 61 | goModPath := map[string]string{ 62 | "github.com/billsmith/module1-2.0.9": "github.com/billsmith/module1", 63 | "github.com/billsmith/module1-2.0.9/v2": "github.com/billsmith/module1/v2", 64 | "github.com/billsmith/module1-2.0.9/v3": "github.com/billsmith/module1/v3", 65 | } 66 | 67 | require.Equal(t, "github.com/billsmith/module1/v2", moduleOf(goModPath, "github.com/billsmith/module1-2.0.9/v2/main.go")) 68 | } 69 | 70 | func Test_moduleOf_v4(t *testing.T) { 71 | goModPath := map[string]string{ 72 | "github.com/billsmith/module1/v2": "github.com/billsmith/module1/v2", 73 | "github.com/billsmith/module1/v3": "github.com/billsmith/module1/v3", 74 | } 75 | 76 | require.Equal(t, "", moduleOf(goModPath, "github.com/billsmith/module1/v4/main.go")) 77 | } 78 | -------------------------------------------------------------------------------- /proxy/internal/web/router.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/gorilla/mux" 8 | 9 | "gophers.dev/pkgs/loggy" 10 | 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/webutil" 13 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 14 | "oss.indeed.com/go/modprox/proxy/internal/problems" 15 | ) 16 | 17 | const ( 18 | get = http.MethodGet 19 | post = http.MethodPost 20 | ) 21 | 22 | func NewRouter( 23 | middles []webutil.Middleware, 24 | index store.Index, 25 | store store.ZipStore, 26 | emitter stats.Sender, 27 | dlProblems problems.Tracker, 28 | history string, 29 | ) http.Handler { 30 | 31 | router := mux.NewRouter() 32 | 33 | // mod operations 34 | // 35 | // e.g. GET http://localhost:9000/github.com/example/toolkit/@v/v1.0.0.info 36 | // e.g. GET http://localhost:9000/github.com/example/toolkit/@v.list 37 | // e.g. POST http://localhost:9000/github.com/example/toolkit/@v/v1.0.0.rm 38 | router.PathPrefix("/").Handler(modList(index, emitter)).MatcherFunc(suffix("list")).Methods(get) 39 | router.PathPrefix("/").Handler(modInfo(index, emitter)).MatcherFunc(suffix(".info")).Methods(get) 40 | router.PathPrefix("/").Handler(modFile(index, emitter)).MatcherFunc(suffix(".mod")).Methods(get) 41 | router.PathPrefix("/").Handler(modZip(store, emitter)).MatcherFunc(suffix(".zip")).Methods(get) 42 | router.PathPrefix("/").Handler(modRM(index, store, emitter)).MatcherFunc(suffix(".rm")).Methods(post) 43 | 44 | // metadata about this app 45 | router.PathPrefix("/history").Handler(appHistory(emitter, history)).Methods(get) 46 | 47 | // api operations 48 | // 49 | router.PathPrefix("/v1/problems/downloads").Handler(newDownloadProblems(dlProblems, emitter)).Methods(get) 50 | 51 | // default behavior (404) 52 | router.PathPrefix("/").HandlerFunc(notFound(emitter)) 53 | 54 | // force middleware 55 | return webutil.Chain(router, middles...) 56 | } 57 | 58 | func suffix(s string) mux.MatcherFunc { 59 | log := loggy.New("suffix-match") 60 | 61 | return func(r *http.Request, rm *mux.RouteMatch) bool { 62 | match := strings.HasSuffix(r.URL.Path, s) 63 | log.Tracef("request from %s matches suffix %q: %t", r.RemoteAddr, s, match) 64 | return match 65 | } 66 | } 67 | 68 | func notFound(emitter stats.Sender) http.HandlerFunc { 69 | log := loggy.New("not-found") 70 | return func(w http.ResponseWriter, r *http.Request) { 71 | log.Infof("request from %s wanted %q which is not found", r.RemoteAddr, r.URL.String()) 72 | http.Error(w, "not found", http.StatusNotFound) 73 | emitter.Count("path-not-found", 1) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /registry/internal/web/home.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "html/template" 6 | "net/http" 7 | 8 | "gophers.dev/pkgs/loggy" 9 | 10 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/proxy/config" 13 | "oss.indeed.com/go/modprox/registry/internal/data" 14 | "oss.indeed.com/go/modprox/registry/static" 15 | ) 16 | 17 | type ProxyState struct { 18 | Heartbeat payloads.Heartbeat 19 | Configuration payloads.Configuration 20 | TransformsText string 21 | } 22 | 23 | type homePage struct { 24 | Proxies []ProxyState 25 | } 26 | 27 | type homeHandler struct { 28 | html *template.Template 29 | store data.Store 30 | emitter stats.Sender 31 | log loggy.Logger 32 | } 33 | 34 | func newHomeHandler(store data.Store, emitter stats.Sender) http.Handler { 35 | html := static.MustParseTemplates( 36 | "static/html/layout.html", 37 | "static/html/navbar.html", 38 | "static/html/home.html", 39 | ) 40 | return &homeHandler{ 41 | html: html, 42 | store: store, 43 | emitter: emitter, 44 | log: loggy.New("home-handler"), 45 | } 46 | } 47 | 48 | func (h *homeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 49 | configs, err := h.store.ListStartConfigs() 50 | if err != nil { 51 | http.Error(w, "failed to list proxy configs", http.StatusInternalServerError) 52 | h.log.Errorf("failed to list proxy configs: %v", err) 53 | h.emitter.Count("ui-home-error", 1) 54 | return 55 | } 56 | 57 | heartbeats, err := h.store.ListHeartbeats() 58 | if err != nil { 59 | http.Error(w, "failed to list proxy heartbeats", http.StatusInternalServerError) 60 | h.log.Errorf("failed to list proxy heartbeats: %v", err) 61 | h.emitter.Count("ui-home-error", 1) 62 | return 63 | } 64 | 65 | var proxyStates []ProxyState 66 | for _, c := range configs { // could be more efficient 67 | state := ProxyState{ 68 | Configuration: c, 69 | TransformsText: transformsText(c.Transforms), 70 | } 71 | for _, h := range heartbeats { 72 | if c.Self == h.Self { 73 | state.Heartbeat = h 74 | break 75 | } 76 | } 77 | proxyStates = append(proxyStates, state) 78 | } 79 | 80 | page := homePage{ 81 | Proxies: proxyStates, 82 | } 83 | 84 | if err := h.html.Execute(w, page); err != nil { 85 | h.log.Errorf("failed to execute homepage template: %v", err) 86 | return 87 | } 88 | 89 | h.emitter.Count("ui-home-ok", 1) 90 | } 91 | 92 | func transformsText(t config.Transforms) string { 93 | bs, err := json.MarshalIndent(t, "", " ") 94 | if err != nil { 95 | return "{}" 96 | } 97 | return string(bs) 98 | } 99 | -------------------------------------------------------------------------------- /registry/internal/tools/finder/finder_test.go: -------------------------------------------------------------------------------- 1 | package finder 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "gophers.dev/pkgs/semantic" 13 | 14 | "oss.indeed.com/go/modprox/pkg/clients/zips" 15 | ) 16 | 17 | func Test_Compatible(t *testing.T) { 18 | try := func(input string, exp bool) { 19 | result := Compatible(input) 20 | require.Equal(t, exp, result) 21 | } 22 | 23 | // github OK 24 | try("github.com/foo/bar", true) 25 | try("github.com/foo/bar/baz", true) 26 | try("github.com/sean-/seed", true) 27 | 28 | // github NOT OK 29 | try("github.com/foo", false) 30 | try("github.com", false) 31 | try("github", false) 32 | 33 | // nothing else supported 34 | try("golang.org/x/y", false) 35 | try("", false) 36 | } 37 | 38 | func Test_finder_Find(t *testing.T) { 39 | ts := httptest.NewServer(http.HandlerFunc( 40 | func(w http.ResponseWriter, r *http.Request) { 41 | if strings.HasSuffix(r.URL.String(), "/tags") { 42 | _, err := w.Write([]byte(tags)) 43 | require.NoError(t, err) 44 | } else { 45 | _, err := w.Write([]byte(head)) 46 | require.NoError(t, err) 47 | } 48 | }), 49 | ) 50 | defer ts.Close() 51 | 52 | client := &http.Client{ 53 | Timeout: 1 * time.Second, 54 | } 55 | 56 | const source = "github.com/octocat/Hello-World" 57 | 58 | proxyClient := zips.NewProxyClientMock(t) 59 | defer proxyClient.MinimockFinish() 60 | proxyClient.ListMock.Expect(source).Return([]semantic.Tag{}, nil) 61 | 62 | f := New(Options{ 63 | Timeout: 1 * time.Second, 64 | Versions: map[string]Versions{ 65 | "github.com": Github(ts.URL, client, proxyClient), 66 | }, 67 | }) 68 | 69 | result, err := f.Find(source) 70 | require.NoError(t, err) 71 | 72 | t.Logf("result %#v", result) 73 | } 74 | 75 | const tags = ` 76 | [ 77 | { 78 | "name": "v0.1", 79 | "commit": { 80 | "sha": "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc", 81 | "url": "https://api.github.com/repos/octocat/Hello-World/commits/c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 82 | }, 83 | "zipball_url": "https://github.com/octocat/Hello-World/zipball/v0.1", 84 | "tarball_url": "https://github.com/octocat/Hello-World/tarball/v0.1" 85 | } 86 | ]` 87 | 88 | const head = ` 89 | { 90 | "sha": "eaae6f7b3e4bb6b3337c1181557e1d44c48235fe", 91 | "node_id": "MDY6Q29tbWl0MTQxOTE5Mzk5OmVhYWU2ZjdiM2U0YmI2YjMzMzdjMTE4MTU1N2UxZDQ0YzQ4MjM1ZmU=", 92 | "commit": { 93 | "author": { 94 | "name": "Seth Hoenig", 95 | "email": "hoenig@indeed.com", 96 | "date": "2018-11-16T20:32:56Z" 97 | } 98 | } 99 | }` 100 | -------------------------------------------------------------------------------- /registry/internal/tools/finder/github_test.go: -------------------------------------------------------------------------------- 1 | package finder 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "gophers.dev/pkgs/semantic" 9 | ) 10 | 11 | func Test_githubCommit_Pseudo(t *testing.T) { 12 | var gc githubCommit 13 | gc.SHA = "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 14 | gc.Commit.Author.Date = "2011-01-26T19:06:43Z" 15 | 16 | pseudo, err := gc.Pseudo([]semantic.Tag{}, true) 17 | require.NoError(t, err) 18 | require.Equal( 19 | t, 20 | "v0.0.0-20110126190643-c5b97d5ae6c1", 21 | pseudo, 22 | ) 23 | } 24 | 25 | func Test_githubCommit_Pseudo_bad_time(t *testing.T) { 26 | var gc githubCommit 27 | gc.SHA = "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 28 | gc.Commit.Author.Date = "2011-01-26T19:06:43" // no Z 29 | 30 | _, err := gc.Pseudo([]semantic.Tag{}, true) 31 | require.Error(t, err) 32 | } 33 | 34 | func Test_githubCommit_Pseudo_previous_pre_version(t *testing.T) { 35 | var gc githubCommit 36 | gc.SHA = "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 37 | gc.Commit.Author.Date = "2011-01-26T19:06:43Z" 38 | 39 | pseudo, err := gc.Pseudo([]semantic.Tag{ 40 | {Major: 1, Minor: 2, Patch: 4, Extension: "pre"}, 41 | {Major: 1, Minor: 2, Patch: 3}, 42 | }, true) 43 | require.NoError(t, err) 44 | require.Equal(t, "v1.2.4-pre.0.20110126190643-c5b97d5ae6c1", pseudo) 45 | } 46 | 47 | func Test_githubCommit_Pseudo_previous_version(t *testing.T) { 48 | var gc githubCommit 49 | gc.SHA = "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 50 | gc.Commit.Author.Date = "2011-01-26T19:06:43Z" 51 | 52 | pseudo, err := gc.Pseudo([]semantic.Tag{ 53 | {Major: 1, Minor: 2, Patch: 4}, 54 | {Major: 1, Minor: 2, Patch: 3}, 55 | }, true) 56 | require.NoError(t, err) 57 | require.Equal(t, "v1.2.5-0.20110126190643-c5b97d5ae6c1", pseudo) 58 | } 59 | 60 | func Test_githubCommit_Pseudo_incompatible(t *testing.T) { 61 | var gc githubCommit 62 | gc.SHA = "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 63 | gc.Commit.Author.Date = "2011-01-26T19:06:43Z" 64 | 65 | pseudo, err := gc.Pseudo([]semantic.Tag{ 66 | {Major: 2, Minor: 2, Patch: 4}, 67 | {Major: 1, Minor: 2, Patch: 3}, 68 | }, false) 69 | require.NoError(t, err) 70 | require.Equal(t, "v2.2.5-0.20110126190643-c5b97d5ae6c1+incompatible", pseudo) 71 | } 72 | 73 | func Test_githubCommit_Pseudo_incompatible_semver1(t *testing.T) { 74 | var gc githubCommit 75 | gc.SHA = "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" 76 | gc.Commit.Author.Date = "2011-01-26T19:06:43Z" 77 | 78 | pseudo, err := gc.Pseudo([]semantic.Tag{ 79 | {Major: 1, Minor: 2, Patch: 4}, 80 | {Major: 1, Minor: 2, Patch: 3}, 81 | }, false) 82 | require.NoError(t, err) 83 | require.Equal(t, "v1.2.5-0.20110126190643-c5b97d5ae6c1", pseudo) 84 | } 85 | 86 | // v3.0.0-rc.2 v3.0.0-rc.1 v2.2.0 2.2.0 2.1.0 2.0.0 1.1.0 1.0.0 87 | -------------------------------------------------------------------------------- /registry/internal/proxies/prune_test.go: -------------------------------------------------------------------------------- 1 | package proxies 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 11 | "oss.indeed.com/go/modprox/pkg/netservice" 12 | "oss.indeed.com/go/modprox/registry/internal/data" 13 | ) 14 | 15 | func Test_Prune(t *testing.T) { 16 | store := data.NewStoreMock(t) 17 | defer store.MinimockFinish() 18 | 19 | now := time.Date(2018, 9, 27, 14, 48, 0, 0, time.UTC) 20 | oneMinuteAgo := now.Add(-1 * time.Minute) 21 | oneHourAgo := now.Add(-1 * time.Hour) 22 | 23 | store.ListHeartbeatsMock.Return( 24 | []payloads.Heartbeat{ 25 | { 26 | Self: netservice.Instance{ 27 | Address: "1.1.1.1", 28 | Port: 23456, 29 | }, 30 | Timestamp: int(oneMinuteAgo.Unix()), 31 | }, 32 | { 33 | Self: netservice.Instance{ 34 | Address: "2.2.2.2", 35 | Port: 34567, 36 | }, 37 | Timestamp: int(oneHourAgo.Unix()), 38 | }, 39 | }, nil, 40 | ) 41 | 42 | // should only purge 2.2.2.2 43 | store.PurgeProxyMock.When(netservice.Instance{ 44 | Address: "2.2.2.2", 45 | Port: 34567, 46 | }).Then(nil) 47 | 48 | p := NewPruner(3*time.Minute, store) 49 | 50 | err := p.Prune(now) 51 | require.NoError(t, err) 52 | } 53 | 54 | func Test_Prune_list_fail(t *testing.T) { 55 | store := data.NewStoreMock(t) 56 | defer store.MinimockFinish() 57 | 58 | now := time.Date(2018, 9, 27, 14, 48, 0, 0, time.UTC) 59 | 60 | store.ListHeartbeatsMock.Return([]payloads.Heartbeat{}, errors.New("db list fail")) 61 | 62 | p := NewPruner(3*time.Minute, store) 63 | 64 | err := p.Prune(now) 65 | require.Error(t, err) 66 | } 67 | 68 | func Test_Prune_purge_fail(t *testing.T) { 69 | store := data.NewStoreMock(t) 70 | defer store.MinimockFinish() 71 | 72 | now := time.Date(2018, 9, 27, 14, 48, 0, 0, time.UTC) 73 | oneMinuteAgo := now.Add(-1 * time.Minute) 74 | oneHourAgo := now.Add(-1 * time.Hour) 75 | 76 | store.ListHeartbeatsMock.Return( 77 | []payloads.Heartbeat{ 78 | { 79 | Self: netservice.Instance{ 80 | Address: "1.1.1.1", 81 | Port: 23456, 82 | }, 83 | Timestamp: int(oneMinuteAgo.Unix()), 84 | }, 85 | { 86 | Self: netservice.Instance{ 87 | Address: "2.2.2.2", 88 | Port: 34567, 89 | }, 90 | Timestamp: int(oneHourAgo.Unix()), 91 | }, 92 | }, nil, 93 | ) 94 | 95 | // should only purge 2.2.2.2 96 | store.PurgeProxyMock.When(netservice.Instance{ 97 | Address: "2.2.2.2", 98 | Port: 34567, 99 | }).Then(errors.New("db purge fail")) 100 | 101 | p := NewPruner(3*time.Minute, store) 102 | 103 | err := p.Prune(now) 104 | require.Error(t, err) 105 | } 106 | -------------------------------------------------------------------------------- /proxy/internal/status/startup/sender_test.go: -------------------------------------------------------------------------------- 1 | package startup 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 12 | "oss.indeed.com/go/modprox/pkg/clients/registry" 13 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 14 | "oss.indeed.com/go/modprox/pkg/netservice" 15 | "oss.indeed.com/go/modprox/pkg/webutil" 16 | "oss.indeed.com/go/modprox/proxy/config" 17 | ) 18 | 19 | func Test_Send_firstTry(t *testing.T) { 20 | ts := httptest.NewServer( 21 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 22 | _, _ = w.Write([]byte("some reply")) 23 | }), 24 | ) 25 | defer ts.Close() 26 | 27 | address, port := webutil.ParseURL(t, ts.URL) 28 | 29 | emitter := stats.Discard() 30 | 31 | client := registry.NewClient(registry.Options{ 32 | Timeout: 1 * time.Second, 33 | Instances: []netservice.Instance{{ 34 | Address: address, 35 | Port: port, 36 | }}, 37 | }) 38 | 39 | apiClient := NewSender(client, 1*time.Second, emitter) 40 | 41 | instance := netservice.Instance{} 42 | storage := config.Storage{} 43 | registries := config.Registry{} 44 | transforms := config.Transforms{} 45 | 46 | err := apiClient.Send(payloads.Configuration{ 47 | Self: instance, 48 | DiskStorage: storage, 49 | Registry: registries, 50 | Transforms: transforms, 51 | }) 52 | require.NoError(t, err) 53 | } 54 | 55 | func Test_Send_secondTry(t *testing.T) { 56 | firstTry := true 57 | executedSecondTry := false 58 | ts := httptest.NewServer( 59 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 | if firstTry { 61 | w.WriteHeader(http.StatusInternalServerError) 62 | firstTry = false 63 | } else { 64 | w.WriteHeader(http.StatusOK) 65 | _, _ = w.Write([]byte("some reply")) 66 | executedSecondTry = true 67 | } 68 | }), 69 | ) 70 | defer ts.Close() 71 | 72 | address, port := webutil.ParseURL(t, ts.URL) 73 | 74 | emitter := stats.Discard() 75 | 76 | client := registry.NewClient(registry.Options{ 77 | Timeout: 1 * time.Second, 78 | Instances: []netservice.Instance{{ 79 | Address: address, 80 | Port: port, 81 | }}, 82 | }) 83 | 84 | apiClient := NewSender(client, 10*time.Millisecond, emitter) 85 | 86 | instance := netservice.Instance{} 87 | storage := config.Storage{} 88 | registries := config.Registry{} 89 | transforms := config.Transforms{} 90 | 91 | err := apiClient.Send(payloads.Configuration{ 92 | Self: instance, 93 | DiskStorage: storage, 94 | Registry: registries, 95 | Transforms: transforms, 96 | }) 97 | require.NoError(t, err) 98 | require.True(t, executedSecondTry) 99 | } 100 | -------------------------------------------------------------------------------- /pkg/upstream/request_test.go: -------------------------------------------------------------------------------- 1 | package upstream 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func dummyRequest() *Request { 10 | return &Request{ 11 | Transport: "https", 12 | Domain: "code.example.com", 13 | Namespace: []string{"go", "foo"}, 14 | Version: "v0.0.1", 15 | Path: "/a/b/c.zip", 16 | GoGetRedirect: true, 17 | Headers: map[string]string{"X-Something": "abc123"}, 18 | } 19 | } 20 | 21 | func Test_Request_String(t *testing.T) { 22 | request := dummyRequest() 23 | 24 | s := request.String() 25 | require.Equal(t, 26 | `["https" "code.example.com" [go foo] "v0.0.1" "/a/b/c.zip" true]`, 27 | s, 28 | ) 29 | } 30 | 31 | func Test_Request_URI(t *testing.T) { 32 | request := dummyRequest() 33 | 34 | uri := request.URI() 35 | require.Equal(t, 36 | `https://code.example.com/a/b/c.zip`, 37 | uri, 38 | ) 39 | } 40 | 41 | func Test_Request_Equals_yes(t *testing.T) { 42 | r1 := dummyRequest() 43 | r2 := dummyRequest() 44 | 45 | require.True(t, r1.Equals(r2)) 46 | require.True(t, r2.Equals(r1)) 47 | } 48 | 49 | func Test_Request_Equals_no_transport(t *testing.T) { 50 | r1 := dummyRequest() 51 | r2 := dummyRequest() 52 | 53 | r2.Transport = "http" 54 | require.False(t, r1.Equals(r2)) 55 | require.False(t, r2.Equals(r1)) 56 | } 57 | 58 | func Test_Request_Equals_no_domain(t *testing.T) { 59 | r1 := dummyRequest() 60 | r2 := dummyRequest() 61 | 62 | r2.Domain = "src.example.com" 63 | require.False(t, r1.Equals(r2)) 64 | require.False(t, r2.Equals(r1)) 65 | } 66 | 67 | func Test_Request_Equals_no_namespace(t *testing.T) { 68 | r1 := dummyRequest() 69 | r2 := dummyRequest() 70 | 71 | r2.Namespace = []string{"x", "y"} 72 | require.False(t, r1.Equals(r2)) 73 | require.False(t, r2.Equals(r1)) 74 | } 75 | 76 | func Test_Request_Equals_no_version(t *testing.T) { 77 | r1 := dummyRequest() 78 | r2 := dummyRequest() 79 | 80 | r2.Version = "v2.2.2" 81 | require.False(t, r1.Equals(r2)) 82 | require.False(t, r2.Equals(r1)) 83 | } 84 | 85 | func Test_Request_Equals_no_path(t *testing.T) { 86 | r1 := dummyRequest() 87 | r2 := dummyRequest() 88 | 89 | r2.Path = "/x/y.zip" 90 | require.False(t, r1.Equals(r2)) 91 | require.False(t, r2.Equals(r1)) 92 | } 93 | 94 | func Test_Request_Equals_no_goget(t *testing.T) { 95 | r1 := dummyRequest() 96 | r2 := dummyRequest() 97 | 98 | r2.GoGetRedirect = false 99 | require.False(t, r1.Equals(r2)) 100 | require.False(t, r2.Equals(r1)) 101 | } 102 | 103 | func Test_Request_Equals_no_headers(t *testing.T) { 104 | r1 := dummyRequest() 105 | r2 := dummyRequest() 106 | 107 | r2.Headers = map[string]string{"foo": "bar"} 108 | require.False(t, r1.Equals(r2)) 109 | require.False(t, r2.Equals(r1)) 110 | } 111 | -------------------------------------------------------------------------------- /registry/internal/tools/finder/finder.go: -------------------------------------------------------------------------------- 1 | package finder 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | "time" 7 | 8 | "github.com/pkg/errors" 9 | 10 | "gophers.dev/pkgs/loggy" 11 | "gophers.dev/pkgs/semantic" 12 | 13 | "oss.indeed.com/go/modprox/pkg/clients/zips" 14 | ) 15 | 16 | type Result struct { 17 | Text string 18 | Latest Head 19 | Tags []semantic.Tag 20 | } 21 | 22 | type Head struct { 23 | // Pseudo represents Go's custom version string for SHAs which are 24 | // not represented by a SemVer string. 25 | // e.g. 26 | Custom string 27 | Commit string 28 | } 29 | 30 | type Tag struct { 31 | SemVer string 32 | Commit string 33 | } 34 | 35 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Versions -s _mock.go 36 | 37 | type Versions interface { 38 | // Request the list of semver tags set in the source git repository. 39 | Request(source string) (*Result, error) 40 | } 41 | 42 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Finder -s _mock.go 43 | 44 | type Finder interface { 45 | // Find returns the special form module name for the latest commit, 46 | // as well as a list of tags that follow proper semver format understood 47 | // by the Go compiler. 48 | Find(string) (*Result, error) 49 | } 50 | 51 | type Options struct { 52 | Timeout time.Duration 53 | Versions map[string]Versions 54 | ProxyClient zips.ProxyClient 55 | } 56 | 57 | func New(opts Options) Finder { 58 | timeout := opts.Timeout 59 | if timeout <= 0 { 60 | timeout = 1 * time.Minute 61 | } 62 | 63 | client := &http.Client{ 64 | Timeout: timeout, 65 | } 66 | 67 | versions := opts.Versions 68 | if versions == nil { 69 | versions = map[string]Versions{ 70 | "github.com": Github("", client, opts.ProxyClient), 71 | } 72 | } 73 | 74 | return &finder{ 75 | versions: versions, 76 | log: loggy.New("finder"), 77 | } 78 | } 79 | 80 | type finder struct { 81 | versions map[string]Versions 82 | log loggy.Logger 83 | } 84 | 85 | func (f *finder) Find(source string) (*Result, error) { 86 | resolver, err := f.forSource(source) 87 | if err != nil { 88 | return nil, err 89 | } 90 | return resolver.Request(source) 91 | } 92 | 93 | func parseDomain(source string) string { 94 | split := strings.Split(source, "/") 95 | return split[0] 96 | } 97 | 98 | func (f *finder) forSource(source string) (Versions, error) { 99 | domain := parseDomain(source) 100 | versions, exists := f.versions[domain] 101 | if !exists { 102 | return nil, errors.Errorf("no version resolver for domain %q", domain) 103 | } 104 | return versions, nil 105 | } 106 | 107 | func Compatible(source string) bool { 108 | // as more things are added, add them here 109 | return githubPkgRe.MatchString(source) 110 | } 111 | -------------------------------------------------------------------------------- /pkg/clients/zips/http.go: -------------------------------------------------------------------------------- 1 | package zips 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/pkg/errors" 9 | 10 | "gophers.dev/pkgs/ignore" 11 | "gophers.dev/pkgs/loggy" 12 | 13 | "oss.indeed.com/go/modprox/pkg/repository" 14 | "oss.indeed.com/go/modprox/pkg/upstream" 15 | ) 16 | 17 | var maxLoggedBody = 500 18 | 19 | type httpClient struct { 20 | client *http.Client 21 | options HTTPOptions 22 | log loggy.Logger 23 | } 24 | 25 | type HTTPOptions struct { 26 | Timeout time.Duration 27 | } 28 | 29 | func NewHTTPClient(options HTTPOptions) UpstreamClient { 30 | if options.Timeout <= 0 { 31 | options.Timeout = 10 * time.Minute 32 | } 33 | return &httpClient{ 34 | options: options, 35 | client: &http.Client{ 36 | Timeout: options.Timeout, 37 | }, 38 | log: loggy.New("zips-http"), 39 | } 40 | } 41 | 42 | func (c *httpClient) Protocols() []string { 43 | return []string{"http", "https"} 44 | 45 | } 46 | 47 | func (c *httpClient) Get(r *upstream.Request) (repository.Blob, error) { 48 | if r == nil { 49 | return nil, errors.New("request is nil") 50 | } 51 | 52 | zipURI := r.URI() 53 | c.log.Tracef("making zip upstream request to %s", zipURI) 54 | 55 | request, err := c.newRequest(r) 56 | if err != nil { 57 | return nil, errors.Wrapf(err, "could not create request from %s", zipURI) 58 | } 59 | 60 | response, err := c.client.Do(request) 61 | if err != nil { 62 | return nil, errors.Wrapf(err, "could not do request for %s", zipURI) 63 | } 64 | defer ignore.Drain(response.Body) 65 | 66 | // if we get a bad response code, try to read the body and log it 67 | if response.StatusCode >= 400 { 68 | bs, err := ioutil.ReadAll(response.Body) 69 | if err != nil { 70 | return nil, errors.Wrapf(err, "could not read body of bad response (%d)", response.StatusCode) 71 | } 72 | body := string(bs) 73 | if len(body) <= maxLoggedBody { 74 | c.log.Errorf( 75 | "bad response (%d), body: %s", 76 | response.StatusCode, 77 | body, 78 | ) 79 | } else { 80 | c.log.Errorf( 81 | "bad response (%d), body: %s...", 82 | response.StatusCode, 83 | body[:maxLoggedBody], 84 | ) 85 | } 86 | return nil, errors.Wrapf( 87 | err, 88 | "unexpected response (%d)", 89 | response.StatusCode, 90 | ) 91 | } 92 | 93 | // response is good, read the bytes 94 | return ioutil.ReadAll(response.Body) 95 | } 96 | 97 | func (c *httpClient) newRequest(r *upstream.Request) (*http.Request, error) { 98 | uri := r.URI() 99 | request, err := http.NewRequest( 100 | http.MethodGet, 101 | uri, 102 | nil, 103 | ) 104 | if err != nil { 105 | return nil, errors.Wrap(err, "unable to create request") 106 | } 107 | 108 | for k, v := range r.Headers { 109 | request.Header.Set(k, v) 110 | } 111 | 112 | return request, nil 113 | } 114 | -------------------------------------------------------------------------------- /registry/internal/web/router.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gorilla/mux" 7 | 8 | petrify "gophers.dev/cmds/petrify/v5" 9 | 10 | "oss.indeed.com/go/modprox/pkg/clients/zips" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/webutil" 13 | "oss.indeed.com/go/modprox/registry/internal/data" 14 | "oss.indeed.com/go/modprox/registry/static" 15 | ) 16 | 17 | const ( 18 | get = http.MethodGet 19 | post = http.MethodPost 20 | ) 21 | 22 | func NewRouter( 23 | middleAPI []webutil.Middleware, 24 | middleUI []webutil.Middleware, 25 | store data.Store, 26 | emitter stats.Sender, 27 | history string, 28 | proxyClient zips.ProxyClient, 29 | ) http.Handler { 30 | 31 | // 1) a router onto which sub-routers will be mounted 32 | router := http.NewServeMux() 33 | 34 | // 2) a static files handler for statics 35 | router.Handle("/static/", routeStatics(http.FileServer(&petrify.AssetFS{ 36 | Asset: static.Asset, 37 | AssetDir: static.AssetDir, 38 | AssetInfo: static.AssetInfo, 39 | Prefix: "static", 40 | }))) 41 | 42 | // 3) an API handler, not CSRF protected 43 | router.Handle("/v1/", routeAPI(middleAPI, store, emitter)) 44 | 45 | // 4) a webUI handler, is CSRF protected 46 | router.Handle("/", routeWebUI(middleUI, store, emitter, history, proxyClient)) 47 | 48 | return router 49 | } 50 | 51 | func routeStatics(files http.Handler) http.Handler { 52 | sub := mux.NewRouter() 53 | sub.Handle("/static/css/{*}", http.StripPrefix("/static/", files)).Methods(get) 54 | sub.Handle("/static/img/{*}", http.StripPrefix("/static/", files)).Methods(get) 55 | return sub 56 | } 57 | 58 | func routeAPI(middles []webutil.Middleware, store data.Store, emitter stats.Sender) http.Handler { 59 | sub := mux.NewRouter() 60 | sub.Handle("/v1/registry/sources/list", newRegistryList(store, emitter)).Methods(get, post) 61 | sub.Handle("/v1/registry/sources/new", registryAdd(store, emitter)).Methods(post) 62 | sub.Handle("/v1/proxy/heartbeat", newHeartbeatHandler(store, emitter)).Methods(post) 63 | sub.Handle("/v1/proxy/configuration", newStartupHandler(store, emitter)).Methods(post) 64 | return webutil.Chain(sub, middles...) 65 | } 66 | 67 | func routeWebUI(middles []webutil.Middleware, store data.Store, emitter stats.Sender, history string, proxyClient zips.ProxyClient) http.Handler { 68 | sub := mux.NewRouter() 69 | sub.Handle("/mods/new", newAddHandler(store, emitter)).Methods(get, post) 70 | sub.Handle("/mods/list", newModsListHandler(store, emitter)).Methods(get) 71 | sub.Handle("/mods/show", newShowHandler(store, emitter)).Methods(get, post) 72 | sub.Handle("/mods/find", newFindHandler(emitter, proxyClient)).Methods(get, post) 73 | sub.Handle("/configure/about", newAboutHandler(emitter)).Methods(get) 74 | sub.Handle("/configure/blocks", newBlocksHandler(emitter)).Methods(get) 75 | sub.Handle("/history", newHistoryHandler(emitter, history)).Methods(get) 76 | sub.Handle("/", newHomeHandler(store, emitter)).Methods(get, post) 77 | return webutil.Chain(sub, middles...) 78 | } 79 | -------------------------------------------------------------------------------- /registry/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/pkg/errors" 9 | 10 | "oss.indeed.com/go/modprox/pkg/configutil" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/setup" 13 | ) 14 | 15 | type Configuration struct { 16 | WebServer WebServer `json:"web_server"` 17 | CSRF CSRF `json:"csrf"` 18 | Database setup.PersistentStore `json:"database_storage"` 19 | Statsd stats.Statsd `json:"statsd"` 20 | Proxies Proxies `json:"proxies"` 21 | ProxyClient ProxyClient `json:"proxy_client"` 22 | } 23 | 24 | type ProxyClient struct { 25 | Protocol string `json:"protocol"` // e.g. "https" 26 | BaseURL string `json:"base_url"` // e.g. "proxy.golang.org" 27 | } 28 | 29 | func (c Configuration) String() string { 30 | return configutil.Format(c) 31 | } 32 | 33 | type WebServer struct { 34 | TLS struct { 35 | Enabled bool `json:"enabled"` 36 | Certificate string `json:"certificate"` 37 | Key string `json:"key"` 38 | } `json:"tls"` 39 | BindAddress string `json:"bind_address"` 40 | Port int `json:"port"` 41 | ReadTimeoutS int `json:"read_timeout_s"` 42 | WriteTimeoutS int `json:"write_timeout_s"` 43 | APIKeys []string `json:"api_keys"` 44 | } 45 | 46 | func (s WebServer) Server(mux http.Handler) (*http.Server, error) { 47 | if s.BindAddress == "" { 48 | return nil, errors.New("server bind address is not set") 49 | } 50 | 51 | if s.Port == 0 { 52 | return nil, errors.New("server port is not set") 53 | } 54 | 55 | if s.TLS.Enabled { 56 | if s.TLS.Certificate == "" { 57 | return nil, errors.New("TLS enabled, but server TLS certificate not set") 58 | } 59 | 60 | if s.TLS.Key == "" { 61 | return nil, errors.New("TLS enabled, but server TLS key not set") 62 | } 63 | } 64 | 65 | if s.ReadTimeoutS == 0 { 66 | s.ReadTimeoutS = 60 67 | } 68 | 69 | if s.WriteTimeoutS == 0 { 70 | s.WriteTimeoutS = 60 71 | } 72 | 73 | address := fmt.Sprintf("%s:%d", s.BindAddress, s.Port) 74 | server := &http.Server{ 75 | Addr: address, 76 | Handler: mux, 77 | ReadTimeout: seconds(s.ReadTimeoutS), 78 | WriteTimeout: seconds(s.WriteTimeoutS), 79 | } 80 | 81 | return server, nil 82 | } 83 | 84 | func seconds(s int) time.Duration { 85 | return time.Duration(s) * time.Second 86 | } 87 | 88 | type CSRF struct { 89 | DevelopmentMode bool `json:"development_mode"` 90 | AuthenticationKey string `json:"authentication_key"` 91 | } 92 | 93 | // Key returns the configured 32 byte CSRF key, and a bool indicating 94 | // whether development mode is enabled. If the CSRF is not well formed, 95 | // an error is returned. 96 | func (c CSRF) Key() ([]byte, bool, error) { 97 | key := c.AuthenticationKey 98 | if len(key) != 32 { 99 | return nil, false, errors.Errorf( 100 | "csrf.authentication_key must be 32 bytes long, got %d", 101 | len(key), 102 | ) 103 | } 104 | return []byte(key), c.DevelopmentMode, nil 105 | } 106 | 107 | type Proxies struct { 108 | PruneAfter int `json:"prune_after_s"` 109 | } 110 | -------------------------------------------------------------------------------- /registry/internal/web/mods_show.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "errors" 5 | "html/template" 6 | "net/http" 7 | "sort" 8 | "strconv" 9 | 10 | "github.com/gorilla/csrf" 11 | 12 | "gophers.dev/pkgs/loggy" 13 | 14 | "oss.indeed.com/go/modprox/pkg/coordinates" 15 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 16 | "oss.indeed.com/go/modprox/registry/internal/data" 17 | "oss.indeed.com/go/modprox/registry/static" 18 | ) 19 | 20 | type showPage struct { 21 | CSRF template.HTML 22 | Source string 23 | Mods []coordinates.SerialModule 24 | } 25 | 26 | type showHandler struct { 27 | html *template.Template 28 | store data.Store 29 | emitter stats.Sender 30 | log loggy.Logger 31 | } 32 | 33 | func newShowHandler(store data.Store, emitter stats.Sender) http.Handler { 34 | html := static.MustParseTemplates( 35 | "static/html/layout.html", 36 | "static/html/navbar.html", 37 | "static/html/mods_show.html", 38 | ) 39 | 40 | return &showHandler{ 41 | html: html, 42 | store: store, 43 | emitter: emitter, 44 | log: loggy.New("show-module-h"), 45 | } 46 | } 47 | 48 | func (h *showHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 49 | var ( 50 | code int 51 | page *showPage 52 | err error 53 | ) 54 | 55 | switch r.Method { 56 | case http.MethodGet: 57 | code, page, err = h.get(r) 58 | case http.MethodPost: 59 | code, page, err = h.post(r) 60 | } 61 | 62 | if err != nil { 63 | h.log.Errorf("failed to serve show modules page: %v", err) 64 | http.Error(w, err.Error(), code) 65 | h.emitter.Count("ui-show-mod-error", 1) 66 | return 67 | } 68 | 69 | if err := h.html.Execute(w, page); err != nil { 70 | h.log.Errorf("failed to execute show modules page: %v", err) 71 | } 72 | 73 | h.emitter.Count("ui-show-mod-ok", 1) 74 | } 75 | 76 | func (h *showHandler) get(r *http.Request) (int, *showPage, error) { 77 | return h.load(r) 78 | } 79 | 80 | func (h *showHandler) post(r *http.Request) (int, *showPage, error) { 81 | id, err := h.parseModToDelete(r) 82 | if err != nil { 83 | return http.StatusBadRequest, nil, err 84 | } 85 | 86 | h.log.Infof("will delete module of id: %d", id) 87 | if err := h.store.DeleteModuleByID(id); err != nil { 88 | return http.StatusInternalServerError, nil, err 89 | } 90 | 91 | // after deletion just load the show page for that package again 92 | return h.load(r) 93 | } 94 | 95 | // both get and post will load the mod show page 96 | // which can be rendered with this load function 97 | func (h *showHandler) load(r *http.Request) (int, *showPage, error) { 98 | source, err := h.parseQuery(r) 99 | if err != nil { 100 | return http.StatusBadRequest, nil, err 101 | } 102 | 103 | mods, err := h.store.ListModulesBySource(source) 104 | if err != nil { 105 | return http.StatusInternalServerError, nil, err 106 | } 107 | 108 | sort.Sort(coordinates.ModsByVersion(mods)) 109 | 110 | return http.StatusOK, &showPage{ 111 | Source: source, 112 | Mods: mods, 113 | CSRF: csrf.TemplateField(r), 114 | }, nil 115 | } 116 | 117 | func (h *showHandler) parseQuery(r *http.Request) (string, error) { 118 | values := r.URL.Query() 119 | m := values.Get("mod") 120 | if m == "" { 121 | return "", errors.New("mod query parameter required") 122 | } 123 | return m, nil 124 | } 125 | 126 | func (h *showHandler) parseModToDelete(r *http.Request) (int, error) { 127 | idText := r.FormValue("delete-mod-id") 128 | return strconv.Atoi(idText) 129 | } 130 | -------------------------------------------------------------------------------- /pkg/webutil/middleware_test.go: -------------------------------------------------------------------------------- 1 | package webutil 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type testHandler struct{} 13 | 14 | func (h testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 15 | w.WriteHeader(http.StatusOK) 16 | return 17 | } 18 | 19 | func Test_Chain(t *testing.T) { 20 | var h testHandler 21 | var called bool 22 | testMiddleware := func(h http.Handler) http.Handler { 23 | called = true 24 | return h 25 | } 26 | _ = Chain(h, testMiddleware) 27 | assert.True(t, called) 28 | } 29 | 30 | func Test_KeyGuard_no_header(t *testing.T) { 31 | guard := KeyGuard([]string{"abc123"}) 32 | 33 | executed := false 34 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 35 | w.Write([]byte("secret stuff")) 36 | executed = true 37 | }) 38 | 39 | protected := Chain(handler, guard) 40 | 41 | request, err := http.NewRequest(http.MethodGet, "/foo", nil) 42 | require.NoError(t, err) 43 | 44 | recorder := httptest.NewRecorder() 45 | protected.ServeHTTP(recorder, request) 46 | 47 | code := recorder.Code 48 | require.Equal(t, http.StatusForbidden, code) 49 | require.False(t, executed) 50 | } 51 | 52 | func Test_KeyGuard_bad_keys(t *testing.T) { 53 | guard := KeyGuard([]string{"abc123"}) 54 | 55 | executed := false 56 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 57 | w.Write([]byte("secret stuff")) 58 | executed = true 59 | }) 60 | 61 | protected := Chain(handler, guard) 62 | 63 | request, err := http.NewRequest(http.MethodGet, "/foo", nil) 64 | require.NoError(t, err) 65 | request.Header.Add(HeaderAPIKey, "foo123") 66 | request.Header.Add(HeaderAPIKey, "bar123") 67 | request.Header.Add(HeaderAPIKey, "baz123") 68 | 69 | recorder := httptest.NewRecorder() 70 | protected.ServeHTTP(recorder, request) 71 | 72 | code := recorder.Code 73 | require.Equal(t, http.StatusForbidden, code) 74 | require.False(t, executed) 75 | } 76 | 77 | func Test_KeyGuard_no_keys(t *testing.T) { 78 | guard := KeyGuard([]string{""}) 79 | 80 | executed := false 81 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 82 | w.Write([]byte("secret stuff")) 83 | executed = true 84 | }) 85 | 86 | protected := Chain(handler, guard) 87 | 88 | request, err := http.NewRequest(http.MethodGet, "/foo", nil) 89 | require.NoError(t, err) 90 | request.Header.Add(HeaderAPIKey, "foo123") 91 | request.Header.Add(HeaderAPIKey, "bar123") 92 | request.Header.Add(HeaderAPIKey, "baz123") 93 | 94 | recorder := httptest.NewRecorder() 95 | protected.ServeHTTP(recorder, request) 96 | 97 | code := recorder.Code 98 | require.Equal(t, http.StatusForbidden, code) 99 | require.False(t, executed) 100 | } 101 | 102 | func Test_KeyGuard_good_key(t *testing.T) { 103 | guard := KeyGuard([]string{"abc123"}) 104 | 105 | executed := false 106 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 107 | w.Write([]byte("secret stuff")) 108 | executed = true 109 | }) 110 | 111 | protected := Chain(handler, guard) 112 | 113 | request, err := http.NewRequest(http.MethodGet, "/foo", nil) 114 | require.NoError(t, err) 115 | request.Header.Set(HeaderAPIKey, "abc123") 116 | 117 | recorder := httptest.NewRecorder() 118 | protected.ServeHTTP(recorder, request) 119 | 120 | code := recorder.Code 121 | require.Equal(t, http.StatusOK, code) 122 | require.True(t, executed) 123 | } 124 | -------------------------------------------------------------------------------- /registry/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func Test_Configuration_CSRF_nokey(t *testing.T) { 11 | var c Configuration 12 | _, _, err := c.CSRF.Key() 13 | require.Error(t, err) 14 | } 15 | 16 | func Test_Configuration_CSRF_badkey(t *testing.T) { 17 | var c Configuration 18 | c.CSRF.AuthenticationKey = "foobar" 19 | _, _, err := c.CSRF.Key() 20 | require.Error(t, err) 21 | } 22 | 23 | func Test_Configuration_CSRF_devmode(t *testing.T) { 24 | key := "12345678901234567890123456789012" 25 | var c Configuration 26 | c.CSRF.DevelopmentMode = true 27 | c.CSRF.AuthenticationKey = key 28 | bs, devMode, err := c.CSRF.Key() 29 | require.NoError(t, err) 30 | require.True(t, devMode) 31 | require.Equal(t, []byte(key), bs) 32 | } 33 | 34 | func Test_Configuration_CSRF(t *testing.T) { 35 | key := "12345678901234567890123456789012" 36 | var c Configuration 37 | c.CSRF.AuthenticationKey = key 38 | bs, devMode, err := c.CSRF.Key() 39 | require.NoError(t, err) 40 | require.False(t, devMode) 41 | require.Equal(t, []byte(key), bs) 42 | } 43 | 44 | func Test_Configuration_Server_notls_defaults_ok(t *testing.T) { 45 | c := Configuration{ 46 | WebServer: WebServer{ 47 | BindAddress: "1.2.3.4", 48 | Port: 9999, 49 | }, 50 | } 51 | 52 | server, err := c.WebServer.Server(nil) 53 | require.NoError(t, err) 54 | require.Equal(t, "1.2.3.4:9999", server.Addr) 55 | require.Equal(t, 60*time.Second, server.ReadTimeout) 56 | require.Equal(t, 60*time.Second, server.WriteTimeout) 57 | } 58 | 59 | func Test_Configuration_Server_notls_ok(t *testing.T) { 60 | c := Configuration{ 61 | WebServer: WebServer{ 62 | BindAddress: "1.2.3.4", 63 | Port: 9999, 64 | ReadTimeoutS: 1, 65 | WriteTimeoutS: 2, 66 | }, 67 | } 68 | 69 | server, err := c.WebServer.Server(nil) 70 | require.NoError(t, err) 71 | require.Equal(t, "1.2.3.4:9999", server.Addr) 72 | require.Equal(t, 1*time.Second, server.ReadTimeout) 73 | require.Equal(t, 2*time.Second, server.WriteTimeout) 74 | } 75 | 76 | func Test_Configuration_Server_tls_noCertificate(t *testing.T) { 77 | c := Configuration{ 78 | WebServer: WebServer{ 79 | BindAddress: "1.2.3.4", 80 | Port: 9999, 81 | ReadTimeoutS: 1, 82 | WriteTimeoutS: 2, 83 | }, 84 | } 85 | c.WebServer.TLS.Enabled = true 86 | c.WebServer.TLS.Key = "key" 87 | 88 | _, err := c.WebServer.Server(nil) 89 | require.Error(t, err) 90 | } 91 | 92 | func Test_Configuration_Server_tls_noKey(t *testing.T) { 93 | c := Configuration{ 94 | WebServer: WebServer{ 95 | BindAddress: "1.2.3.4", 96 | Port: 9999, 97 | ReadTimeoutS: 1, 98 | WriteTimeoutS: 2, 99 | }, 100 | } 101 | c.WebServer.TLS.Enabled = true 102 | c.WebServer.TLS.Certificate = "cert" 103 | 104 | _, err := c.WebServer.Server(nil) 105 | require.Error(t, err) 106 | } 107 | 108 | func Test_Configuration_Server_tls_no_files(t *testing.T) { 109 | c := Configuration{ 110 | WebServer: WebServer{ 111 | BindAddress: "1.2.3.4", 112 | Port: 9999, 113 | ReadTimeoutS: 1, 114 | WriteTimeoutS: 2, 115 | }, 116 | } 117 | c.WebServer.TLS.Enabled = true 118 | c.WebServer.TLS.Certificate = "cert" 119 | c.WebServer.TLS.Key = "key" 120 | 121 | server, err := c.WebServer.Server(nil) 122 | require.NoError(t, err) 123 | require.Equal(t, "1.2.3.4:9999", server.Addr) 124 | require.Equal(t, 1*time.Second, server.ReadTimeout) 125 | require.Equal(t, 2*time.Second, server.WriteTimeout) 126 | } 127 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [opensource@indeed.com](mailto:opensource@indeed.com). All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /registry/internal/service/init.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "time" 7 | 8 | "github.com/gorilla/csrf" 9 | 10 | "github.com/pkg/errors" 11 | 12 | "gophers.dev/pkgs/repeat/x" 13 | 14 | "oss.indeed.com/go/modprox/pkg/clients/zips" 15 | "oss.indeed.com/go/modprox/pkg/history" 16 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 17 | "oss.indeed.com/go/modprox/pkg/webutil" 18 | "oss.indeed.com/go/modprox/registry/internal/data" 19 | "oss.indeed.com/go/modprox/registry/internal/proxies" 20 | "oss.indeed.com/go/modprox/registry/internal/web" 21 | ) 22 | 23 | type initer func(*Registry) error 24 | 25 | func initSender(r *Registry) error { 26 | cfg := r.config.Statsd.Agent 27 | if cfg.Port == 0 || cfg.Address == "" { 28 | r.emitter = stats.Discard() 29 | r.log.Warnf("stats emitter is set to discard client - no metrics will be reported") 30 | return nil 31 | } 32 | 33 | emitter, err := stats.New(stats.Registry, r.config.Statsd) 34 | if err != nil { 35 | return err 36 | } 37 | r.emitter = emitter 38 | return nil 39 | } 40 | 41 | func initProxyClient(r *Registry) error { 42 | r.proxyClient = zips.NewProxyClient( 43 | zips.ProxyClientOptions{ 44 | Protocol: r.config.ProxyClient.Protocol, 45 | BaseURL: r.config.ProxyClient.BaseURL, 46 | Timeout: 1 * time.Minute, 47 | }, 48 | ) 49 | 50 | return nil 51 | } 52 | 53 | func initStore(r *Registry) error { 54 | kind, dsn, err := r.config.Database.DSN() 55 | if err != nil { 56 | return errors.Wrap(err, "failed to configure database") 57 | } 58 | r.log.Infof("using database of kind: %q", kind) 59 | r.log.Infof("database dsn: %s", dsn) 60 | store, err := data.Connect(kind, dsn, r.emitter) 61 | r.store = store 62 | return err 63 | } 64 | 65 | func initProxyPrune(r *Registry) error { 66 | maxAge := time.Duration(r.config.Proxies.PruneAfter) * time.Second 67 | pruner := proxies.NewPruner(maxAge, r.store) 68 | go func() { 69 | _ = x.Interval(1*time.Minute, func() error { 70 | _ = pruner.Prune(time.Now()) 71 | return nil 72 | }) 73 | }() 74 | return nil 75 | } 76 | 77 | func initWebServer(r *Registry) error { 78 | var middleAPI []webutil.Middleware 79 | if len(r.config.WebServer.APIKeys) > 0 { 80 | middleAPI = append( 81 | middleAPI, 82 | webutil.KeyGuard(r.config.WebServer.APIKeys), 83 | ) 84 | } 85 | 86 | middleUI := []webutil.Middleware{ 87 | csrf.Protect( 88 | // the key is used to generate CSRF tokens to hand 89 | // out on html form loads 90 | []byte(r.config.CSRF.AuthenticationKey), 91 | 92 | // CSRF cookies are https-only normally, so for development 93 | //// mode make sure the CSRF package knows we are using http 94 | csrf.Secure(!r.config.CSRF.DevelopmentMode), 95 | ), 96 | } 97 | 98 | mux := web.NewRouter( 99 | middleAPI, 100 | middleUI, 101 | r.store, 102 | r.emitter, 103 | r.history, 104 | r.proxyClient, 105 | ) 106 | 107 | server, err := r.config.WebServer.Server(mux) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | go func(h http.Handler) { 113 | var err error 114 | if r.config.WebServer.TLS.Enabled { 115 | err = server.ListenAndServeTLS( 116 | r.config.WebServer.TLS.Certificate, 117 | r.config.WebServer.TLS.Key, 118 | ) 119 | } else { 120 | err = server.ListenAndServe() 121 | } 122 | 123 | // should never get to this point 124 | r.log.Errorf("server stopped serving: %v", err) 125 | os.Exit(1) 126 | }(mux) 127 | 128 | return nil 129 | } 130 | 131 | func initHistory(r *Registry) error { 132 | historyBytes, err := history.Asset("history.txt") 133 | if err != nil { 134 | return err 135 | } 136 | 137 | r.history = string(historyBytes) 138 | return nil 139 | } 140 | -------------------------------------------------------------------------------- /pkg/upstream/go-get.go: -------------------------------------------------------------------------------- 1 | package upstream 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "regexp" 9 | "strings" 10 | 11 | "github.com/pkg/errors" 12 | 13 | "gophers.dev/pkgs/ignore" 14 | "gophers.dev/pkgs/loggy" 15 | ) 16 | 17 | var maxLoggedBody = 500 18 | 19 | type goGetMeta struct { 20 | transport string 21 | domain string 22 | path string 23 | } 24 | 25 | func (t *GoGetTransform) doGoGetRequest(r *Request) (goGetMeta, error) { 26 | var meta goGetMeta 27 | uri := fmt.Sprintf("%s://%s/%s?go-get=1", r.Transport, r.Domain, strings.Join(r.Namespace, "/")) 28 | request, err := http.NewRequest(http.MethodGet, uri, nil) 29 | if err != nil { 30 | return meta, err 31 | } 32 | 33 | response, err := t.httpClient.Do(request) 34 | if err != nil { 35 | return meta, err 36 | } 37 | defer ignore.Drain(response.Body) 38 | 39 | bs, err := ioutil.ReadAll(response.Body) 40 | if err != nil { 41 | return meta, err 42 | } 43 | 44 | code := response.StatusCode 45 | body := string(bs) 46 | 47 | if code >= 400 { 48 | t.log.Errorf("failed to do go-get redirect, received code %d from %s", code, uri) 49 | if len(body) <= maxLoggedBody { 50 | t.log.Errorf("response body: %s", body) 51 | } else { 52 | t.log.Errorf("response body: %s...", body[:maxLoggedBody]) 53 | } 54 | return meta, errors.Errorf("bad response code (%d) from %s", code, uri) 55 | } 56 | 57 | return parseGoGetMetadata(body) 58 | } 59 | 60 | var ( 61 | sourceRe = regexp.MustCompile(`(http[s]?)://([\w-.]+)/([\w-./]+)`) 62 | log = loggy.New("go-get") 63 | ) 64 | 65 | // gives us transport, domain, path 66 | func parseGoGetMetadata(content string) (goGetMeta, error) { 67 | if ggm, exists, err := tryParseGoMetaTag("go-source", content); err != nil { 68 | return ggm, err 69 | } else if exists { 70 | log.Infof("found go-source tag: %#v", ggm) 71 | return ggm, nil 72 | } 73 | 74 | if ggm, exists, err := tryParseGoMetaTag("go-import", content); err != nil { 75 | return ggm, err 76 | } else if exists { 77 | log.Infof("found go-import tag %#v", ggm) 78 | return ggm, nil 79 | } 80 | 81 | return goGetMeta{}, errors.New("neither go-source or go-import meta tag found") 82 | } 83 | 84 | // ghetto hack where we look for go-source first, which is usually 85 | // the true github.com source 86 | // 87 | // only when this does not work do we use the go-import line, which 88 | // may redirect to a vcs protocol. 89 | func tryParseGoMetaTag(tag, content string) (goGetMeta, bool, error) { 90 | content = formatContent(content) // pre-process html 91 | 92 | var meta goGetMeta 93 | scanner := bufio.NewScanner(strings.NewReader(content)) 94 | for scanner.Scan() { 95 | line := strings.TrimSpace(scanner.Text()) 96 | metaTag := fmt.Sprintf("name=%q", tag) 97 | if strings.Contains(line, metaTag) { 98 | groups := sourceRe.FindStringSubmatch(line) 99 | if len(groups) != 4 { 100 | return meta, false, errors.Errorf("malformed meta tag: %q", line) 101 | } 102 | return goGetMeta{ 103 | transport: groups[1], 104 | domain: groups[2], 105 | path: cleanupPath(groups[3]), 106 | }, true, nil 107 | } 108 | } 109 | if err := scanner.Err(); err != nil { 110 | return meta, false, err 111 | } 112 | return meta, false, nil 113 | } 114 | 115 | func cleanupPath(p string) string { 116 | a := strings.TrimSuffix(p, "/") 117 | b := strings.TrimSuffix(a, ".git") 118 | return b 119 | } 120 | 121 | // need to rewrite newlines not preceded by closing angle bracket to be spaces 122 | func formatContent(content string) string { 123 | content = strings.Replace(content, "\n", " ", -1) 124 | content = strings.Replace(content, ">", ">\n", -1) 125 | return content 126 | } 127 | -------------------------------------------------------------------------------- /registry/internal/web/mods_find.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "errors" 5 | "html/template" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/gorilla/csrf" 10 | 11 | "gophers.dev/pkgs/loggy" 12 | 13 | "oss.indeed.com/go/modprox/pkg/clients/zips" 14 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 15 | "oss.indeed.com/go/modprox/registry/internal/tools/finder" 16 | "oss.indeed.com/go/modprox/registry/static" 17 | ) 18 | 19 | type findPage struct { 20 | CSRF template.HTML 21 | Found []findResult 22 | Query string 23 | } 24 | 25 | type findHandler struct { 26 | html *template.Template 27 | emitter stats.Sender 28 | finder finder.Finder 29 | log loggy.Logger 30 | } 31 | 32 | func newFindHandler(emitter stats.Sender, proxyClient zips.ProxyClient) http.Handler { 33 | html := static.MustParseTemplates( 34 | "static/html/layout.html", 35 | "static/html/navbar.html", 36 | "static/html/mods_find.html", 37 | ) 38 | 39 | return &findHandler{ 40 | html: html, 41 | emitter: emitter, 42 | finder: finder.New(finder.Options{ 43 | Timeout: 1 * time.Minute, 44 | ProxyClient: proxyClient, 45 | }), 46 | log: loggy.New("find-modules-handler"), 47 | } 48 | } 49 | 50 | func (h *findHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 51 | var ( 52 | code int 53 | err error 54 | page *findPage 55 | ) 56 | 57 | switch r.Method { 58 | case http.MethodGet: 59 | code, page, err = h.get(r) 60 | case http.MethodPost: 61 | code, page, err = h.post(r) 62 | } 63 | 64 | if err != nil { 65 | h.log.Errorf("failed to serve find-module page: %v", err) 66 | http.Error(w, err.Error(), code) 67 | h.emitter.Count("ui-find-mod-error", 1) 68 | return 69 | } 70 | 71 | if err := h.html.Execute(w, page); err != nil { 72 | h.log.Errorf("failed to execute find-module page: %v", err) 73 | return 74 | } 75 | 76 | h.emitter.Count("ui-find-mod-ok", 1) 77 | } 78 | 79 | func (h *findHandler) get(r *http.Request) (int, *findPage, error) { 80 | return http.StatusOK, &findPage{ 81 | CSRF: csrf.TemplateField(r), 82 | }, nil 83 | } 84 | 85 | func (h *findHandler) post(r *http.Request) (int, *findPage, error) { 86 | results, query, err := h.parseTextArea(r) 87 | if err != nil { 88 | return http.StatusBadRequest, nil, err 89 | } 90 | 91 | return http.StatusOK, &findPage{ 92 | CSRF: csrf.TemplateField(r), 93 | Found: results, 94 | Query: query, 95 | }, nil 96 | } 97 | 98 | type findResult struct { 99 | Text string 100 | Result *finder.Result 101 | Err error 102 | } 103 | 104 | func (h *findHandler) parseTextArea(r *http.Request) ([]findResult, string, error) { 105 | if err := r.ParseForm(); err != nil { 106 | return nil, "", err 107 | } 108 | 109 | text := r.PostForm.Get("sources-input") 110 | 111 | lines := linesOfText(text) 112 | if len(lines) == 0 { 113 | return nil, "", errors.New("no sources listed") 114 | } 115 | 116 | results := h.processLines(lines) 117 | 118 | return results, text, nil 119 | } 120 | 121 | func (h *findHandler) processLines(lines []string) []findResult { 122 | results := make([]findResult, 0, len(lines)) 123 | for _, line := range lines { 124 | result := h.processLine(line) 125 | results = append(results, result) 126 | } 127 | return results 128 | } 129 | 130 | func (h *findHandler) processLine(line string) findResult { 131 | if !finder.Compatible(line) { 132 | return findResult{ 133 | Text: line, 134 | Err: errors.New("does not match regexp"), 135 | } 136 | } 137 | 138 | result, err := h.finder.Find(line) 139 | if err != nil { 140 | h.log.Warnf("failed to find result for %s: %v", line, err) 141 | return findResult{ 142 | Text: line, 143 | Err: err, 144 | } 145 | } 146 | 147 | return findResult{ 148 | Text: line, 149 | Result: result, 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /proxy/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "oss.indeed.com/go/modprox/pkg/configutil" 10 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 11 | "oss.indeed.com/go/modprox/pkg/netservice" 12 | "oss.indeed.com/go/modprox/pkg/setup" 13 | ) 14 | 15 | type Configuration struct { 16 | APIServer APIServer `json:"api_server"` 17 | Registry Registry `json:"registry"` 18 | Statsd stats.Statsd `json:"statsd"` 19 | ModuleStorage *Storage `json:"module_storage,omitempty"` 20 | ModuleDBStorage *setup.PersistentStore `json:"module_db_storage,omitempty"` 21 | Transforms Transforms `json:"transforms"` 22 | ZipProxy ZipProxy `json:"zip_proxy"` 23 | } 24 | 25 | func (c Configuration) String() string { 26 | return configutil.Format(c) 27 | } 28 | 29 | type ZipProxy struct { 30 | Protocol string `json:"protocol"` // e.g. "https" 31 | BaseURL string `json:"base_url"` // e.g. "proxy.golang.org" 32 | } 33 | 34 | type APIServer struct { 35 | TLS struct { 36 | Enabled bool `json:"enabled"` 37 | Certificate string `json:"certificate"` 38 | Key string `json:"key"` 39 | } `json:"tls"` 40 | BindAddress string `json:"bind_address"` 41 | Port int `json:"port"` 42 | ReadTimeoutS int `json:"read_timeout_s"` 43 | WriteTimeoutS int `json:"write_timeout_s"` 44 | } 45 | 46 | func (s APIServer) Server(mux http.Handler) (*http.Server, error) { 47 | if s.BindAddress == "" { 48 | return nil, errors.New("server bind address is not set") 49 | } 50 | 51 | if s.Port == 0 { 52 | return nil, errors.New("server port is not set") 53 | } 54 | 55 | if s.TLS.Enabled { 56 | if s.TLS.Certificate == "" { 57 | return nil, errors.New("TLS enabled, but server TLS certificate not set") 58 | } 59 | 60 | if s.TLS.Key == "" { 61 | return nil, errors.New("TLS enabled, but server TLS key not set") 62 | } 63 | } 64 | 65 | if s.ReadTimeoutS == 0 { 66 | s.ReadTimeoutS = 60 67 | } 68 | 69 | if s.WriteTimeoutS == 0 { 70 | s.WriteTimeoutS = 60 71 | } 72 | 73 | address := fmt.Sprintf("%s:%d", s.BindAddress, s.Port) 74 | server := &http.Server{ 75 | Addr: address, 76 | Handler: mux, 77 | ReadTimeout: seconds(s.ReadTimeoutS), 78 | WriteTimeout: seconds(s.WriteTimeoutS), 79 | } 80 | 81 | return server, nil 82 | } 83 | 84 | func seconds(s int) time.Duration { 85 | return time.Duration(s) * time.Second 86 | } 87 | 88 | type Storage struct { 89 | DataPath string `json:"data_path"` 90 | IndexPath string `json:"index_path"` 91 | TmpPath string `json:"tmp_path"` 92 | } 93 | 94 | type instances = []netservice.Instance 95 | 96 | type Registry struct { 97 | Instances instances `json:"instances"` 98 | PollFrequencyS int `json:"poll_frequency_s"` 99 | RequestTimeoutS int `json:"request_timeout_s"` 100 | APIKey string `json:"api_key"` 101 | } 102 | 103 | type Transforms struct { 104 | // Deprecated, AutomaticRedirect is now ignored and treated as always-on 105 | AutomaticRedirect bool `json:"auto_redirect"` 106 | DomainRedirects []struct { 107 | Original string `json:"original"` 108 | Substitution string `json:"substitution"` 109 | } `json:"domain_redirects,omitempty"` 110 | DomainHeaders []struct { 111 | Domain string `json:"domain"` 112 | Headers map[string]string `json:"headers"` 113 | } `json:"domain_headers,omitempty"` 114 | DomainPath []struct { 115 | Domain string `json:"domain"` 116 | Path string `json:"path"` 117 | } `json:"domain_paths,omitempty"` 118 | DomainTransport []struct { 119 | Domain string `json:"domain"` 120 | Transport string `json:"transport"` 121 | } `json:"domain_transports,omitempty"` 122 | } 123 | -------------------------------------------------------------------------------- /registry/internal/web/v1_registry_list.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/clients/registry" 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/webutil" 13 | "oss.indeed.com/go/modprox/registry/internal/data" 14 | ) 15 | 16 | type registryList struct { 17 | store data.Store 18 | emitter stats.Sender 19 | log loggy.Logger 20 | } 21 | 22 | func newRegistryList(store data.Store, emitter stats.Sender) http.Handler { 23 | return ®istryList{ 24 | store: store, 25 | emitter: emitter, 26 | log: loggy.New("registry-list-api"), 27 | } 28 | } 29 | 30 | func (h *registryList) ServeHTTP(w http.ResponseWriter, r *http.Request) { 31 | var send toSend 32 | 33 | switch r.Method { 34 | case http.MethodGet: 35 | send = h.get(w, r) 36 | case http.MethodPost: 37 | send = h.post(w, r) 38 | } 39 | 40 | if send.err != nil { 41 | h.log.Errorf("failed to serve request: %v", send.err) 42 | http.Error(w, send.err.Error(), send.code) 43 | h.emitter.Count("api-listmods-error", 1) 44 | return 45 | } 46 | 47 | response := registry.ReqModsResp{ 48 | Mods: send.mods, 49 | } 50 | 51 | webutil.WriteJSON(w, response) 52 | h.emitter.Count("api-listmods-ok", 1) 53 | } 54 | 55 | type toSend struct { 56 | err error 57 | code int 58 | mods []coordinates.SerialModule 59 | } 60 | 61 | func (h *registryList) get(w http.ResponseWriter, r *http.Request) toSend { 62 | h.log.Tracef("listing entire contents of registry") 63 | modules, err := h.store.ListModules() 64 | if err != nil { 65 | return toSend{ 66 | err: err, 67 | code: http.StatusInternalServerError, 68 | mods: nil, 69 | } 70 | } 71 | return toSend{ 72 | err: nil, 73 | code: http.StatusOK, 74 | mods: modules, 75 | } 76 | } 77 | 78 | func (h *registryList) post(w http.ResponseWriter, r *http.Request) toSend { 79 | h.log.Tracef("listing optimized contents of registry") 80 | 81 | // read the body of the incoming request 82 | var inbound registry.ReqMods 83 | if err := json.NewDecoder(r.Body).Decode(&inbound); err != nil { 84 | return toSend{ 85 | err: err, 86 | code: http.StatusBadRequest, 87 | mods: nil, 88 | } 89 | } 90 | 91 | ids, err := h.store.ListModuleIDs() 92 | if err != nil { 93 | return toSend{ 94 | err: err, 95 | code: http.StatusInternalServerError, 96 | mods: nil, 97 | } 98 | } 99 | 100 | h.log.Tracef("proxy sent ID ranges: %v", inbound.IDs) 101 | 102 | // compare that with the modules in the registry 103 | 104 | // return a list of the difference 105 | neededIDs := inListButNotRange(ids, inbound.IDs) 106 | h.log.Tracef("needed ID ranges: %v", neededIDs) 107 | 108 | needed, err := h.store.ListModulesByIDs(neededIDs) 109 | if err != nil { 110 | return toSend{ 111 | err: err, 112 | code: http.StatusInternalServerError, 113 | mods: nil, 114 | } 115 | } 116 | 117 | h.log.Tracef("proxy needs %d mods", len(needed)) 118 | 119 | return toSend{ 120 | err: nil, 121 | code: http.StatusOK, 122 | mods: needed, 123 | } 124 | } 125 | 126 | // this could be optimized doing a kind of skipping merge, but for 127 | // now the O(n) scan should be okay 128 | func inListButNotRange(ids []int64, ranges coordinates.RangeIDs) []int64 { 129 | var neededIDs []int64 130 | for _, id := range ids { 131 | needsID := true 132 | for _, r := range ranges { 133 | if inRange(id, r) { 134 | needsID = false 135 | break // move on to next id 136 | } 137 | } 138 | if needsID { 139 | neededIDs = append(neededIDs, id) 140 | } 141 | } 142 | return neededIDs 143 | } 144 | 145 | func inRange(i int64, rangeID coordinates.RangeID) bool { 146 | left := rangeID[0] 147 | right := rangeID[1] 148 | return i >= left && i <= right 149 | } 150 | -------------------------------------------------------------------------------- /registry/static/html/home.html: -------------------------------------------------------------------------------- 1 | {{define "body"}} 2 |
3 |


4 |
5 |

proxies reporting to registry

6 |
7 |
8 | 9 | 10 | {{range .Proxies}} 11 | 12 | 64 | 74 | 75 | 76 | 77 | 78 | {{end}} 79 | 80 |
13 | 14 | 15 | 18 | 19 | 20 | 37 | 38 | 39 | 61 | 62 |
16 | {{.Configuration.Self.Address}}:{{.Configuration.Self.Port}} 17 |
21 |

Latest Heartbeat

22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 |
Last Seen{{.Heartbeat.TimeSince}}
Modules{{.Heartbeat.NumModules}}
Versions{{.Heartbeat.NumVersions}}
36 |
40 |

Registries Configuration

41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | {{range .Configuration.Registry.Instances}} 54 | 55 | 56 | 57 | 58 | {{end}} 59 |
Poll Frequency{{.Configuration.Registry.PollFrequencyS}}s
Request Timeout{{.Configuration.Registry.RequestTimeoutS}}s
Registry Instances
- {{.Address}}:{{.Port}}
60 |
63 | 		65 | 66 | 67 | 71 | 72 |
68 | 69 | 
{{.TransformsText}}
70 |
73 |
81 | 82 |
83 |
84 | {{end}} 85 | -------------------------------------------------------------------------------- /proxy/internal/modules/store/fs.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | 10 | "github.com/pkg/errors" 11 | 12 | "gophers.dev/pkgs/atomicfs" 13 | "gophers.dev/pkgs/loggy" 14 | 15 | "oss.indeed.com/go/modprox/pkg/coordinates" 16 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 17 | "oss.indeed.com/go/modprox/pkg/repository" 18 | ) 19 | 20 | const ( 21 | filePerm = 0660 22 | directoryPerm = 0770 23 | ) 24 | 25 | type fsStore struct { 26 | options Options 27 | emitter stats.Sender 28 | writer atomicfs.FileWriter 29 | log loggy.Logger 30 | } 31 | 32 | type Options struct { 33 | Directory string 34 | TmpDirectory string 35 | } 36 | 37 | func NewStore(options Options, emitter stats.Sender) ZipStore { 38 | if options.Directory == "" { 39 | panic("no directory set for store") 40 | } 41 | 42 | writer := atomicfs.NewFileWriter(atomicfs.Options{ 43 | TmpDirectory: options.TmpDirectory, 44 | Mode: filePerm, 45 | }) 46 | 47 | return &fsStore{ 48 | options: options, 49 | emitter: emitter, 50 | writer: writer, 51 | log: loggy.New("fs-store"), 52 | } 53 | } 54 | 55 | func (s *fsStore) GetZip(mod coordinates.Module) (repository.Blob, error) { 56 | s.log.Tracef("retrieving module %s", mod) 57 | 58 | start := time.Now() 59 | blob, err := s.getZip(mod) 60 | if err != nil { 61 | s.emitter.Count("fsstore-getzip-failure", 1) 62 | return nil, err 63 | } 64 | 65 | s.emitter.GaugeMS("fsstore-getzip-elapsed-ms", start) 66 | return blob, nil 67 | } 68 | 69 | func (s *fsStore) getZip(mod coordinates.Module) (repository.Blob, error) { 70 | zipFile := filepath.Join( 71 | s.fullPathOf(mod), 72 | zipName(mod), 73 | ) 74 | return ioutil.ReadFile(zipFile) 75 | } 76 | 77 | func (s *fsStore) DelZip(mod coordinates.Module) error { 78 | s.log.Tracef("removing module %s", mod) 79 | 80 | start := time.Now() 81 | err := s.removeZip(mod) 82 | if err != nil { 83 | s.emitter.Count("fsstore-rmzip-failure", 1) 84 | return err 85 | } 86 | 87 | s.emitter.GaugeMS("fsstore-rmzip-elapsed-ms", start) 88 | return nil 89 | } 90 | 91 | func (s *fsStore) removeZip(mod coordinates.Module) error { 92 | zipFile := filepath.Join( 93 | s.fullPathOf(mod), 94 | zipName(mod), 95 | ) 96 | return os.Remove(zipFile) 97 | } 98 | 99 | func (s *fsStore) PutZip(mod coordinates.Module, blob repository.Blob) error { 100 | s.log.Infof("will save %s to disk, %d bytes", mod, len(blob)) 101 | 102 | start := time.Now() 103 | if err := s.putZip(mod, blob); err != nil { 104 | s.emitter.Count("fsstore-putzip-failure", 1) 105 | return err 106 | } 107 | 108 | s.emitter.GaugeMS("fsstore-putzip-elapsed-ms", start) 109 | return nil 110 | } 111 | 112 | func (s *fsStore) putZip(mod coordinates.Module, blob repository.Blob) error { 113 | exists, err := s.exists(mod) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | if exists { 119 | s.log.Warnf("not saving %s because we already have it @ %s", mod, pathOf) 120 | return errors.Errorf("already have a copy of %s", mod) 121 | } 122 | 123 | if err := s.safeWriteZip(mod, blob); err != nil { 124 | s.log.Errorf("failed to write zip for %s, %v", mod, err) 125 | return err 126 | } 127 | 128 | return nil 129 | } 130 | 131 | func (s *fsStore) safeWriteZip(mod coordinates.Module, blob repository.Blob) error { 132 | modPath := s.fullPathOf(mod) 133 | s.log.Tracef("writing module zip into path: %s", modPath) 134 | 135 | // writing the zip always goes first, make sure the tree exists 136 | if err := os.MkdirAll(modPath, directoryPerm); err != nil { 137 | return err 138 | } 139 | 140 | zipFile := filepath.Join(modPath, zipName(mod)) 141 | reader := bytes.NewReader(blob) 142 | return s.writer.Write(reader, zipFile) 143 | } 144 | 145 | func zipName(mod coordinates.Module) string { 146 | return mod.Version + ".zip" 147 | } 148 | 149 | func (s *fsStore) exists(mod coordinates.Module) (bool, error) { 150 | modPath := s.fullPathOf(mod) 151 | _, err := os.Stat(modPath) 152 | if os.IsNotExist(err) { 153 | return false, nil 154 | } 155 | return err != nil, err 156 | } 157 | 158 | func (s *fsStore) fullPathOf(mod coordinates.Module) string { 159 | return filepath.Join( 160 | s.options.Directory, 161 | pathOf(mod), 162 | ) 163 | } 164 | 165 | func pathOf(mod coordinates.Module) string { 166 | return filepath.FromSlash(mod.Source) // eh windows? 167 | } 168 | -------------------------------------------------------------------------------- /proxy/internal/modules/get/downloader.go: -------------------------------------------------------------------------------- 1 | package get 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "gophers.dev/pkgs/loggy" 8 | 9 | "oss.indeed.com/go/modprox/pkg/clients/zips" 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/pkg/repository" 13 | "oss.indeed.com/go/modprox/pkg/upstream" 14 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 15 | ) 16 | 17 | //go:generate go run github.com/gojuno/minimock/v3/cmd/minimock -g -i Downloader -s _mock.go 18 | 19 | type Downloader interface { 20 | Download(module coordinates.SerialModule) error 21 | } 22 | 23 | func New( 24 | proxyClient zips.ProxyClient, 25 | upstreamClient zips.UpstreamClient, 26 | resolver upstream.Resolver, 27 | store store.ZipStore, 28 | index store.Index, 29 | emitter stats.Sender, 30 | ) Downloader { 31 | return &downloader{ 32 | proxyClient: proxyClient, 33 | upstreamClient: upstreamClient, 34 | resolver: resolver, 35 | store: store, 36 | index: index, 37 | emitter: emitter, 38 | log: loggy.New("downloader"), 39 | } 40 | } 41 | 42 | type downloader struct { 43 | proxyClient zips.ProxyClient 44 | upstreamClient zips.UpstreamClient 45 | resolver upstream.Resolver 46 | store store.ZipStore 47 | index store.Index 48 | emitter stats.Sender 49 | log loggy.Logger 50 | } 51 | 52 | func (d *downloader) downloadFromProxy(mod coordinates.SerialModule) (repository.Blob, error) { 53 | d.log.Infof("going to download from proxy: %s", mod.String()) 54 | 55 | // download the well-formed zip from the proxy 56 | start := time.Now() 57 | blob, err := d.proxyClient.Get(mod.Module) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | d.emitter.GaugeMS("download-mod-elapsed-ms", start) 63 | d.log.Infof("downloaded upstream blob of size: %d", len(blob)) 64 | 65 | // no need to re-write, this is already a correctly formatted zip 66 | return blob, nil 67 | } 68 | 69 | func (d *downloader) downloadFromUpstream(mod coordinates.SerialModule) (repository.Blob, error) { 70 | d.log.Infof("going to download from upstream: %s", mod.String()) 71 | 72 | request, err := d.resolver.Resolve(mod.Module) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | // download the raw-zip from the upstream source 78 | start := time.Now() 79 | blob, err := d.upstreamClient.Get(request) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | d.emitter.GaugeMS("download-mod-elapsed-ms", start) 85 | d.log.Infof("downloaded upstream blob of size: %d", len(blob)) 86 | 87 | rewritten, err := zips.Rewrite(mod.Module, blob) 88 | if err != nil { 89 | d.log.Errorf("failed to rewrite blob for %s, %v", mod, err) 90 | return nil, err 91 | } 92 | 93 | return rewritten, nil 94 | } 95 | 96 | func (d *downloader) storeBlob(mod coordinates.SerialModule, blob repository.Blob) error { 97 | 98 | if err := d.store.PutZip(mod.Module, blob); err != nil { 99 | d.log.Errorf("failed to save blob to zip store for %s, %v", mod, err) 100 | return err 101 | } 102 | 103 | modFile, exists, err := blob.ModFile() 104 | if err != nil { 105 | d.log.Errorf("failed to re-read re-written zip file for %s, %v", mod, err) 106 | return err 107 | } 108 | if !exists { 109 | modFile = emptyModFile(mod) 110 | } 111 | 112 | ma := store.ModuleAddition{ 113 | Mod: mod.Module, 114 | UniqueID: mod.SerialID, 115 | ModFile: modFile, 116 | } 117 | 118 | if err := d.index.Put(ma); err != nil { 119 | d.log.Errorf("failed to updated index for %s, %v", mod, err) 120 | return err 121 | } 122 | 123 | d.log.Tracef("stored %s, was %d bytes", mod, len(blob)) 124 | 125 | return nil 126 | } 127 | 128 | func (d *downloader) Download(mod coordinates.SerialModule) error { 129 | useProxy, err := d.resolver.UseProxy(mod.Module) 130 | if err != nil { 131 | d.log.Errorf("could not decide on using proxy:", err) 132 | return err 133 | } 134 | { 135 | var ( 136 | blob repository.Blob 137 | err error 138 | ) 139 | switch useProxy { 140 | case true: 141 | blob, err = d.downloadFromProxy(mod) 142 | default: 143 | blob, err = d.downloadFromUpstream(mod) 144 | } 145 | 146 | if err != nil { 147 | d.log.Errorf("failed to download %s: %v", mod, err) 148 | return err 149 | } 150 | 151 | return d.storeBlob(mod, blob) 152 | } 153 | } 154 | 155 | func emptyModFile(mod coordinates.SerialModule) string { 156 | return fmt.Sprintf("module %s\n", mod.Module.Source) 157 | } 158 | -------------------------------------------------------------------------------- /registry/internal/web/mods_add.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "errors" 5 | "html/template" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/gorilla/csrf" 10 | 11 | "gophers.dev/pkgs/loggy" 12 | 13 | "oss.indeed.com/go/modprox/pkg/coordinates" 14 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 15 | "oss.indeed.com/go/modprox/pkg/repository" 16 | "oss.indeed.com/go/modprox/registry/internal/data" 17 | "oss.indeed.com/go/modprox/registry/static" 18 | ) 19 | 20 | type newPage struct { 21 | Mods []Parsed 22 | CSRF template.HTML 23 | Query string 24 | } 25 | 26 | type newHandler struct { 27 | html *template.Template 28 | store data.Store 29 | emitter stats.Sender 30 | log loggy.Logger 31 | } 32 | 33 | func newAddHandler(store data.Store, emitter stats.Sender) http.Handler { 34 | html := static.MustParseTemplates( 35 | "static/html/layout.html", 36 | "static/html/navbar.html", 37 | "static/html/mods_add.html", 38 | ) 39 | 40 | return &newHandler{ 41 | html: html, 42 | store: store, 43 | emitter: emitter, 44 | log: loggy.New("add-modules-handler"), 45 | } 46 | } 47 | 48 | func (h *newHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 49 | var ( 50 | code int 51 | err error 52 | page *newPage 53 | ) 54 | 55 | switch r.Method { 56 | case http.MethodGet: 57 | code, page, err = h.get(r) 58 | case http.MethodPost: 59 | code, page, err = h.post(r) 60 | } 61 | 62 | if err != nil { 63 | h.log.Errorf("failed to serve add-module page: %v", err) 64 | http.Error(w, err.Error(), code) 65 | h.emitter.Count("ui-add-mod-error", 1) 66 | return 67 | } 68 | 69 | if err := h.html.Execute(w, page); err != nil { 70 | h.log.Errorf("failed to execute add-module page: %v", err) 71 | return 72 | } 73 | 74 | h.emitter.Count("ui-add-mod-ok", 1) 75 | } 76 | 77 | func (h *newHandler) get(r *http.Request) (int, *newPage, error) { 78 | if err := r.ParseForm(); err != nil { 79 | return http.StatusBadRequest, nil, err 80 | } 81 | packages := r.Form["packages"] 82 | return http.StatusOK, &newPage{ 83 | Mods: nil, 84 | CSRF: csrf.TemplateField(r), 85 | Query: strings.Join(packages, "\n"), 86 | }, nil 87 | } 88 | 89 | func (h *newHandler) post(r *http.Request) (int, *newPage, error) { 90 | mods, query, err := h.parseTextArea(r) 91 | if err != nil { 92 | return http.StatusBadRequest, nil, err 93 | } 94 | 95 | modulesAdded, err := h.storeNewMods(mods) 96 | if err != nil { 97 | return http.StatusInternalServerError, nil, err 98 | } 99 | 100 | h.log.Infof("added %d new modules", modulesAdded) 101 | 102 | return http.StatusOK, &newPage{ 103 | Mods: mods, 104 | CSRF: csrf.TemplateField(r), 105 | Query: query, 106 | }, nil 107 | } 108 | 109 | func (h *newHandler) storeNewMods(mods []Parsed) (int, error) { 110 | ableToAdd := make([]coordinates.Module, 0, len(mods)) 111 | for _, parsed := range mods { 112 | if parsed.Err == nil { 113 | ableToAdd = append(ableToAdd, parsed.Module) 114 | } 115 | } 116 | 117 | for _, able := range ableToAdd { 118 | h.log.Tracef("[web] adding to registry: %s@%s", able.Source, able.Version) 119 | } 120 | 121 | return h.store.InsertModules(ableToAdd) 122 | } 123 | 124 | type Parsed struct { 125 | Text string 126 | Module coordinates.Module 127 | Err error 128 | } 129 | 130 | func (h *newHandler) parseTextArea(r *http.Request) ([]Parsed, string, error) { 131 | // get the text from form and use a scanner to get each line 132 | if err := r.ParseForm(); err != nil { 133 | return nil, "", err 134 | } 135 | text := r.PostForm.Get("modules-input") 136 | 137 | // parse the text field into lines then module + tag 138 | lines := linesOfText(text) 139 | if len(lines) == 0 { 140 | return nil, "", errors.New("no modules listed") 141 | } 142 | results := h.parseLines(lines) 143 | 144 | return results, text, nil 145 | } 146 | 147 | func (h *newHandler) parseLines(lines []string) []Parsed { 148 | results := make([]Parsed, 0, len(lines)) 149 | for _, line := range lines { 150 | if !h.skipLine(line) { 151 | result := h.parseLine(line) 152 | results = append(results, result) 153 | } 154 | } 155 | return results 156 | } 157 | 158 | func (h *newHandler) skipLine(line string) bool { 159 | if strings.HasPrefix(line, "module ") { 160 | return true 161 | } 162 | if strings.Contains(line, "(") { 163 | return true 164 | } 165 | if strings.Contains(line, ")") { 166 | return true 167 | } 168 | return false 169 | } 170 | 171 | func (h *newHandler) parseLine(line string) Parsed { 172 | mod, err := repository.Parse(line) 173 | return Parsed{ 174 | Text: line, 175 | Module: mod, 176 | Err: err, 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /proxy/internal/modules/bg/worker.go: -------------------------------------------------------------------------------- 1 | package bg 2 | 3 | import ( 4 | "time" 5 | 6 | "gophers.dev/pkgs/loggy" 7 | "gophers.dev/pkgs/repeat/x" 8 | 9 | "oss.indeed.com/go/modprox/pkg/clients/registry" 10 | "oss.indeed.com/go/modprox/pkg/coordinates" 11 | "oss.indeed.com/go/modprox/pkg/metrics/stats" 12 | "oss.indeed.com/go/modprox/proxy/internal/modules/get" 13 | "oss.indeed.com/go/modprox/proxy/internal/modules/store" 14 | "oss.indeed.com/go/modprox/proxy/internal/problems" 15 | ) 16 | 17 | type Options struct { 18 | // Frequency determines how often the worker will check in 19 | // with the registry, looking for new modules that need to be 20 | // downloaded by this instance of the proxy. A typical value 21 | // would be something like 30 seconds - not too slow, but also 22 | // not spamming the network with polling traffic. 23 | Frequency time.Duration 24 | } 25 | 26 | // A Worker runs in the background, polling the registry for new 27 | // modules that need to be downloaded, and downloading those modules 28 | // as needed. 29 | type Worker interface { 30 | Start(options Options) 31 | } 32 | 33 | type worker struct { 34 | registryClient registry.Client 35 | emitter stats.Sender 36 | dlTracker problems.Tracker 37 | index store.Index 38 | store store.ZipStore 39 | downloader get.Downloader 40 | registryRequester get.RegistryAPI 41 | log loggy.Logger 42 | } 43 | 44 | func New( 45 | emitter stats.Sender, 46 | dlTracker problems.Tracker, 47 | index store.Index, 48 | store store.ZipStore, 49 | registryRequester get.RegistryAPI, 50 | downloader get.Downloader, 51 | ) Worker { 52 | return &worker{ 53 | emitter: emitter, 54 | dlTracker: dlTracker, 55 | index: index, 56 | store: store, 57 | downloader: downloader, 58 | registryRequester: registryRequester, 59 | log: loggy.New("bg-worker"), 60 | } 61 | } 62 | 63 | func (w *worker) Start(options Options) { 64 | go func() { 65 | _ = x.Interval(options.Frequency, func() error { 66 | if err := w.loop(); err != nil { 67 | w.log.Errorf("worker loop iteration had error: %v", err) 68 | // never return an error, which would stop the worker 69 | // instead, we remain hopeful the next iteration will work 70 | } 71 | return nil 72 | }) 73 | }() 74 | } 75 | 76 | func (w *worker) loop() error { 77 | w.log.Infof("worker loop starting") 78 | 79 | mods, err := w.acquireMods() 80 | if err != nil { 81 | return err 82 | } 83 | 84 | _ = mods 85 | 86 | // we have a list of modules already downloaded to fs 87 | // we have a list of modules from registry that we want 88 | // do a diff, finding: 89 | // - modules we have but do not need anymore 90 | // - modules we need but to not have yet 91 | // then prune modules we do not want 92 | // then DL and save modules we do want 93 | // also, take into account redirects and such 94 | return nil 95 | } 96 | 97 | func (w *worker) acquireMods() ([]coordinates.SerialModule, error) { 98 | ids, err := w.index.IDs() 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | mods, err := w.registryRequester.ModulesNeeded(ids) 104 | if err != nil { 105 | w.log.Errorf("failed to acquire list of needed mods from registry, %v", err) 106 | return nil, err 107 | } 108 | w.log.Infof("acquired list of %d mods from registry", len(mods)) 109 | 110 | for _, mod := range mods { 111 | w.log.Tracef("- %s @ %s", mod.Source, mod.Version) 112 | } 113 | 114 | for _, mod := range mods { 115 | // only download mod if we do not already have it 116 | exists, indexID, err := w.index.Contains(mod.Module) 117 | if err != nil { 118 | w.log.Errorf("problem with index lookups: %v", err) 119 | continue // may as well try the others 120 | } 121 | 122 | if exists { 123 | w.log.Tracef("already have %s, not going to download it again", mod) 124 | // set indexID to newID if they do not match 125 | if indexID != mod.SerialID { 126 | w.log.Infof( 127 | "indexed ID of %d for %s does not match ID %d, will update", 128 | indexID, 129 | mod, 130 | mod.SerialID, 131 | ) 132 | if err = w.index.UpdateID(mod); err != nil { 133 | w.log.Errorf("problem updating index ID: %v", err) 134 | } 135 | } 136 | continue // move on to the next one 137 | } 138 | 139 | if err := w.downloader.Download(mod); err != nil { 140 | w.log.Errorf("failed to download %s, %v", mod, err) 141 | w.dlTracker.Set(problems.Create(mod.Module, err)) 142 | continue // may as well try the others 143 | } 144 | w.log.Tracef("downloaded %s!", mod) 145 | } 146 | 147 | return mods, nil 148 | } 149 | -------------------------------------------------------------------------------- /registry/static/css/registry.css: -------------------------------------------------------------------------------- 1 | /* ----- bootstrap stuff ----- */ 2 | .container { 3 | margin-left: 80px 4 | } 5 | 6 | .navbar { 7 | background-color: #660066; 8 | } 9 | 10 | .navbar-inverse .navbar-brand { 11 | color: white; 12 | } 13 | 14 | .navbar-inverse .navbar-brand:hover { 15 | color: lightgray; 16 | } 17 | 18 | .navbar-inverse .navbar-nav>li>a { 19 | color: white; 20 | } 21 | 22 | .navbar-inverse .navbar-nav>li>a:hover { 23 | color: lightgray; 24 | } 25 | 26 | .navbar-inverse .navbar-nav>.open>a, .navbar-inverse .navbar-nav>.open>a:focus, .navbar-inverse .navbar-nav>.open>a:hover { 27 | background-color: white; 28 | color: #660066; 29 | } 30 | 31 | /* ----- modprox stuff ------- */ 32 | 33 | .homelink { 34 | font-weight: bold; 35 | font-size: x-large; 36 | } 37 | 38 | .otherlink { 39 | font-size: large; 40 | } 41 | 42 | .bigheader { 43 | alignment: left; 44 | padding-bottom: 20px; 45 | } 46 | 47 | .new-module-instructions { 48 | padding-left: 15px; 49 | padding-bottom: 20px; 50 | font-size: medium; 51 | } 52 | 53 | .new-module-instructions em { 54 | font-style: normal; 55 | font-family: monospace; 56 | } 57 | 58 | .list-mods tr td { 59 | padding-right: 20px; 60 | } 61 | 62 | .list-mods tr td a { 63 | color: black; 64 | } 65 | 66 | .mod-tag { 67 | font-size: medium; 68 | font-weight: bold; 69 | font-family: monospace; 70 | } 71 | 72 | .mod-text { 73 | font-size: large; 74 | font-weight: bold; 75 | font-family: monospace; 76 | } 77 | 78 | .mod-ok { 79 | color: green; 80 | font-weight: bold; 81 | } 82 | 83 | .mod-bad { 84 | color: darkred; 85 | font-weight: bold; 86 | } 87 | 88 | .mod-pkg-name { 89 | font-family: monospace; 90 | font-size: large; 91 | padding-bottom: 5px; 92 | } 93 | 94 | .mod-pkg-versions { 95 | font-size: small; 96 | padding-left: 10px; 97 | } 98 | 99 | .mod-tbl tr td { 100 | padding-right: 20px; 101 | } 102 | 103 | .mod-h-source { 104 | color: #660066; 105 | font-family: monospace; 106 | } 107 | 108 | .mod-none { 109 | color: gray; 110 | font-size: large; 111 | padding-left: 10px; 112 | } 113 | 114 | .mod-show tr td { 115 | padding-right: 10px; 116 | font-size: x-large; 117 | } 118 | 119 | .proxy-addr { 120 | font-size: xx-large; 121 | font-family: monospace; 122 | font-weight: bold; 123 | } 124 | 125 | .proxy-heartbeat { 126 | padding-top: 5px; 127 | padding-left: 10px; 128 | } 129 | 130 | .proxy-heartbeat p { 131 | font-weight: bold; 132 | font-size: large; 133 | } 134 | 135 | .proxy-heartbeat-label { 136 | padding-left: 1em; 137 | font-weight: bold; 138 | } 139 | 140 | .proxy-heartbeat-value { 141 | padding-left: 3em; 142 | font-family: monospace; 143 | } 144 | 145 | .proxy-storage { 146 | padding-top: 5px; 147 | padding-left: 10px; 148 | } 149 | 150 | .proxy-storage p { 151 | font-weight: bold; 152 | font-size: large; 153 | } 154 | 155 | .proxy-storage .proxy-storage-label { 156 | padding-left: 1em; 157 | font-weight: bold; 158 | } 159 | 160 | .proxy-storage .proxy-storage-value { 161 | padding-left: 3em; 162 | font-family: monospace; 163 | } 164 | 165 | .proxy-registry { 166 | padding-top: 5px; 167 | padding-left: 10px; 168 | } 169 | 170 | .proxy-registry p { 171 | font-weight: bold; 172 | font-size: large; 173 | } 174 | 175 | .proxy-registry .proxy-registry-label { 176 | padding-left: 1em; 177 | font-weight: bold; 178 | } 179 | 180 | .proxy-registry .proxy-registry-value { 181 | padding-left: 3em; 182 | font-family: monospace; 183 | } 184 | 185 | .proxy-registry .proxy-registry-instances td { 186 | padding-left: 50px; 187 | font-family: monospace; 188 | } 189 | 190 | .proxy-transforms { 191 | padding-top: 5px; 192 | padding-left: 20px; 193 | } 194 | 195 | .proxy-transforms p { 196 | font-weight: bold; 197 | font-size: large; 198 | } 199 | 200 | .find-desc { 201 | padding-left: 10px; 202 | } 203 | 204 | .find-red { 205 | color: red; 206 | } 207 | 208 | .find-space { 209 | width: 40px; 210 | } 211 | 212 | .find-notfound { 213 | color: gray; 214 | } 215 | 216 | .find-pkg { 217 | font-family: monospace; 218 | color: black; 219 | } 220 | 221 | .find-label { 222 | font-weight: bold; 223 | } 224 | 225 | .find-code { 226 | font-family: monospace; 227 | padding-left: 2em; 228 | } 229 | 230 | .find-code-sm { 231 | font-family: monospace; 232 | padding-left: 2em; 233 | font-size: xx-small; 234 | } 235 | 236 | /* --- about page --- */ 237 | .about-text { 238 | width: 40em; 239 | } 240 | 241 | .bigheader h3 { 242 | color: #505050; 243 | } 244 | 245 | .about-text h4 { 246 | color: #565656; 247 | } 248 | 249 | .about-text p { 250 | color: #666666; 251 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # modprox - The Private Go Module Proxy 2 | modprox is a Go Module Proxy focused on the private internal hosting use case 3 | 4 | documentation @ [modprox.org](https://modprox.org) 5 | 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/modprox/mp)](https://goreportcard.com/report/github.com/modprox/mp) 7 | [![Build Status](https://travis-ci.org/modprox/mp.svg?branch=master)](https://travis-ci.org/modprox/mp) 8 | [![GoDoc](https://godoc.org/github.com/modprox/mp?status.svg)](https://godoc.org/github.com/modprox/mp) 9 | [![NetflixOSS Lifecycle](https://img.shields.io/github.com/modprox/mp.svg)](OSSMETADATA) 10 | [![License](https://img.shields.io/github/license/modprox/mp.svg?style=flat-square)](LICENSE) 11 | 12 | # Project Overview 13 | 14 | Module `oss.indeed.com/go/modprox` provides a solution for hosting an internal 15 | Go Module Proxy that is capable of communicating with private authenticated 16 | git repositories. 17 | 18 | # Contributing 19 | 20 | We welcome contributions! Feel free to help make `modprox` better. 21 | 22 | #### Process 23 | 24 | - We track bugs / features in [Issues](https://github.com/modprox/mp/issues) 25 | - Open an issue and describe the desired feature / bug fix before making 26 | changes. It's useful to get a second pair of eyes before investing development 27 | effort. 28 | - Make the change. If adding a new feature, remember to provide tests that 29 | demonstrate the new feature works, including any error paths. If contributing 30 | a bug fix, add tests that demonstrate the erroneous behavior is fixed. 31 | - Open a pull request. Automated CI tests will run. If the tests fail, please 32 | make changes to fix the behavior, and repeat until the tests pass. 33 | - Once everything looks good, one of the indeedeng members will review the 34 | PR and provide feedback. 35 | 36 | ### Setting up modprox in your environment 37 | For setting up your own instances of the modprox components, check out the 38 | extensive documentation on [modprox.org](https://modprox.org/#starting) 39 | 40 | #### Hacking on the Registry 41 | 42 | The registry needs a persistent store, and for local development we have a docker image 43 | with MySQL setup to automatically create tables and users. To make things super simple, in 44 | the `hack/` directory there is a `docker-compose` file already configured to setup the basic 45 | containers needed for local developemnt. Simply run 46 | ```bash 47 | $ docker-compose up 48 | ``` 49 | in the `hack/` directory to get them going. Also in the `hack/` directory is a script for 50 | connecting to the MySQL that is running in the docker container, for ease of poking around. 51 | ```bash 52 | $ compose up 53 | Starting modprox-fakeadog ... done 54 | Starting modprox-mysql-proxy ... done 55 | Starting modprox-mysql-registry ... done 56 | Attaching to modprox-mysql-proxy, modprox-mysql-registry, modprox-fakeadog 57 | modprox-mysql-registry | [Entrypoint] MySQL Docker Image 5.7.26-1.1.11 58 | modprox-mysql-registry | [Entrypoint] Initializing database 59 | modprox-mysql-proxy | [Entrypoint] MySQL Docker Image 5.7.26-1.1.11 60 | modprox-mysql-proxy | [Entrypoint] Initializing database 61 | modprox-mysql-proxy | [Entrypoint] Database initialized 62 | modprox-fakeadog | time="2019-06-26T18:19:14Z" level=info msg="listening on 0.0.0.0:8125" 63 | modprox-mysql-registry | [Entrypoint] Database initialized 64 | ``` 65 | 66 | Also in the `hack/` directory are some sample configuration files. By default, the included `run-dev.sh` 67 | script will use the `hack/configs/registry-local.mysql.json` file, which works well with the included 68 | `docker-compose.yaml` file. 69 | 70 | #### Hacking on the Proxy 71 | 72 | The Proxy needs to persist its data-store of downloaded modules. It can be configured to either persist them to disk 73 | or to MySQL. 74 | 75 | ##### local disk config 76 | ```json 77 | "module_storage": { 78 | "data_path": "", 79 | "index_path": "", 80 | "tmp_path": "" 81 | } 82 | ``` 83 | Note that `data_path` and `tmp_path` should point to paths on the same filesystem. 84 | 85 | ##### MySQL config 86 | ```json 87 | "module_db_storage": { 88 | "mysql": { 89 | "user": "docker", 90 | "password": "docker", 91 | "address": "localhost:3306", 92 | "database": "modproxdb-prox", 93 | "allow_native_passwords": true 94 | } 95 | } 96 | ``` 97 | 98 | # Asking Questions 99 | 100 | For technical questions about `modprox`, just file an issue in the GitHub tracker. 101 | 102 | For questions about Open Source in Indeed Engineering, send us an email at 103 | opensource@indeed.com 104 | 105 | # Maintainers 106 | 107 | The `oss.indeed.com/go/modprox` module is maintained by Indeed Engineering. 108 | 109 | While we are always busy helping people get jobs, we will try to respond to 110 | GitHub issues, pull requests, and questions within a couple of business days. 111 | 112 | # Code of Conduct 113 | 114 | `oss.indeed.com/go/modprox` is governed by the[Contributer Covenant v1.4.1](CODE_OF_CONDUCT.md) 115 | 116 | For more information please contact opensource@indeed.com. 117 | 118 | # License 119 | 120 | The `oss.indeed.com/go/modprox` module is open source under the [BSD-3-Clause](LICENSE) 121 | license. 122 | -------------------------------------------------------------------------------- /registry/internal/data/pokes.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "io" 7 | "time" 8 | 9 | "oss.indeed.com/go/modprox/pkg/clients/payloads" 10 | "oss.indeed.com/go/modprox/pkg/netservice" 11 | ) 12 | 13 | func (s *store) SetStartConfig(config payloads.Configuration) error { 14 | storageText, registriesText, transformsText, err := config.Texts() 15 | if err != nil { 16 | return err 17 | } 18 | _, err = s.statements[insertStartupConfigSQL].Exec( 19 | config.Self.Address, 20 | config.Self.Port, 21 | storageText, 22 | registriesText, 23 | transformsText, 24 | storageText, // on dup 25 | registriesText, // on dup 26 | transformsText, // on dup 27 | ) 28 | return err 29 | } 30 | 31 | func ignoreClose(c io.Closer) { 32 | _ = c.Close() 33 | } 34 | 35 | func (s *store) ListStartConfigs() ([]payloads.Configuration, error) { 36 | start := time.Now() 37 | configs, err := s.listStartConfigs() 38 | if err != nil { 39 | s.emitter.Count("db-list-start-configs-error", 1) 40 | return nil, err 41 | } 42 | 43 | s.emitter.GaugeMS("db-list-start-configs-elapsed-ms", start) 44 | return configs, nil 45 | } 46 | 47 | func (s *store) listStartConfigs() ([]payloads.Configuration, error) { 48 | rows, err := s.statements[selectStartupConfigsSQL].Query() 49 | if err != nil { 50 | return nil, err 51 | } 52 | defer ignoreClose(rows) 53 | 54 | var configs []payloads.Configuration 55 | for rows.Next() { 56 | var ( 57 | hostname string 58 | port int 59 | storageText string 60 | registryText string 61 | transformsText string 62 | ) 63 | if err := rows.Scan( 64 | &hostname, 65 | &port, 66 | &storageText, 67 | ®istryText, 68 | &transformsText, 69 | ); err != nil { 70 | return nil, err 71 | } 72 | 73 | c, err := newConfig( 74 | hostname, 75 | port, 76 | storageText, 77 | registryText, 78 | transformsText, 79 | ) 80 | if err != nil { 81 | return nil, err 82 | } 83 | configs = append(configs, c) 84 | } 85 | if err := rows.Err(); err != nil { 86 | return nil, err 87 | } 88 | 89 | return configs, nil 90 | } 91 | 92 | func newConfig( 93 | hostname string, 94 | port int, 95 | storageText, 96 | registryText, 97 | transformsText string, 98 | ) (payloads.Configuration, error) { 99 | c := payloads.Configuration{ 100 | Self: netservice.Instance{ 101 | Address: hostname, 102 | Port: port, 103 | }, 104 | } 105 | 106 | if err := json.Unmarshal([]byte(storageText), &c.DiskStorage); err != nil { 107 | return c, err 108 | } 109 | 110 | if err := json.Unmarshal([]byte(registryText), &c.Registry); err != nil { 111 | return c, err 112 | } 113 | 114 | if err := json.Unmarshal([]byte(transformsText), &c.Transforms); err != nil { 115 | return c, err 116 | } 117 | 118 | return c, nil 119 | } 120 | 121 | func (s *store) SetHeartbeat(heartbeat payloads.Heartbeat) error { 122 | start := time.Now() 123 | err := s.setHeartbeat(heartbeat) 124 | if err != nil { 125 | s.emitter.Count("db-set-heartbeat-failure", 1) 126 | return err 127 | } 128 | 129 | s.emitter.GaugeMS("db-set-heartbeat-elapsed-ms", start) 130 | return nil 131 | } 132 | 133 | func (s *store) setHeartbeat(heartbeat payloads.Heartbeat) error { 134 | _, err := s.statements[insertHeartbeatSQL].Exec( 135 | heartbeat.Self.Address, 136 | heartbeat.Self.Port, 137 | heartbeat.NumModules, 138 | heartbeat.NumVersions, 139 | heartbeat.NumModules, 140 | heartbeat.NumVersions, 141 | ) 142 | return err 143 | } 144 | 145 | func (s *store) ListHeartbeats() ([]payloads.Heartbeat, error) { 146 | start := time.Now() 147 | heartbeats, err := s.listHeartbeats() 148 | if err != nil { 149 | s.emitter.Count("db-list-heartbeats-failure", 1) 150 | return nil, err 151 | } 152 | 153 | s.emitter.GaugeMS("db-list-heartbeats-elapsed-ms", start) 154 | return heartbeats, nil 155 | } 156 | 157 | func (s *store) listHeartbeats() ([]payloads.Heartbeat, error) { 158 | rows, err := s.statements[selectHeartbeatsSQL].Query() 159 | if err != nil { 160 | return nil, err 161 | } 162 | defer ignoreClose(rows) 163 | 164 | var heartbeats []payloads.Heartbeat 165 | for rows.Next() { 166 | var heartbeat payloads.Heartbeat 167 | if err := rows.Scan( 168 | &heartbeat.Self.Address, 169 | &heartbeat.Self.Port, 170 | &heartbeat.NumModules, 171 | &heartbeat.NumVersions, 172 | &heartbeat.Timestamp, 173 | ); err != nil { 174 | return nil, err 175 | } 176 | heartbeats = append(heartbeats, heartbeat) 177 | } 178 | if err := rows.Err(); err != nil { 179 | return nil, err 180 | } 181 | return heartbeats, nil 182 | } 183 | 184 | func ignoreRollback(tx *sql.Tx) { 185 | _ = tx.Rollback() 186 | } 187 | 188 | func (s *store) PurgeProxy(instance netservice.Instance) error { 189 | tx, err := s.db.Begin() 190 | if err != nil { 191 | return err 192 | } 193 | defer ignoreRollback(tx) 194 | 195 | // 1) remove heartbeat for proxy 196 | if _, err := tx.Stmt(s.statements[deleteHeartbeatSQL]).Exec( 197 | instance.Address, 198 | instance.Port, 199 | ); err != nil { 200 | return err 201 | } 202 | 203 | // 2) remove startup configuration for proxy 204 | if _, err := tx.Stmt(s.statements[deleteStartupConfigSQL]).Exec( 205 | instance.Address, 206 | instance.Port, 207 | ); err != nil { 208 | return err 209 | } 210 | 211 | return tx.Commit() 212 | } 213 | -------------------------------------------------------------------------------- /registry/internal/tools/finder/github.go: -------------------------------------------------------------------------------- 1 | package finder 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "regexp" 9 | "strings" 10 | "time" 11 | 12 | "github.com/pkg/errors" 13 | 14 | "gophers.dev/pkgs/ignore" 15 | "gophers.dev/pkgs/loggy" 16 | "gophers.dev/pkgs/semantic" 17 | 18 | "oss.indeed.com/go/modprox/pkg/clients/zips" 19 | ) 20 | 21 | func Github(baseURL string, client *http.Client, proxyClient zips.ProxyClient) Versions { 22 | if baseURL == "" { 23 | baseURL = "https://api.github.com" 24 | } 25 | return &github{ 26 | baseURL: baseURL, 27 | client: client, 28 | log: loggy.New("github-versions"), 29 | proxyClient: proxyClient, 30 | } 31 | } 32 | 33 | type github struct { 34 | baseURL string 35 | client *http.Client 36 | log loggy.Logger 37 | proxyClient zips.ProxyClient 38 | } 39 | 40 | func (g *github) Request(source string) (*Result, error) { 41 | namespace, project, err := g.parseSource(source) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | g.log.Tracef("requesting available versions from the official go proxy") 47 | 48 | tags, err := g.proxyClient.List(source) 49 | if err != nil { 50 | return nil, errors.Wrap(err, "failed to query list of versions from proxy.golang.org") 51 | } 52 | 53 | g.log.Tracef("checking if %s is module-compatible", source) 54 | isModuleCompatible, err := g.isModuleCompatible(tags) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | headURI := g.headURI(namespace, project) 60 | 61 | g.log.Tracef("requesting latest commit from URI: %s", headURI) 62 | 63 | head, err := g.requestHead(headURI, tags, isModuleCompatible) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return &Result{ 69 | Latest: head, 70 | Tags: tags, 71 | }, nil 72 | } 73 | 74 | // isModuleCompatible isn't 100% accurate. It will only return false if the latest semver 75 | // major >= 2 76 | // This is good enough for the usecase intended, which is to decide on adding "+incompatible" 77 | // to version strings. "+incompatible" isn't needed for major versions < 2 78 | func (g *github) isModuleCompatible(versions []semantic.Tag) (bool, error) { 79 | return len(versions) > 0 && strings.Contains(versions[len(versions)-1].Extension, "+incompatible"), nil 80 | } 81 | 82 | func (g *github) requestHead(uri string, tags []semantic.Tag, isModuleCompatible bool) (Head, error) { 83 | response, err := g.client.Get(uri) 84 | if err != nil { 85 | return Head{}, err 86 | } 87 | defer ignore.Drain(response.Body) 88 | 89 | return g.decodeHead(response.Body, tags, isModuleCompatible) 90 | } 91 | 92 | func (g *github) decodeHead(r io.Reader, tags []semantic.Tag, isModuleCompatible bool) (Head, error) { 93 | var gCommit githubCommit 94 | if err := json.NewDecoder(r).Decode(&gCommit); err != nil { 95 | return Head{}, err 96 | } 97 | 98 | custom, err := gCommit.Pseudo(tags, isModuleCompatible) 99 | if err != nil { 100 | return Head{}, err 101 | } 102 | 103 | return Head{ 104 | Commit: gCommit.SHA, 105 | Custom: custom, 106 | }, nil 107 | } 108 | 109 | type githubCommit struct { 110 | SHA string `json:"sha"` 111 | Commit struct { 112 | Author struct { 113 | Date string `json:"date"` 114 | } `json:"author"` 115 | } `json:"commit"` 116 | } 117 | 118 | func (gc githubCommit) Pseudo(tags []semantic.Tag, isModuleCompatible bool) (string, error) { 119 | naked, semver, err := gc.nakedPseudo(tags) 120 | if err != nil { 121 | return "", err 122 | } 123 | 124 | if semver == nil || semver.Major < 2 || isModuleCompatible { 125 | return naked, nil 126 | } 127 | return naked + "+incompatible", nil 128 | } 129 | 130 | func (gc githubCommit) nakedPseudo(tags []semantic.Tag) (string, *semantic.Tag, error) { 131 | ts, err := time.Parse(time.RFC3339, gc.Commit.Author.Date) 132 | if err != nil { 133 | return "", nil, err 134 | } 135 | 136 | date := ts.Format("20060102150405") 137 | shortSHA := gc.SHA[0:12] // what Go does 138 | 139 | if len(tags) == 0 { 140 | return fmt.Sprintf("v0.0.0-%s-%s", date, shortSHA), nil, nil 141 | } 142 | 143 | // tags are guaranteed to be logically reverse-ordered by proxyClient 144 | semver := tags[0] 145 | 146 | if semver.Extension == "pre" { 147 | // TODO should this always be ".0" or do we increment the pre version? 148 | return fmt.Sprintf("%s.0.%s-%s", semver.String(), date, shortSHA), &semver, nil 149 | } 150 | 151 | return fmt.Sprintf("v%d.%d.%d-0.%s-%s", semver.Major, semver.Minor, semver.Patch+1, date, shortSHA), &semver, nil 152 | } 153 | 154 | // -rc is commonly used, but not in the spec 155 | //var semVerRe = regexp.MustCompile(`^v(\d+)(?:\.(\d+)(?:\.(\d+(-pre|-rc)?))?)?$`) 156 | 157 | // only github.com things are supported for now 158 | var githubPkgRe = regexp.MustCompile(`(github\.com)/([[:alnum:]_-]+)/([[:alnum:]_-]+)`) 159 | 160 | func (g *github) headURI(namespace, project string) string { 161 | return fmt.Sprintf( 162 | "%s/repos/%s/%s/commits/HEAD", 163 | g.baseURL, 164 | namespace, 165 | project, 166 | ) 167 | } 168 | 169 | func (g *github) parseSource(source string) (string, string, error) { 170 | groups := githubPkgRe.FindStringSubmatch(source) 171 | if len(groups) != 4 { 172 | return "", "", errors.New("source does not conform to format") 173 | } 174 | 175 | if groups[1] != "github.com" { 176 | return "", "", errors.New("only github.com is currently supported") 177 | } 178 | 179 | return groups[2], groups[3], nil 180 | } 181 | --------------------------------------------------------------------------------