├── .dockerignore ├── logo.png ├── main.go ├── Makefile ├── .gitignore ├── go.mod ├── cli ├── purge.go ├── list.go ├── utils.go ├── main.go └── root.go ├── Dockerfile ├── example └── main.go ├── proxy ├── proxy.go ├── database_redis.go ├── server.go └── cache.go ├── license ├── go.sum └── readme.md /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !proxy 3 | !cli 4 | !main.go 5 | !go.mod 6 | !go.sum 7 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackdumper/npm-cache-proxy/HEAD/logo.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/pkgems/npm-cache-proxy/cli" 5 | ) 6 | 7 | func main() { 8 | cli.Run() 9 | } 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run 2 | run: 3 | go run main.go 4 | 5 | .PHONY: build 6 | build: 7 | gox -output=build/ncp_{{.OS}}_{{.Arch}} 8 | 9 | .PHONY: test 10 | test: 11 | go test ./... 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Build 15 | build/ 16 | 17 | # APFS cache 18 | .DS_Store 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/pkgems/npm-cache-proxy 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/gin-contrib/zap v0.0.0-20190405225521-7c4b822813e7 7 | github.com/gin-gonic/gin v1.3.0 8 | github.com/go-redis/redis v6.15.2+incompatible 9 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 10 | github.com/spf13/cobra v0.0.3 11 | github.com/spf13/pflag v1.0.3 // indirect 12 | go.uber.org/zap v1.9.1 13 | ) 14 | -------------------------------------------------------------------------------- /cli/purge.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | npmproxy "github.com/pkgems/npm-cache-proxy/proxy" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | // start a server 9 | var purgeCmd = &cobra.Command{ 10 | Use: "purge", 11 | Short: "Purge all cached paths", 12 | Run: func(cmd *cobra.Command, args []string) { 13 | proxy := getProxy() 14 | 15 | err := proxy.PurgeCachedPaths(npmproxy.Options{ 16 | DatabasePrefix: persistentOptions.RedisPrefix, 17 | }) 18 | if err != nil { 19 | panic(err) 20 | } 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /cli/list.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | 6 | npmproxy "github.com/pkgems/npm-cache-proxy/proxy" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // start a server 11 | var listCmd = &cobra.Command{ 12 | Use: "list", 13 | Short: "List all cached paths", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | proxy := getProxy() 16 | 17 | metadatas, err := proxy.ListCachedPaths(npmproxy.Options{ 18 | DatabasePrefix: persistentOptions.RedisPrefix, 19 | }) 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | for _, metadata := range metadatas { 25 | fmt.Println(metadata) 26 | } 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /cli/utils.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | ) 7 | 8 | func getEnvString(env string, def string) string { 9 | value := os.Getenv(env) 10 | 11 | if value != "" { 12 | return value 13 | } else { 14 | return def 15 | } 16 | } 17 | 18 | func getEnvInt(env string, def string) int { 19 | value := getEnvString(env, def) 20 | 21 | // TODO: handle error 22 | converted, _ := strconv.Atoi(value) 23 | 24 | return converted 25 | } 26 | 27 | func getEnvBool(env string, def string) bool { 28 | value := getEnvString(env, def) 29 | 30 | // TODO: handle error 31 | converted, _ := strconv.ParseBool(value) 32 | 33 | return converted 34 | } 35 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # === BUILD STAGE === # 2 | FROM golang:1.12-alpine as build 3 | 4 | ARG ACCESS_TOKEN 5 | 6 | RUN apk add --no-cache git 7 | 8 | WORKDIR /srv/app 9 | ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 10 | 11 | COPY go.mod go.sum ./ 12 | RUN go mod download 13 | 14 | COPY . . 15 | RUN go test -v ./... 16 | RUN go build -ldflags="-w -s" -o build 17 | 18 | # === RUN STAGE === # 19 | FROM alpine as run 20 | 21 | RUN apk update \ 22 | && apk upgrade \ 23 | && apk add --no-cache ca-certificates \ 24 | && update-ca-certificates \ 25 | && rm -rf /var/cache/apk/* 26 | 27 | WORKDIR /srv/app 28 | COPY --from=build /srv/app/build /srv/app/build 29 | 30 | ENV LISTEN_ADDRESS 0.0.0.0:8080 31 | ENV GIN_MODE release 32 | 33 | CMD ["/srv/app/build"] 34 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | npmproxy "github.com/pkgems/npm-cache-proxy/proxy" 8 | "github.com/go-redis/redis" 9 | ) 10 | 11 | func main() { 12 | proxy := npmproxy.Proxy{ 13 | Database: npmproxy.DatabaseRedis{ 14 | Client: redis.NewClient(&redis.Options{ 15 | Addr: "localhost:6379", 16 | DB: 0, 17 | Password: "", 18 | }), 19 | }, 20 | HttpClient: &http.Client{}, 21 | } 22 | 23 | proxy.Server(npmproxy.ServerOptions{ 24 | ListenAddress: "localhost:8080", 25 | GetOptions: func() (npmproxy.Options, error) { 26 | return npmproxy.Options{ 27 | DatabasePrefix: "ncp-", 28 | DatabaseExpiration: 1 * time.Hour, 29 | UpstreamAddress: "https://registry.npmjs.org", 30 | }, nil 31 | }, 32 | }).ListenAndServe() 33 | } 34 | -------------------------------------------------------------------------------- /proxy/proxy.go: -------------------------------------------------------------------------------- 1 | // 2 | // Package proxy implements a HTTP caching proxy for Node package registry (NPM). 3 | // See https://github.com/pkgems/npm-cache-proxy/ for more information about proxy. 4 | // 5 | package proxy 6 | 7 | import ( 8 | "net/http" 9 | "time" 10 | ) 11 | 12 | // Proxy is the proxy instance, it contains Database and HttpClient as static options 13 | // and GetOptions as dynamic options provider 14 | type Proxy struct { 15 | Database Database 16 | HttpClient *http.Client 17 | } 18 | 19 | // Options provides dynamic options for Proxy. 20 | // This can be used for namespace separation, 21 | // allowing multiple users use the same proxy instance simultaneously. 22 | type Options struct { 23 | DatabasePrefix string 24 | DatabaseExpiration time.Duration 25 | UpstreamAddress string 26 | } 27 | 28 | // Database provides interface for data storage. 29 | type Database interface { 30 | Get(key string) (string, error) 31 | Set(key string, value string, ttl time.Duration) error 32 | Delete(key string) error 33 | Keys(prefix string) ([]string, error) 34 | Health() error 35 | } 36 | -------------------------------------------------------------------------------- /proxy/database_redis.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/go-redis/redis" 7 | ) 8 | 9 | // DatabaseRedis implements Database interface for Redis database. 10 | type DatabaseRedis struct { 11 | Client *redis.Client 12 | } 13 | 14 | // Get returns data by key. 15 | func (db DatabaseRedis) Get(key string) (string, error) { 16 | return db.Client.Get(key).Result() 17 | } 18 | 19 | // Set stores value identified by key with expiration timeout. 20 | func (db DatabaseRedis) Set(key string, value string, expiration time.Duration) error { 21 | return db.Client.Set(key, value, expiration).Err() 22 | } 23 | 24 | // Delete deletes data by key. 25 | func (db DatabaseRedis) Delete(key string) error { 26 | return db.Client.Del(key).Err() 27 | } 28 | 29 | // Keys returns stored keys filtered by prefix. 30 | func (db DatabaseRedis) Keys(prefix string) ([]string, error) { 31 | return db.Client.Keys(prefix + "*").Result() 32 | } 33 | 34 | // Health returns an error if database connection cannot be estabilished. 35 | func (db DatabaseRedis) Health() error { 36 | return db.Client.Ping().Err() 37 | } 38 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 stackdumper 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cli/main.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | 8 | npmproxy "github.com/pkgems/npm-cache-proxy/proxy" 9 | "github.com/go-redis/redis" 10 | ) 11 | 12 | // global options 13 | var persistentOptions struct { 14 | RedisAddress string 15 | RedisDatabase int 16 | RedisPassword string 17 | RedisPrefix string 18 | } 19 | 20 | // initialize global options 21 | func init() { 22 | rootCmd.PersistentFlags().StringVar(&persistentOptions.RedisAddress, "redis-address", getEnvString("REDIS_ADDRESS", "localhost:6379"), "Redis address") 23 | rootCmd.PersistentFlags().IntVar(&persistentOptions.RedisDatabase, "redis-database", getEnvInt("REDIS_DATABASE", "0"), "Redis database") 24 | rootCmd.PersistentFlags().StringVar(&persistentOptions.RedisPassword, "redis-password", getEnvString("REDIS_PASSWORD", ""), "Redis password") 25 | rootCmd.PersistentFlags().StringVar(&persistentOptions.RedisPrefix, "redis-prefix", getEnvString("REDIS_PREFIX", "ncp-"), "Redis prefix") 26 | } 27 | 28 | func getProxy() *npmproxy.Proxy { 29 | return &npmproxy.Proxy{ 30 | Database: npmproxy.DatabaseRedis{ 31 | Client: redis.NewClient(&redis.Options{ 32 | Addr: persistentOptions.RedisAddress, 33 | DB: persistentOptions.RedisDatabase, 34 | Password: persistentOptions.RedisPassword, 35 | }), 36 | }, 37 | HttpClient: &http.Client{ 38 | Transport: http.DefaultTransport, 39 | }, 40 | } 41 | } 42 | 43 | // Run starts the CLI 44 | func Run() { 45 | rootCmd.AddCommand(listCmd) 46 | rootCmd.AddCommand(purgeCmd) 47 | 48 | if err := rootCmd.Execute(); err != nil { 49 | fmt.Println(err) 50 | os.Exit(1) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cli/root.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | npmproxy "github.com/pkgems/npm-cache-proxy/proxy" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | // start a server 12 | var rootCmd = &cobra.Command{ 13 | Use: "ncp", 14 | Short: "ncp is a fast npm cache proxy that stores data in Redis", 15 | Run: run, 16 | } 17 | 18 | var rootOptions struct { 19 | Silent bool 20 | ListenAddress string 21 | UpstreamAddress string 22 | CacheLimit string 23 | CacheTTL int 24 | } 25 | 26 | func init() { 27 | rootCmd.Flags().BoolVar(&rootOptions.Silent, "silent", getEnvBool("SILENT", "0"), "Disable logging") 28 | rootCmd.Flags().StringVar(&rootOptions.ListenAddress, "listen", getEnvString("LISTEN_ADDRESS", "localhost:8080"), "Address to listen") 29 | rootCmd.Flags().StringVar(&rootOptions.UpstreamAddress, "upstream", getEnvString("UPSTREAM_ADDRESS", "https://registry.npmjs.org"), "Upstream registry address") 30 | rootCmd.Flags().StringVar(&rootOptions.CacheLimit, "cache-limit", getEnvString("CACHE_LIMIT", "0"), "Cached packages count limit") 31 | rootCmd.Flags().IntVar(&rootOptions.CacheTTL, "cache-ttl", getEnvInt("CACHE_TTL", "3600"), "Cache expiration timeout in seconds") 32 | } 33 | 34 | func run(cmd *cobra.Command, args []string) { 35 | proxy := getProxy() 36 | 37 | log.Print("Listening on " + rootOptions.ListenAddress) 38 | 39 | err := proxy.Server(npmproxy.ServerOptions{ 40 | ListenAddress: rootOptions.ListenAddress, 41 | Silent: rootOptions.Silent, 42 | 43 | GetOptions: func() (npmproxy.Options, error) { 44 | return npmproxy.Options{ 45 | DatabasePrefix: persistentOptions.RedisPrefix, 46 | DatabaseExpiration: time.Duration(rootOptions.CacheTTL) * time.Second, 47 | UpstreamAddress: rootOptions.UpstreamAddress, 48 | }, nil 49 | }, 50 | }).ListenAndServe() 51 | 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /proxy/server.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | "strings" 7 | "time" 8 | 9 | ginzap "github.com/gin-contrib/zap" 10 | gin "github.com/gin-gonic/gin" 11 | zap "go.uber.org/zap" 12 | ) 13 | 14 | // ServerOptions provides configuration for Server method 15 | type ServerOptions struct { 16 | ListenAddress string 17 | Silent bool 18 | 19 | GetOptions func() (Options, error) 20 | } 21 | 22 | // Server creates http proxy server 23 | func (proxy Proxy) Server(options ServerOptions) *http.Server { 24 | gin.SetMode("release") 25 | 26 | router := gin.New() 27 | 28 | if options.Silent { 29 | router.Use(gin.Recovery()) 30 | } else { 31 | logger, _ := zap.NewProduction() 32 | router.Use(ginzap.Ginzap(logger, time.RFC3339, true)) 33 | router.Use(ginzap.RecoveryWithZap(logger, true)) 34 | } 35 | 36 | router.GET("/:scope/:name", proxy.getPackageHandler(options)) 37 | router.GET("/:scope", proxy.getPackageHandler(options)) 38 | router.NoRoute(proxy.noRouteHandler(options)) 39 | 40 | return &http.Server{ 41 | Handler: router, 42 | Addr: options.ListenAddress, 43 | } 44 | } 45 | 46 | func (proxy Proxy) getPackageHandler(options ServerOptions) gin.HandlerFunc { 47 | return func(c *gin.Context) { 48 | options, err := options.GetOptions() 49 | 50 | if err != nil { 51 | c.AbortWithError(500, err) 52 | } else { 53 | pkg, err := proxy.GetCachedPath(options, c.Request.URL.Path, c.Request) 54 | 55 | if err != nil { 56 | c.AbortWithError(500, err) 57 | } else { 58 | c.Header("Cache-Control", "public, max-age="+strconv.Itoa(int(options.DatabaseExpiration.Seconds()))) 59 | c.Data(200, "application/json", pkg) 60 | } 61 | } 62 | } 63 | } 64 | 65 | func (proxy Proxy) noRouteHandler(options ServerOptions) gin.HandlerFunc { 66 | tarballHandler := proxy.getPackageHandler(options) 67 | 68 | return func(c *gin.Context) { 69 | if strings.Contains(c.Request.URL.Path, ".tgz") { 70 | // get tarball 71 | tarballHandler(c) 72 | } else if c.Request.URL.Path == "/" { 73 | // get health 74 | err := proxy.Database.Health() 75 | 76 | if err != nil { 77 | c.AbortWithStatusJSON(503, err) 78 | } else { 79 | c.AbortWithStatusJSON(200, gin.H{"ok": true}) 80 | } 81 | } else { 82 | // redirect 83 | options, err := options.GetOptions() 84 | 85 | if err != nil { 86 | c.AbortWithStatusJSON(500, err) 87 | } else { 88 | c.Redirect(http.StatusTemporaryRedirect, options.UpstreamAddress+c.Request.URL.Path) 89 | } 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /proxy/cache.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "compress/gzip" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | // GetCachedPath returns cached upstream response for a given url path. 13 | func (proxy Proxy) GetCachedPath(options Options, path string, request *http.Request) ([]byte, error) { 14 | key := options.DatabasePrefix + path 15 | 16 | // get package from database 17 | pkg, err := proxy.Database.Get(key) 18 | 19 | // either package doesn't exist or there's some other problem 20 | if err != nil { 21 | 22 | // check if error is caused by nonexistend package 23 | // if no, return error 24 | if err.Error() != "redis: nil" { 25 | return nil, err 26 | } 27 | 28 | // error is caused by nonexistent package 29 | // fetch package 30 | req, err := http.NewRequest("GET", options.UpstreamAddress+path, nil) 31 | 32 | req.Header = request.Header 33 | req.Header.Set("Accept-Encoding", "gzip") 34 | 35 | res, err := proxy.HttpClient.Do(req) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | if res.Header.Get("Content-Encoding") == "gzip" { 41 | zr, err := gzip.NewReader(res.Body) 42 | if err != nil { 43 | log.Fatal(err) 44 | } 45 | 46 | res.Body = zr 47 | } 48 | 49 | defer res.Body.Close() 50 | body, err := ioutil.ReadAll(res.Body) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | pkg = string(body) 56 | 57 | // TODO: avoid calling MustCompile every time 58 | // find "dist": "https?://.*/ and replace to "dist": "{localurl}/ 59 | pkg = regexp.MustCompile(`(?U)"tarball":"https?://.*/`).ReplaceAllString(string(body), `"tarball": "http://`+request.Host+"/") 60 | 61 | // save to redis 62 | err = proxy.Database.Set(key, pkg, options.DatabaseExpiration) 63 | if err != nil { 64 | return nil, err 65 | } 66 | } 67 | 68 | return []byte(pkg), nil 69 | } 70 | 71 | // ListCachedPaths returns list of all cached url paths. 72 | func (proxy Proxy) ListCachedPaths(options Options) ([]string, error) { 73 | metadata, err := proxy.Database.Keys(options.DatabasePrefix) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | deprefixedMetadata := make([]string, 0) 79 | for _, record := range metadata { 80 | deprefixedMetadata = append(deprefixedMetadata, strings.Replace(record, options.DatabasePrefix, "", 1)) 81 | } 82 | 83 | return deprefixedMetadata, nil 84 | } 85 | 86 | // PurgeCachedPaths deletes all cached url paths. 87 | func (proxy Proxy) PurgeCachedPaths(options Options) error { 88 | metadata, err := proxy.Database.Keys(options.DatabasePrefix) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | for _, record := range metadata { 94 | err := proxy.Database.Delete(record) 95 | if err != nil { 96 | return err 97 | } 98 | } 99 | 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 h1:AzN37oI0cOS+cougNAV9szl6CVoj2RYwzS3DpUQNtlY= 3 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 4 | github.com/gin-contrib/zap v0.0.0-20190405225521-7c4b822813e7 h1:v6rNWmMnVDBVMc1pUUSCTobu2p4BukiPMwoj0pLqBhA= 5 | github.com/gin-contrib/zap v0.0.0-20190405225521-7c4b822813e7/go.mod h1:pQKeeey3PeRN2SbZe1jWiIkTJkylO9hL1K0Hf4Wbtt4= 6 | github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs= 7 | github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= 8 | github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= 9 | github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 10 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 11 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 12 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 13 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 14 | github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 15 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 16 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 17 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 18 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 19 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= 22 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 23 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 24 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 25 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 26 | github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 h1:EICbibRW4JNKMcY+LsWmuwob+CRS1BmdRdjphAm9mH4= 27 | github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 28 | go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= 29 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 30 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 31 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 32 | go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= 33 | go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 34 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 35 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 36 | golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 37 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 38 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 39 | gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= 40 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 41 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 42 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 43 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |

npm-cache-proxy

5 | 6 | 7 | Current Release 8 | 9 | 10 | CI Build 11 | 12 | 13 | Licence 14 | 15 |
16 | 17 |
18 |
19 | 20 | ## Introduction 21 | 22 | #### ⚡️ Performance 23 | NCP is a tiny but very fast caching proxy written in Go. It uses Redis for data storage, which in combination with the speed of Go makes it incredibly fast. NCP is well-optimized and can be run on almost any platform, so if you have a Raspberry Pi, you can install NCP as your local cache there. 24 | 25 | #### ✨ Modularity 26 | NCP is modular. Now it has only one database adapter which is Redis. If you need support for any other database, feel free to open an issue or implement it [on your own](https://github.com/pkgems/npm-cache-proxy/blob/7c8b90ff6ba0656f60e3de915b9fb4eaabfb467b/proxy/proxy.go#L29) and then open a pull request (_bonus points_). 27 | 28 | #### 💡 Simplicity 29 | NCP is very simple. It just proxies requests to an upstream registry, caches response and returns cached response for next requests to the same package. Cached data are stored in Redis with an original request URL as a key. 30 | 31 | 32 |
33 | 34 | 35 | ## Installation 36 | NCP binaries for different paltforms can be downloaded can be downloaded on the [Releases](https://github.com/pkgems/npm-cache-proxy/releases) page. Also, Docker image is provided on [Docker Hub](https://cloud.docker.com/u/pkgems/repository/docker/pkgems/npm-cache-proxy). 37 | 38 | #### 💫 Quick Start 39 | The quickies way to get started with NCP is to use Docker. 40 | 41 | ```bash 42 | # run proxy inside of docker container in background 43 | docker run -e REDIS_ADDRESS=host.docker.internal:6379 -p 8080:8080 -it -d pkgems/npm-cache-proxy 44 | 45 | # configure npm to use caching proxy as registry 46 | npm config set registry http://localhost:8080 47 | ``` 48 | 49 |
50 | 51 | ## CLI 52 | NCP provides command line interface for interaction with a cached data. 53 | 54 |
55 | Options 56 | 57 | | Options | Env | Default | Description | 58 | | ----------------------------- | ------------------ | ---------------------------- | ----------------------------------- | 59 | | `--listen
` | `LISTEN_ADDRESS` | `locahost:8080` | Address to listen | 60 | | `--upstream
` | `UPSTREAM_ADDRESS` | `https://registry.npmjs.org` | Upstream registry address | 61 | | `--silent
` | `SILENT` | `0` | Disable logs | 62 | | `--cache-limit ` | `CACHE_LIMIT` | - | Cached packages count limit | 63 | | `--cache-ttl ` | `CACHE_TTL` | `3600` | Cache expiration timeout in seconds | 64 | | `--redis-address
` | `REDIS_ADDRESS` | `http://localhost:6379` | Redis address | 65 | | `--redis-database ` | `REDIS_DATABASE` | `0` | Redis database | 66 | | `--redis-password ` | `REDIS_PASSWORD` | - | Redis password | 67 | | `--redis-prefix ` | `REDIS_PREFIX` | `ncp-` | Redis keys prefix | 68 | 69 |
70 | 71 | #### `ncp` 72 | Start NCP server. 73 | 74 | #### `ncp list` 75 | List cached url paths. 76 | 77 | #### `ncp purge` 78 | Purge cached url paths. 79 | 80 | 81 |
82 | 83 | 84 | ## Benchmark 85 | Benchmark is run on Macbook Pro 15″ 2017, Intel Core i7-7700HQ. 86 | 87 | #### 1️⃣ 1 process 88 | 89 | ```bash 90 | # GOMAXPROCS=1 ncp --silent 91 | 92 | $ go-wrk -c 100 -d 6 http://localhost:8080/tiny-tarball 93 | Running 6s test @ http://localhost:8080/tiny-tarball 94 | 100 goroutine(s) running concurrently 95 | 96 | 70755 requests in 5.998378587s, 91.16MB read 97 | 98 | Requests/sec: 11795.69 99 | Transfer/sec: 15.20MB 100 | Avg Req Time: 8.477674ms 101 | Fastest Request: 947.743µs 102 | Slowest Request: 815.787409ms 103 | Number of Errors: 0 104 | ``` 105 | 106 | #### ♾ unlimited processes 107 | 108 | ```bash 109 | # ncp --silent 110 | 111 | $ go-wrk -c 100 -d 6 http://localhost:8080/tiny-tarball 112 | Running 6s test @ http://localhost:8080/tiny-tarball 113 | 100 goroutine(s) running concurrently 114 | 115 | 115674 requests in 5.98485984s, 149.04MB read 116 | 117 | Requests/sec: 19327.77 118 | Transfer/sec: 24.90MB 119 | Avg Req Time: 5.173902ms 120 | Fastest Request: 273.015µs 121 | Slowest Request: 34.777963ms 122 | Number of Errors: 0 123 | ``` 124 | 125 | 126 |
127 | 128 | 129 | ## Programmatic Usage 130 | NCP provides `proxy` go package that can be used programmatically. Docs are available on [godoc.org](https://godoc.org/github.com/pkgems/npm-cache-proxy/proxy). 131 | 132 | #### 🤖 Example 133 | ```golang 134 | package main 135 | 136 | import ( 137 | "net/http" 138 | "time" 139 | 140 | npmproxy "github.com/pkgems/npm-cache-proxy/proxy" 141 | "github.com/go-redis/redis" 142 | ) 143 | 144 | func main() { 145 | // create proxy 146 | proxy := npmproxy.Proxy{ 147 | // use redis as database 148 | Database: npmproxy.DatabaseRedis{ 149 | // see github.com/go-redis/redis 150 | Client: redis.NewClient(&redis.Options{ 151 | Addr: "localhost:6379", 152 | }), 153 | }, 154 | 155 | // reuse connections 156 | HttpClient: &http.Client{}, 157 | } 158 | 159 | // create and start server 160 | proxy.Server(npmproxy.ServerOptions{ 161 | ListenAddress: "localhost:8080", 162 | 163 | // allow fetching options dynamically on each request 164 | GetOptions: func() (npmproxy.Options, error) { 165 | return npmproxy.Options{ 166 | DatabasePrefix: "ncp-", 167 | DatabaseExpiration: 1 * time.Hour, 168 | UpstreamAddress: "https://registry.npmjs.org", 169 | }, nil 170 | }, 171 | }).ListenAndServe() 172 | } 173 | ``` 174 | 175 | 176 |
177 | 178 | 179 | ## License 180 | [MIT](./license) 181 | --------------------------------------------------------------------------------