├── .dockerignore ├── .editorconfig ├── .gitignore ├── Dockerfile ├── README.md ├── go.mod ├── go.sum ├── handlers.go ├── handlers_test.go ├── logger ├── logger.go └── logger_test.go ├── main.go ├── pb ├── dwarf.pb.go ├── dwarf.proto └── gen.sh ├── start-dev.sh └── storage ├── redis.go ├── redis_test.go └── storage.go /.dockerignore: -------------------------------------------------------------------------------- 1 | dwarf 2 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = LF 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | max_line_length = 80 11 | 12 | [*.sh] 13 | indent_style = tab 14 | 15 | [*.go] 16 | indent_style = tab 17 | 18 | [Makefile] 19 | indent_style=tab 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | vendor/ 3 | dwarf 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.11.4-alpine 2 | 3 | WORKDIR /src/github.com/LevInterctive/dwarf 4 | ADD . /src/github.com/LevInterctive/dwarf 5 | 6 | RUN apk add bash ca-certificates git gcc g++ libc-dev 7 | 8 | RUN go build -o bin/dwarf 9 | CMD ["./bin/dwarf"] 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dwarf 2 | 3 | A high-throughput URL shortener microservice built with Go. 4 | 5 | * gRPC for communication 6 | * Redis store out of the box 7 | * Fast & simple 8 | 9 | # Usage 10 | 11 | See [start-dev.sh](start-dev.sh) for a complete list of available environmental variables. To use 12 | in a docker container, we have an image [here](https://hub.docker.com/r/levinteractive/dwarf/tags/). 13 | 14 | #### GET `/{short-hash}` -> 301 redirection 15 | 16 | Dwarf will deliver a 301 redirection to the destination URL or redirect to the fallback 17 | URL specified with [`NOTFOUND_REDIRECT_URL`](start-dev.sh). 18 | 19 | #### Creating short links 20 | 21 | You must communicate with dwarf via gRPC in order to generate new shortened URLs. 22 | 23 | ```proto 24 | service Dwarf { 25 | rpc Create(CreateRequest) returns (CreateResponse) {} 26 | } 27 | 28 | message CreateRequest { 29 | repeated string urls = 1; 30 | } 31 | 32 | message CreateResponse { 33 | repeated string urls = 2; 34 | } 35 | ``` 36 | 37 | Your response will return a set of shortened urls in the same order that they were sent: 38 | 39 | ```json 40 | // -> Request 41 | { "urls": ["http://long-url.com/1", "http://long-url.com/2"] } 42 | 43 | // -> Response 44 | { "urls": ["http://sh.ort/Mp", "http://sh.ort/uJ"] } 45 | ``` 46 | 47 | #### A dwarf gRPC client written with node.js 48 | 49 | To generate short urls, use a gRPC client such as this [node client](https://github.com/LevInteractive/dwarf-client-javascript). 50 | 51 | 52 | # Development 53 | 54 | * [Protobuf/gRPC is required.](https://grpc.io/docs/quickstart/go.html) 55 | * Go & dep 56 | 57 | 58 | ### Redis Store 59 | 60 | Spin up an instance of redis with: 61 | 62 | ```bash 63 | docker run -p "6379:6379" --rm --name dwarf-redis redis:4-alpine 64 | ``` 65 | 66 | ### Testing 67 | 68 | `go test github.com/LevInteractive/dwarf/ -v` 69 | 70 | Note that the tests rely on a running redis instance. 71 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/LevInteractive/dwarf 2 | 3 | require ( 4 | github.com/go-redis/redis v6.14.1+incompatible 5 | github.com/golang/protobuf v1.2.0 6 | github.com/gorilla/context v1.1.1 7 | github.com/gorilla/mux v1.6.2 8 | golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 9 | golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611 10 | golang.org/x/text v0.3.0 11 | google.golang.org/genproto v0.0.0-20180928223349-c7e5094acea1 12 | google.golang.org/grpc v1.15.0 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 3 | github.com/go-redis/redis v6.14.1+incompatible h1:kSJohAREGMr344uMa8PzuIg5OU6ylCbyDkWkkNOfEik= 4 | github.com/go-redis/redis v6.14.1+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 5 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 6 | github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= 7 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 8 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 9 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 10 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 11 | github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= 12 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 13 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 14 | golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 15 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 16 | golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 h1:dgd4x4kJt7G4k4m93AYLzM8Ni6h2qLTfh9n9vXJT3/0= 17 | golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 18 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 19 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 20 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 21 | golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 22 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 23 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 24 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 25 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 26 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 27 | google.golang.org/genproto v0.0.0-20180928223349-c7e5094acea1 h1:y+7ra8GA+PNVmm+pBIWTKIK+YaBeRiGH+3544JQqm58= 28 | google.golang.org/genproto v0.0.0-20180928223349-c7e5094acea1/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 29 | google.golang.org/grpc v1.15.0 h1:Az/KuahOM4NAidTEuJCv/RonAA7rYsTPkqXVjr+8OOw= 30 | google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= 31 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 32 | -------------------------------------------------------------------------------- /handlers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | 9 | "github.com/LevInteractive/dwarf/logger" 10 | "github.com/LevInteractive/dwarf/pb" 11 | "github.com/LevInteractive/dwarf/storage" 12 | "github.com/gorilla/mux" 13 | ) 14 | 15 | // LoadResult represents what's returned from the Load method. 16 | type LoadResult struct { 17 | LongURL string `json:"longUrls"` 18 | } 19 | 20 | // CreateServer for the gRPC server. 21 | type CreateServer struct { 22 | Store storage.IStorage 23 | } 24 | 25 | // Create new short url given a set of long urls via gRPC. 26 | func (s *CreateServer) Create(ctx context.Context, req *pb.CreateRequest) (*pb.CreateResponse, error) { 27 | res := &pb.CreateResponse{} 28 | 29 | // Validate incoming urls. Don't allow any non-legit URL's. 30 | for _, u := range req.Urls { 31 | _, err := url.ParseRequestURI(u) 32 | if err != nil { 33 | logger.Error("CreateHandler.error: recieved bad payload: %v", err) 34 | return res, err 35 | } 36 | } 37 | 38 | // Create/get the short code for the URL and build. 39 | for _, u := range req.Urls { 40 | code, err := s.Store.Save(u) 41 | if err != nil { 42 | logger.Error("CreateHandler.error: failed to shorten url: %v", err) 43 | return res, err 44 | } 45 | 46 | res.Urls = append( 47 | res.Urls, 48 | fmt.Sprintf("%s/%s", BaseURL, code), 49 | ) 50 | } 51 | 52 | return res, nil 53 | } 54 | 55 | // LookupHandler for adding a new URL to the store via HTTP. 56 | func LookupHandler(store storage.IStorage) func(http.ResponseWriter, *http.Request) { 57 | fnc := func(w http.ResponseWriter, r *http.Request) { 58 | vars := mux.Vars(r) 59 | code := vars["hash"] 60 | fullURL, err := store.Load(code) 61 | 62 | if err == storage.ErrNotFound { 63 | logger.Error("LookupHander.warn: could not find url with code %s", code) 64 | http.Redirect(w, r, NotFoundURL, 301) 65 | return 66 | } 67 | 68 | if err != nil { 69 | logger.Error("LookupHandler.error: receieved error while looking up url: %v", err) 70 | http.Redirect(w, r, NotFoundURL, 301) 71 | return 72 | } 73 | 74 | http.Redirect(w, r, fullURL, 301) 75 | } 76 | return fnc 77 | } 78 | -------------------------------------------------------------------------------- /handlers_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "testing" 7 | 8 | "github.com/LevInteractive/dwarf/pb" 9 | "github.com/LevInteractive/dwarf/storage" 10 | "github.com/go-redis/redis" 11 | ) 12 | 13 | func TestGRPCCreate(t *testing.T) { 14 | store := storage.Redis{ 15 | CharFloor: 2, 16 | Conn: &redis.Options{ 17 | Addr: "localhost:6379", 18 | Password: "", 19 | DB: 2, 20 | }, 21 | } 22 | 23 | store.Init() 24 | 25 | server := &CreateServer{ 26 | Store: store, 27 | } 28 | 29 | tests := []struct { 30 | req *pb.CreateRequest 31 | value int 32 | }{ 33 | { 34 | req: &pb.CreateRequest{Urls: []string{"http://hello.com"}}, 35 | value: 1, 36 | }, 37 | { 38 | req: &pb.CreateRequest{Urls: []string{"http://a.com", "http://foo.com"}}, 39 | value: 2, 40 | }, 41 | } 42 | 43 | for _, tt := range tests { 44 | resp, err := server.Create(context.Background(), tt.req) 45 | if err != nil { 46 | log.Fatal(err) 47 | } 48 | 49 | if len(resp.Urls) != tt.value { 50 | t.Fatalf("Length of response should be 1. Got %v", len(tt.req.Urls)) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | var infoLogger *log.Logger 9 | var errorLogger *log.Logger 10 | 11 | func init() { 12 | infoLogger = log.New(os.Stdout, "", 0) 13 | errorLogger = log.New(os.Stderr, "", 0) 14 | } 15 | 16 | // Info message into stdout. 17 | func Info(msg string, v ...interface{}) *log.Logger { 18 | infoLogger.Printf(msg, v...) 19 | return infoLogger 20 | } 21 | 22 | // Error message into stderr. 23 | func Error(msg string, v ...interface{}) *log.Logger { 24 | errorLogger.Printf(msg, v...) 25 | return errorLogger 26 | } 27 | -------------------------------------------------------------------------------- /logger/logger_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import "testing" 4 | 5 | func TestLogger(t *testing.T) { 6 | Error("Hi, I'm an error") 7 | Info("Hi, I'm an info") 8 | Error("Hi, I'm an error") 9 | Info("Hi, I'm an info") 10 | t.Log("successfully logged to stdout and stderr") 11 | } 12 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "net/http" 7 | "os" 8 | "strconv" 9 | 10 | "github.com/LevInteractive/dwarf/logger" 11 | "github.com/LevInteractive/dwarf/pb" 12 | "github.com/LevInteractive/dwarf/storage" 13 | "github.com/go-redis/redis" 14 | "github.com/gorilla/mux" 15 | "google.golang.org/grpc" 16 | ) 17 | 18 | // BaseURL is the base URL of the application. Short codes are appended on to it 19 | // to ultimately construct the full short url. 20 | var BaseURL string 21 | 22 | // NotFoundURL is the URL used when Dwarf needs to redirect if a short code 23 | // doesn't exist. We also redirect to this if the user lands on the root (/) of 24 | // Dwarf. 25 | var NotFoundURL string 26 | 27 | func init() { 28 | BaseURL = GetEnv("APP_BASE_URL", "https://example.com") 29 | NotFoundURL = GetEnv("NOTFOUND_REDIRECT_URL", "https://google.com") 30 | } 31 | 32 | // GetEnv grabs env with a fallback 33 | func GetEnv(key, fallback string) string { 34 | if value, ok := os.LookupEnv(key); ok { 35 | return value 36 | } 37 | return fallback 38 | } 39 | 40 | func main() { 41 | // Configure the Redis store. Redis is all we have now so eh. 42 | db, err := strconv.Atoi(GetEnv("REDIS_DB", "0")) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | 47 | charFloor, err := strconv.Atoi(GetEnv("CHAR_FLOOR", "3")) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | 52 | store := storage.Redis{ 53 | CharFloor: charFloor, 54 | Conn: &redis.Options{ 55 | Addr: GetEnv("REDIS_SERVER", "localhost:6379"), 56 | Password: GetEnv("REDIS_PASS", ""), 57 | DB: db, 58 | }, 59 | } 60 | 61 | store.Init() 62 | 63 | go setupHTTPDiscovery(store) 64 | setupGrpcDiscovery(store) 65 | } 66 | 67 | func setupGrpcDiscovery(store storage.IStorage) { 68 | lis, err := net.Listen("tcp", GetEnv("GRPC_PORT", ":8001")) 69 | if err != nil { 70 | log.Fatalf("Failed to listen: %v", err) 71 | } 72 | 73 | s := grpc.NewServer() 74 | 75 | pb.RegisterDwarfServer(s, &CreateServer{ 76 | Store: store, 77 | }) 78 | 79 | logger.Info( 80 | "Dwarf's gRPC server is listening here -> %s", 81 | GetEnv("GRPC_PORT", ":8001"), 82 | ) 83 | 84 | if err := s.Serve(lis); err != nil { 85 | log.Fatalf("Failed to serve: %v", err) 86 | } 87 | } 88 | 89 | func setupHTTPDiscovery(store storage.IStorage) { 90 | appPort := GetEnv("APP_PORT", ":8000") 91 | 92 | r := mux.NewRouter() 93 | r.HandleFunc("/{hash:.*}", LookupHandler(store)).Methods("GET") 94 | http.Handle("/", r) 95 | logger.Info( 96 | "Dwarf's public HTTP server is listening here -> %s and exposed here -> %s", 97 | appPort, 98 | BaseURL, 99 | ) 100 | log.Fatal(http.ListenAndServe(appPort, r)) 101 | } 102 | -------------------------------------------------------------------------------- /pb/dwarf.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: dwarf.proto 3 | 4 | package pb 5 | 6 | import ( 7 | fmt "fmt" 8 | proto "github.com/golang/protobuf/proto" 9 | context "golang.org/x/net/context" 10 | grpc "google.golang.org/grpc" 11 | math "math" 12 | ) 13 | 14 | // Reference imports to suppress errors if they are not otherwise used. 15 | var _ = proto.Marshal 16 | var _ = fmt.Errorf 17 | var _ = math.Inf 18 | 19 | // This is a compile-time assertion to ensure that this generated file 20 | // is compatible with the proto package it is being compiled against. 21 | // A compilation error at this line likely means your copy of the 22 | // proto package needs to be updated. 23 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 24 | 25 | type CreateRequest struct { 26 | Urls []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"` 27 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 28 | XXX_unrecognized []byte `json:"-"` 29 | XXX_sizecache int32 `json:"-"` 30 | } 31 | 32 | func (m *CreateRequest) Reset() { *m = CreateRequest{} } 33 | func (m *CreateRequest) String() string { return proto.CompactTextString(m) } 34 | func (*CreateRequest) ProtoMessage() {} 35 | func (*CreateRequest) Descriptor() ([]byte, []int) { 36 | return fileDescriptor_287e70e6da97d931, []int{0} 37 | } 38 | 39 | func (m *CreateRequest) XXX_Unmarshal(b []byte) error { 40 | return xxx_messageInfo_CreateRequest.Unmarshal(m, b) 41 | } 42 | func (m *CreateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 43 | return xxx_messageInfo_CreateRequest.Marshal(b, m, deterministic) 44 | } 45 | func (m *CreateRequest) XXX_Merge(src proto.Message) { 46 | xxx_messageInfo_CreateRequest.Merge(m, src) 47 | } 48 | func (m *CreateRequest) XXX_Size() int { 49 | return xxx_messageInfo_CreateRequest.Size(m) 50 | } 51 | func (m *CreateRequest) XXX_DiscardUnknown() { 52 | xxx_messageInfo_CreateRequest.DiscardUnknown(m) 53 | } 54 | 55 | var xxx_messageInfo_CreateRequest proto.InternalMessageInfo 56 | 57 | func (m *CreateRequest) GetUrls() []string { 58 | if m != nil { 59 | return m.Urls 60 | } 61 | return nil 62 | } 63 | 64 | type CreateResponse struct { 65 | Urls []string `protobuf:"bytes,2,rep,name=urls,proto3" json:"urls,omitempty"` 66 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 67 | XXX_unrecognized []byte `json:"-"` 68 | XXX_sizecache int32 `json:"-"` 69 | } 70 | 71 | func (m *CreateResponse) Reset() { *m = CreateResponse{} } 72 | func (m *CreateResponse) String() string { return proto.CompactTextString(m) } 73 | func (*CreateResponse) ProtoMessage() {} 74 | func (*CreateResponse) Descriptor() ([]byte, []int) { 75 | return fileDescriptor_287e70e6da97d931, []int{1} 76 | } 77 | 78 | func (m *CreateResponse) XXX_Unmarshal(b []byte) error { 79 | return xxx_messageInfo_CreateResponse.Unmarshal(m, b) 80 | } 81 | func (m *CreateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 82 | return xxx_messageInfo_CreateResponse.Marshal(b, m, deterministic) 83 | } 84 | func (m *CreateResponse) XXX_Merge(src proto.Message) { 85 | xxx_messageInfo_CreateResponse.Merge(m, src) 86 | } 87 | func (m *CreateResponse) XXX_Size() int { 88 | return xxx_messageInfo_CreateResponse.Size(m) 89 | } 90 | func (m *CreateResponse) XXX_DiscardUnknown() { 91 | xxx_messageInfo_CreateResponse.DiscardUnknown(m) 92 | } 93 | 94 | var xxx_messageInfo_CreateResponse proto.InternalMessageInfo 95 | 96 | func (m *CreateResponse) GetUrls() []string { 97 | if m != nil { 98 | return m.Urls 99 | } 100 | return nil 101 | } 102 | 103 | func init() { 104 | proto.RegisterType((*CreateRequest)(nil), "pb.CreateRequest") 105 | proto.RegisterType((*CreateResponse)(nil), "pb.CreateResponse") 106 | } 107 | 108 | func init() { proto.RegisterFile("dwarf.proto", fileDescriptor_287e70e6da97d931) } 109 | 110 | var fileDescriptor_287e70e6da97d931 = []byte{ 111 | // 127 bytes of a gzipped FileDescriptorProto 112 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4e, 0x29, 0x4f, 0x2c, 113 | 0x4a, 0xd3, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2a, 0x48, 0x52, 0x52, 0xe6, 0xe2, 0x75, 114 | 0x2e, 0x4a, 0x4d, 0x2c, 0x49, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x12, 0xe2, 0x62, 115 | 0x29, 0x2d, 0xca, 0x29, 0x96, 0x60, 0x54, 0x60, 0xd6, 0xe0, 0x0c, 0x02, 0xb3, 0x95, 0x54, 0xb8, 116 | 0xf8, 0x60, 0x8a, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0xe1, 0xaa, 0x98, 0x10, 0xaa, 0x8c, 0xac, 117 | 0xb8, 0x58, 0x5d, 0x40, 0xa6, 0x0b, 0x19, 0x72, 0xb1, 0x41, 0x94, 0x0b, 0x09, 0xea, 0x15, 0x24, 118 | 0xe9, 0xa1, 0x98, 0x2f, 0x25, 0x84, 0x2c, 0x04, 0x31, 0x4d, 0x89, 0x21, 0x89, 0x0d, 0xec, 0x22, 119 | 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x16, 0xd8, 0x31, 0x50, 0xa0, 0x00, 0x00, 0x00, 120 | } 121 | 122 | // Reference imports to suppress errors if they are not otherwise used. 123 | var _ context.Context 124 | var _ grpc.ClientConn 125 | 126 | // This is a compile-time assertion to ensure that this generated file 127 | // is compatible with the grpc package it is being compiled against. 128 | const _ = grpc.SupportPackageIsVersion4 129 | 130 | // DwarfClient is the client API for Dwarf service. 131 | // 132 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. 133 | type DwarfClient interface { 134 | Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateResponse, error) 135 | } 136 | 137 | type dwarfClient struct { 138 | cc *grpc.ClientConn 139 | } 140 | 141 | func NewDwarfClient(cc *grpc.ClientConn) DwarfClient { 142 | return &dwarfClient{cc} 143 | } 144 | 145 | func (c *dwarfClient) Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateResponse, error) { 146 | out := new(CreateResponse) 147 | err := c.cc.Invoke(ctx, "/pb.Dwarf/Create", in, out, opts...) 148 | if err != nil { 149 | return nil, err 150 | } 151 | return out, nil 152 | } 153 | 154 | // DwarfServer is the server API for Dwarf service. 155 | type DwarfServer interface { 156 | Create(context.Context, *CreateRequest) (*CreateResponse, error) 157 | } 158 | 159 | func RegisterDwarfServer(s *grpc.Server, srv DwarfServer) { 160 | s.RegisterService(&_Dwarf_serviceDesc, srv) 161 | } 162 | 163 | func _Dwarf_Create_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 164 | in := new(CreateRequest) 165 | if err := dec(in); err != nil { 166 | return nil, err 167 | } 168 | if interceptor == nil { 169 | return srv.(DwarfServer).Create(ctx, in) 170 | } 171 | info := &grpc.UnaryServerInfo{ 172 | Server: srv, 173 | FullMethod: "/pb.Dwarf/Create", 174 | } 175 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 176 | return srv.(DwarfServer).Create(ctx, req.(*CreateRequest)) 177 | } 178 | return interceptor(ctx, in, info, handler) 179 | } 180 | 181 | var _Dwarf_serviceDesc = grpc.ServiceDesc{ 182 | ServiceName: "pb.Dwarf", 183 | HandlerType: (*DwarfServer)(nil), 184 | Methods: []grpc.MethodDesc{ 185 | { 186 | MethodName: "Create", 187 | Handler: _Dwarf_Create_Handler, 188 | }, 189 | }, 190 | Streams: []grpc.StreamDesc{}, 191 | Metadata: "dwarf.proto", 192 | } 193 | -------------------------------------------------------------------------------- /pb/dwarf.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | service Dwarf { 5 | rpc Create(CreateRequest) returns (CreateResponse) {} 6 | } 7 | 8 | message CreateRequest { 9 | repeated string urls = 1; 10 | } 11 | 12 | message CreateResponse { 13 | repeated string urls = 2; 14 | } 15 | -------------------------------------------------------------------------------- /pb/gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | protoc -I . --go_out=plugins=grpc:. ./*.proto 4 | -------------------------------------------------------------------------------- /start-dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | go build 3 | 4 | # Redis Credentials 5 | export REDIS_DB="0" 6 | export REDIS_SERVER="localhost:32768" 7 | export REDIS_PASS="" 8 | 9 | # Port for the public application to listen. 10 | export APP_PORT=":8000" 11 | 12 | # Port for the internal gRPC server to listen. 13 | export GRPC_PORT=":8001" 14 | 15 | # The starting point for the short code algorithm. 16 | export CHAR_FLOOR="2" 17 | 18 | # URL to redirect to if the link isn't found. 19 | export NOTFOUND_REDIRECT_URL="https://github.com" 20 | 21 | # Full URL for the public dwarf app to run. 22 | # No trailing slash. 23 | export APP_BASE_URL="http://localhost:8000" 24 | 25 | ./dwarf 26 | -------------------------------------------------------------------------------- /storage/redis.go: -------------------------------------------------------------------------------- 1 | // Redis Store Strategy: 2 | // 3 | // Will store 2 keys per URL: 4 | // 5 | // [prefix]:code:[fullURL] = [code] 6 | // [prefix]:url:[code] = [fullURL] 7 | // 8 | // Using this we can lookup both ways very efficiently. 9 | 10 | package storage 11 | 12 | import ( 13 | "fmt" 14 | "log" 15 | 16 | "github.com/LevInteractive/dwarf/logger" 17 | "github.com/go-redis/redis" 18 | ) 19 | 20 | const prefix = "dwarf" 21 | 22 | // Redis storage. 23 | type Redis struct { 24 | CharFloor int 25 | Client *redis.Client 26 | Conn *redis.Options 27 | } 28 | 29 | // Init the connection to redis. 30 | func (s *Redis) Init() { 31 | s.Client = redis.NewClient(s.Conn) 32 | _, err := s.Client.Ping().Result() 33 | 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | } 38 | 39 | // Save the url in store with a new code if it isn't already there. 40 | func (s Redis) Save(u string) (string, error) { 41 | codehash := fmt.Sprintf("%s:code:%s", prefix, u) 42 | existingCode, err := s.Client.Get(codehash).Result() 43 | if existingCode != "" { 44 | logger.Info("redis.Save.info: didn't need to create new - existed: %s with code %s", u, existingCode) 45 | return existingCode, nil 46 | } 47 | 48 | code, err := discover(s.Client, s.CharFloor) 49 | if err != nil { 50 | return "", err 51 | } 52 | 53 | err = set(s.Client, u, code) 54 | 55 | if err != nil { 56 | return "", err 57 | } 58 | 59 | logger.Info("redis.Save.info: created new short url for %s / %s", u, code) 60 | return code, nil 61 | } 62 | 63 | // Load will lookup the code and return the full URL. 64 | func (s Redis) Load(code string) (string, error) { 65 | hash := fmt.Sprintf("%s:url:%s", prefix, code) 66 | fullURL, err := s.Client.Do("get", hash).String() 67 | 68 | if err == redis.Nil { 69 | return "", ErrNotFound 70 | } else if err != nil { 71 | logger.Error("redis.Load.error: had redis error %v", err) 72 | return "", err 73 | } 74 | 75 | logger.Info("redis.Load.info: loaded url %s", fullURL) 76 | 77 | return fullURL, nil 78 | } 79 | 80 | // Discover a truly unqiue key. 81 | func discover(c *redis.Client, n int) (string, error) { 82 | code := GenCode(n) 83 | hash := fmt.Sprintf("%s:url:%s", prefix, code) 84 | exists := c.Exists(hash).Val() 85 | 86 | if exists == 0 { 87 | return code, nil 88 | } 89 | 90 | logger.Info("redis.discover.info: had key collision, incrementing 1 char and looking again") 91 | return discover(c, n+1) 92 | } 93 | 94 | // Set each of our key hashes. 95 | func set(c *redis.Client, fullURL string, code string) error { 96 | codehash := fmt.Sprintf("%s:code:%s", prefix, fullURL) 97 | urlhash := fmt.Sprintf("%s:url:%s", prefix, code) 98 | if err := c.Set(codehash, code, 0).Err(); err != nil { 99 | return err 100 | } 101 | if err := c.Set(urlhash, fullURL, 0).Err(); err != nil { 102 | logger.Error("redis.set.error: hit error with setting url hash. rolling back previous codehash. %v", err) 103 | c.Del(codehash) 104 | return err 105 | } 106 | 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /storage/redis_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/go-redis/redis" 8 | ) 9 | 10 | func inArray(val string, array []string) bool { 11 | for i := range array { 12 | if array[i] == val { 13 | return true 14 | } 15 | } 16 | return false 17 | } 18 | 19 | func TestCreation(t *testing.T) { 20 | store := Redis{ 21 | CharFloor: 2, 22 | Conn: &redis.Options{ 23 | Addr: "localhost:6379", 24 | Password: "", 25 | DB: 2, 26 | }, 27 | } 28 | 29 | store.Init() 30 | 31 | var codes []string 32 | 33 | // Should create new without dupes. 34 | for i := 1; i <= 1000; i++ { 35 | code, err := store.Save(fmt.Sprintf("http://google.com/%d", i)) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | if inArray(code, codes) != false { 40 | t.Fatalf("there was a duplicate code made: %s", code) 41 | } 42 | codes = append(codes, code) 43 | } 44 | 45 | // Should retrieve existing codes. 46 | for j := 1; j <= 1000; j++ { 47 | code, err := store.Save(fmt.Sprintf("http://google.com/%d", j)) 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | if inArray(code, codes) == false { 52 | t.Fatalf("code should have already existed: %s", code) 53 | } 54 | } 55 | store.Client.FlushDB() 56 | } 57 | 58 | func TestLoad(t *testing.T) { 59 | store := Redis{ 60 | CharFloor: 2, 61 | Conn: &redis.Options{ 62 | Addr: "localhost:6379", 63 | Password: "", 64 | DB: 2, 65 | }, 66 | } 67 | 68 | store.Init() 69 | 70 | u := "http://google.com/" 71 | 72 | code, err := store.Save(u) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | 77 | ru, err := store.Load(code) 78 | store.Client.FlushDB() 79 | 80 | if err != nil { 81 | t.Fatal(err) 82 | } 83 | 84 | if ru != u { 85 | t.Fatalf("the url saved was not the same as the one returned %s / %s", u, ru) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | "time" 7 | ) 8 | 9 | // IStorage defines the interface for storages to implement. Any store that is 10 | // implemented must conform this. 11 | type IStorage interface { 12 | Save(string) (string, error) 13 | Load(string) (string, error) 14 | } 15 | 16 | // ErrNotFound is returned when a url can't be found with a given code. 17 | var ErrNotFound = errors.New("not found") 18 | 19 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 20 | 21 | const ( 22 | letterIdxBits = 6 // 6 bits to represent a letter index 23 | letterIdxMask = 1<= 0; { 34 | if remain == 0 { 35 | cache, remain = src.Int63(), letterIdxMax 36 | } 37 | if idx := int(cache & letterIdxMask); idx < len(letterBytes) { 38 | b[i] = letterBytes[idx] 39 | i-- 40 | } 41 | cache >>= letterIdxBits 42 | remain-- 43 | } 44 | 45 | return string(b) 46 | } 47 | --------------------------------------------------------------------------------