├── .dockerignore ├── .gitignore ├── Dockerfile ├── Dockerfile.worker ├── README.md ├── auth └── authstore.go ├── cmd ├── admin │ └── main.go ├── server │ ├── README.md │ ├── auth.go │ ├── balancer.go │ ├── balancer_test.go │ ├── content_type.go │ ├── limiter.go │ ├── logger.go │ └── main.go └── worker │ ├── README.md │ └── worker.go ├── docker-compose.yml ├── glide.lock ├── glide.yaml ├── job ├── job.go └── jobstore.go ├── proxy-service.yaml ├── sandbox-service.yaml └── stats-service.yaml /.dockerignore: -------------------------------------------------------------------------------- 1 | vendor 2 | *.a 3 | *.lib 4 | *.dylib 5 | setup.sh 6 | google-cloud-sdk 7 | ccurld-pipe 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | *.exe 3 | *.bat 4 | *.a 5 | *.lib 6 | *.dylib 7 | *.json 8 | *.deploy 9 | setup.sh 10 | google-cloud-sdk 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1-alpine 2 | 3 | RUN apk add --no-cache git gcc libc-dev 4 | 5 | RUN go get -u github.com/Masterminds/glide 6 | 7 | RUN mkdir -p /go/src/github.com/iotaledger/sandbox 8 | 9 | COPY . /go/src/github.com/iotaledger/sandbox 10 | 11 | WORKDIR /go/src/github.com/iotaledger/sandbox 12 | 13 | RUN glide install 14 | 15 | RUN go build -o sandbox-server ./cmd/server 16 | CMD ["./sandbox-server"] 17 | 18 | EXPOSE "8080" 19 | -------------------------------------------------------------------------------- /Dockerfile.worker: -------------------------------------------------------------------------------- 1 | FROM ubuntu:17.04 2 | 3 | # gcc for cgo 4 | RUN apt-get update && apt-get install -y --no-install-recommends \ 5 | g++ \ 6 | gcc \ 7 | libc6-dev \ 8 | make \ 9 | curl \ 10 | pkg-config 11 | 12 | ENV GOLANG_VERSION 1.8.3 13 | 14 | RUN apt-get install -y ca-certificates 15 | ENV capath /etc/ssl/certs/ 16 | ENV cacert /etc/ssl/certs/ca-certificates.crt 17 | RUN curl -sL https://golang.org/dl/go1.8.3.linux-amd64.tar.gz|tar -C /usr/local -xzf - 18 | ENV PATH /usr/local/go/bin:${PATH} 19 | RUN go version 20 | 21 | ENV GOPATH /go 22 | ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH 23 | 24 | RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" 25 | WORKDIR $GOPATH 26 | 27 | RUN apt-get install -y git build-essential cmake xxd gdb && rm -rf /var/lib/apt/lists/* 28 | 29 | RUN mkdir -p /opt 30 | RUN git clone https://github.com/iotaledger/ccurl 31 | RUN cd ccurl && git submodule update --init --recursive 32 | RUN cd ccurl && mkdir build && cd build && cmake .. && cmake --build . && mv bin/ccurl-cli /opt/ccurl 33 | 34 | RUN go get -u github.com/Masterminds/glide 35 | 36 | RUN mkdir -p /go/src/github.com/iotaledger/sandbox 37 | 38 | COPY . /go/src/github.com/iotaledger/sandbox 39 | 40 | WORKDIR /go/src/github.com/iotaledger/sandbox 41 | 42 | RUN glide install 43 | 44 | RUN go build -o sandbox-worker ./cmd/worker 45 | CMD ["./sandbox-worker"] 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IOTA sandbox 2 | 3 | The sandbox is currently required to run inside the google cloud. 4 | 5 | ## Building 6 | 7 | Use glide to install the dependencies: 8 | 9 | ``` 10 | # glide install 11 | ``` 12 | 13 | and then build the server (or the worker): 14 | 15 | ``` 16 | # go build -o sandbox-server ./cmd/server 17 | ``` 18 | 19 | 20 | ## Usage 21 | 22 | The easiest way to run the full system is to use the supplied `docker-compose.yml` 23 | file, which is going to run one instance of the API server, worker and emulators for 24 | the two google cloud services it relies on—pubsub and datastore. 25 | 26 | If you want to run either without using the provided docker images then you need to 27 | provide emulators—or run them inside the google cloud—, which can be done after installing 28 | the google cloud sdk. 29 | 30 | ``` 31 | $ CLOUDSDK_INSTALL_DIR=./ curl https://sdk.cloud.google.com | bash 32 | $ ./google-cloud-sdk/bin/gcloud init 33 | $ ./google-cloud-sdk/bin/gcloud components install -q pubsub-emulator beta 34 | $ ./google-cloud-sdk/bin/gcloud components install -q cloud-datastore-emulator beta 35 | $ ./google-cloud-sdk/bin/gcloud beta emulators pubsub start 36 | $ $(./google-cloud-sdk/bin/gcloud beta emulators pubsub env-init) # sets PUBSUB_EMULATOR_HOST 37 | $ ./google-cloud-sdk/bin/gcloud beta emulators datastore start 38 | $ $(./google-cloud-sdk/bin/gcloud beta emulators datastore env-init) # sets DATASTORE_EMULATOR_HOST 39 | ``` 40 | 41 | ### Sandbox API 42 | -------------------------------------------------------------------------------- /auth/authstore.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | //uuid "github.com/satori/go.uuid" 6 | "cloud.google.com/go/datastore" 7 | "google.golang.org/api/option" 8 | ) 9 | 10 | type Auth struct { 11 | Token string `json:"token"` 12 | Active bool `json:"active" bson:"active"` 13 | EMail string `json:"email" bson:"email"` 14 | CreatedAt int64 `json:"createdAt"` 15 | ModifiedAt int64 `json:"modifiedAt"` 16 | } 17 | 18 | // AuthStore describes backend 19 | type AuthStore interface { 20 | ValidToken(ctx context.Context, tok string) (bool, error) 21 | Add(ctx context.Context, a *Auth) error 22 | Get(ctx context.Context, tok string) (*Auth, error) 23 | } 24 | 25 | // DummyStore just accepts any non empty token as valid. 26 | type DummyStore struct{} 27 | 28 | func NewDummyStore() (*DummyStore, error) { 29 | return &DummyStore{}, nil 30 | } 31 | 32 | func (d *DummyStore) ValidToken(_ context.Context, tok string) (bool, error) { 33 | return tok != "", nil 34 | } 35 | 36 | func (d *DummyStore) Add(_ context.Context, a *Auth) error { 37 | return nil 38 | } 39 | 40 | func (d *DummyStore) Get(_ context.Context, tok string) (*Auth, error) { 41 | return &Auth{Token: tok}, nil 42 | } 43 | 44 | type GCloudDataStore struct { 45 | client *datastore.Client 46 | } 47 | 48 | func NewGCloudDataStore(projectID, credPath string) (*GCloudDataStore, error) { 49 | ctx := context.Background() 50 | var dsClient *datastore.Client 51 | if credPath != "" { 52 | c, err := datastore.NewClient(ctx, projectID, option.WithServiceAccountFile(credPath)) 53 | if err != nil { 54 | return nil, err 55 | } 56 | dsClient = c 57 | } else { 58 | c, err := datastore.NewClient(ctx, projectID) 59 | if err != nil { 60 | return nil, err 61 | } 62 | dsClient = c 63 | } 64 | 65 | g := &GCloudDataStore{client: dsClient} 66 | return g, nil 67 | } 68 | 69 | func (g *GCloudDataStore) ValidToken(ctx context.Context, tok string) (bool, error) { 70 | a, err := g.Get(ctx, tok) 71 | if err != nil { 72 | return false, err 73 | } 74 | 75 | return a.Active, nil 76 | } 77 | 78 | func (g *GCloudDataStore) Add(ctx context.Context, a *Auth) error { 79 | k := datastore.NameKey("Auth", a.Token, nil) 80 | if _, err := g.client.Put(ctx, k, a); err != nil { 81 | return err 82 | } 83 | 84 | return nil 85 | } 86 | 87 | func (g *GCloudDataStore) Get(ctx context.Context, tok string) (*Auth, error) { 88 | a := &Auth{} 89 | k := datastore.NameKey("Auth", tok, nil) 90 | if err := g.client.Get(ctx, k, a); err != nil { 91 | return nil, err 92 | } 93 | 94 | return a, nil 95 | } 96 | -------------------------------------------------------------------------------- /cmd/admin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "github.com/iotaledger/sandbox/auth" 10 | 11 | uuid "github.com/satori/go.uuid" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | var ( 16 | rootCmd = &cobra.Command{ 17 | Use: "admin", 18 | Short: "admin is a simple cli interface for the iota sandbox", 19 | } 20 | ) 21 | 22 | var ( 23 | credPath = "" 24 | projectID = "" 25 | ) 26 | 27 | var ( 28 | authEMail = "" 29 | authActive = true 30 | ) 31 | 32 | func init() { 33 | rootCmd.PersistentFlags().StringVar(&credPath, "credentials", "./credentials.json", "google cloud credentials") 34 | rootCmd.PersistentFlags().StringVar(&projectID, "project-id", "", "google cloud project id") 35 | } 36 | 37 | func main() { 38 | var authStoreCmd = &cobra.Command{ 39 | Use: "auth", 40 | Short: "interact with the auth storage", 41 | } 42 | 43 | authStoreCmd.PersistentFlags().StringVarP(&authEMail, "email", "", "", "email address to store along with the token") 44 | authStoreCmd.PersistentFlags().BoolVarP(&authActive, "active", "", true, "is the token enabled or not") 45 | 46 | var authStoreAddCmd = &cobra.Command{ 47 | Use: "add", 48 | Short: "add a new auth store entry, and prints token if created successfully", 49 | Example: `admin auth add cd1bf18a-1f8a-4d0e-ad87-dd52739c567f --active=true --email=foo@bar.com --project-id=sandbox 50 | # if you don't provide a token then a uuidv4 will be used 51 | admin auth add --project-id=sandbox 52 | `, 53 | RunE: func(cmd *cobra.Command, args []string) error { 54 | return RunAddAuth(cmd, args) 55 | }, 56 | } 57 | 58 | var authStoreGetCmd = &cobra.Command{ 59 | Use: "get", 60 | Short: "retrieve auth store entry with supplied token", 61 | Example: `admin auth get cd1bf18a-1f8a-4d0e-ad87-dd52739c567f --project-id=sandbox`, 62 | RunE: func(cmd *cobra.Command, args []string) error { 63 | return RunGetAuth(cmd, args) 64 | }, 65 | } 66 | 67 | authStoreCmd.AddCommand(authStoreAddCmd) 68 | authStoreCmd.AddCommand(authStoreGetCmd) 69 | rootCmd.AddCommand(authStoreCmd) 70 | 71 | if err := rootCmd.Execute(); err != nil { 72 | fmt.Println(err) 73 | os.Exit(1) 74 | } 75 | } 76 | 77 | func RunAddAuth(cmd *cobra.Command, args []string) error { 78 | if err := rootCmd.ParseFlags(args); err != nil { 79 | return err 80 | } 81 | 82 | ds, err := auth.NewGCloudDataStore(projectID, credPath) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | var tok string 88 | if len(args) < 1 { 89 | tok = uuid.NewV4().String() 90 | } else { 91 | tok = args[0] 92 | } 93 | 94 | a := &auth.Auth{ 95 | Token: tok, 96 | Active: authActive, 97 | EMail: authEMail, 98 | CreatedAt: time.Now().Unix(), 99 | ModifiedAt: time.Now().Unix(), 100 | } 101 | 102 | ctx, cf := context.WithTimeout(context.Background(), time.Second*15) 103 | defer cf() 104 | 105 | if err := ds.Add(ctx, a); err != nil { 106 | return err 107 | } 108 | 109 | fmt.Printf("%s\n", a.Token) 110 | return nil 111 | } 112 | 113 | func RunGetAuth(cmd *cobra.Command, args []string) error { 114 | if err := rootCmd.ParseFlags(args); err != nil { 115 | return err 116 | } 117 | 118 | ds, err := auth.NewGCloudDataStore(projectID, credPath) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | if len(args) < 1 { 124 | return fmt.Errorf("no token supplied") 125 | } 126 | tok := args[0] 127 | 128 | ctx, cf := context.WithTimeout(context.Background(), time.Second*15) 129 | defer cf() 130 | 131 | a, err := ds.Get(ctx, tok) 132 | if err != nil { 133 | return err 134 | } 135 | 136 | fmt.Printf("%#v\n", a) 137 | return nil 138 | } 139 | -------------------------------------------------------------------------------- /cmd/server/README.md: -------------------------------------------------------------------------------- 1 | # Server 2 | 3 | ## environment variables 4 | 5 | required: 6 | 7 | - `AWS_ACCESS_KEY_ID` 8 | - `AWS_SECRET_ACCESS_KEY` 9 | - `AWS_REGION` 10 | - `INCOMING_QUEUE_NAME` 11 | - `FINISHED_QUEUE_NAME` 12 | 13 | optional: 14 | 15 | - `IRI_URI` default `http://localhost:14265/` 16 | - `DEBUG` set to `1` for debug logging 17 | -------------------------------------------------------------------------------- /cmd/server/auth.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/iotaledger/sandbox/auth" 11 | ) 12 | 13 | var headerSplitter = regexp.MustCompile(`\s+`) 14 | 15 | // AuthMiddleware adds the `authorization` value to the request context, 16 | // so that handlers after it can check for the authentication status of 17 | // the current request. 18 | type AuthMiddleware struct { 19 | store auth.AuthStore 20 | } 21 | 22 | func NewAuthMiddleware(as auth.AuthStore) *AuthMiddleware { 23 | return &AuthMiddleware{store: as} 24 | } 25 | 26 | // parseAuthorizationHeader expects a string in the form of 27 | // `token authtoken` and returns the `authtoken` or an empty string. 28 | func parseAuthorizationHeader(h string) string { 29 | parts := headerSplitter.Split(h, 2) 30 | if len(parts) != 2 { 31 | return "" 32 | } else if strings.ToLower(parts[0]) != "token" { 33 | return "" 34 | } 35 | 36 | return strings.TrimSpace(parts[1]) 37 | } 38 | 39 | func (a *AuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 40 | hdr := r.Header.Get(http.CanonicalHeaderKey("Authorization")) 41 | token := parseAuthorizationHeader(hdr) 42 | if token == "" { 43 | next(w, r) 44 | return 45 | } 46 | 47 | auth, err := a.store.ValidToken(r.Context(), token) 48 | if err != nil { 49 | fmt.Printf("ValidToken error: %s\n", err) 50 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "could not authenticate, please try again later"}) 51 | return 52 | } 53 | 54 | ogCtx := r.Context() 55 | freshCtx := context.WithValue(ogCtx, "authorization", auth) 56 | next(w, r.WithContext(freshCtx)) 57 | } 58 | -------------------------------------------------------------------------------- /cmd/server/balancer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "sync" 8 | "time" 9 | 10 | "github.com/iotaledger/giota" 11 | ) 12 | 13 | type IRIBalancer struct { 14 | mu sync.Mutex // Protect the healthy nodes slice. 15 | nodes []*giota.API 16 | healthy []int // Indices of healthy nodes 17 | next int // Index for the next healthy node index to return, i.e. 18 | // nodes[healthy[next]] 19 | 20 | done chan struct{} 21 | ticker *time.Ticker 22 | } 23 | 24 | func NewIRIBalancer(urls []string) *IRIBalancer { 25 | ib := &IRIBalancer{} 26 | nodes := []*giota.API{} 27 | healthy := []int{} 28 | for i, u := range urls { 29 | tr := &http.Transport{ 30 | DialContext: (&net.Dialer{ 31 | Timeout: 10 * time.Second, 32 | KeepAlive: 30 * time.Second, 33 | }).DialContext, 34 | MaxIdleConns: 100, 35 | IdleConnTimeout: 90 * time.Second, 36 | TLSHandshakeTimeout: 10 * time.Second, 37 | ExpectContinueTimeout: 1 * time.Second, 38 | } 39 | client := &http.Client{ 40 | Transport: tr, 41 | } 42 | nodes = append(nodes, giota.NewAPI(u, client)) 43 | healthy = append(healthy, i) 44 | } 45 | 46 | return ib 47 | } 48 | 49 | // Start checking the health of the given node every interval. 50 | func (ib *IRIBalancer) Start(interval time.Duration) error { 51 | if interval < 0 { 52 | return fmt.Errorf("invalid interval duration") 53 | } 54 | 55 | ib.done = make(chan struct{}) 56 | ib.ticker = time.NewTicker(interval) 57 | 58 | go func() { 59 | for { 60 | select { 61 | case _ = <-ib.ticker.C: 62 | if err := ib.checkNodes(); err != nil { 63 | return 64 | } 65 | case <-ib.done: 66 | return 67 | } 68 | } 69 | }() 70 | 71 | return nil 72 | } 73 | 74 | func (ib *IRIBalancer) checkNodes() error { 75 | ib.mu.Lock() 76 | defer ib.mu.Unlock() 77 | 78 | healthy := []int{} 79 | for i, n := range ib.nodes { 80 | if err := ib.isHealthyNode(n); err == nil { 81 | healthy = append(healthy, i) 82 | } 83 | } 84 | 85 | if ib.next > len(healthy) { 86 | ib.next = 0 87 | } 88 | 89 | ib.healthy = healthy 90 | 91 | return nil 92 | } 93 | 94 | func (ib *IRIBalancer) isHealthyNode(n *giota.API) error { 95 | if n == nil { 96 | return fmt.Errorf("received nil node") 97 | } 98 | 99 | // XXX: Consider coming up with better metrics for node health. 100 | if _, err := n.GetNodeInfo(); err != nil { 101 | return err 102 | } 103 | 104 | return nil 105 | } 106 | 107 | // Get the next available node or return an error if none are available. 108 | func (ib *IRIBalancer) Get() (*giota.API, error) { 109 | // XXX: Maybe this methdd should block. 110 | // If it were to block then it should take a context to let the 111 | // caller specify a timeout/deadline. 112 | ib.mu.Lock() 113 | defer ib.mu.Unlock() 114 | 115 | if len(ib.healthy) == 0 { 116 | return nil, fmt.Errorf("no available nodes") 117 | } 118 | 119 | ib.next += 1 120 | if ib.next >= len(ib.healthy) { 121 | ib.next = 0 122 | } 123 | 124 | return ib.nodes[ib.healthy[ib.next]], nil 125 | } 126 | 127 | // Close shuts down the balancer. 128 | func (ib *IRIBalancer) Close() error { 129 | ib.mu.Lock() 130 | defer ib.mu.Unlock() 131 | close(ib.done) 132 | ib.ticker.Stop() 133 | 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /cmd/server/balancer_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestBalancer(t *testing.T) { 9 | b := NewIRIBalancer([]string{"http://localhost:11111", "http://localhost:22222", "http://localhost:33333"}) 10 | b.Start(time.Microsecond) 11 | time.Sleep(time.Millisecond * 500) 12 | _, err := b.Get() 13 | if err == nil { 14 | t.Fatalf("expected to find no available nodes but got one instead") 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /cmd/server/content_type.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/urfave/negroni" 7 | ) 8 | 9 | func hasCT(ct string, cts []string) bool { 10 | for _, c := range cts { 11 | if c == ct { 12 | return true 13 | } 14 | } 15 | 16 | return false 17 | } 18 | 19 | func ContentTypeEnforcer(cts ...string) negroni.HandlerFunc { 20 | return negroni.HandlerFunc(func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 21 | ct := r.Header.Get(http.CanonicalHeaderKey("Content-Type")) 22 | if hasCT(ct, cts) || (r.Method != "POST" && r.Method != "PUT") { 23 | w.Header().Set("Content-Type", "application/json") 24 | next(w, r) 25 | } else { 26 | http.Error(w, http.StatusText(http.StatusUnsupportedMediaType), http.StatusUnsupportedMediaType) 27 | } 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /cmd/server/limiter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/didip/tollbooth" 8 | "github.com/didip/tollbooth/errors" 9 | "github.com/didip/tollbooth/libstring" 10 | "github.com/didip/tollbooth/limiter" 11 | "github.com/julienschmidt/httprouter" 12 | "github.com/urfave/negroni" 13 | ) 14 | 15 | // CmdLimiter limits API calls based on the command used. If no limiter 16 | // was specified for a given command, then a global fallback will be used. 17 | type CmdLimiter struct { 18 | limiters map[string]*limiter.Limiter 19 | fallback *limiter.Limiter 20 | } 21 | 22 | func NewCmdLimiter(limits map[string]int64, def int64) *CmdLimiter { 23 | limiters := map[string]*limiter.Limiter{} 24 | for k, v := range limits { 25 | limiters[k] = tollbooth.NewLimiter(v, 1*time.Minute, nil) 26 | } 27 | 28 | defLim := tollbooth.NewLimiter(def, 1*time.Minute, nil) 29 | clim := &CmdLimiter{fallback: defLim, limiters: limiters} 30 | 31 | return clim 32 | } 33 | 34 | func (c *CmdLimiter) Limit(cmd string, r *http.Request) *errors.HTTPError { 35 | l, ok := c.limiters[cmd] 36 | remoteIP := libstring.RemoteIP(c.fallback.GetIPLookups(), 0, r) 37 | keys := []string{remoteIP, cmd} 38 | if !ok { // Use fallback if cmd was not found. 39 | return tollbooth.LimitByKeys(c.fallback, keys) 40 | } 41 | 42 | return tollbooth.LimitByKeys(l, keys) 43 | } 44 | 45 | // From: https://github.com/didip/tollbooth/tree/master/thirdparty/tollbooth_negroni 46 | 47 | func LimitHandler(limiter *limiter.Limiter) negroni.HandlerFunc { 48 | return negroni.HandlerFunc(func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 49 | httpError := tollbooth.LimitByRequest(limiter, w, r) 50 | if httpError != nil { 51 | writeError(w, httpError.StatusCode, ErrorResp{Message: httpError.Message}) 52 | return 53 | } else { 54 | next(w, r) 55 | } 56 | }) 57 | } 58 | 59 | func AttachLimitHandler(handler httprouter.Handle, limiter *limiter.Limiter) httprouter.Handle { 60 | return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 61 | httpError := tollbooth.LimitByRequest(limiter, w, r) 62 | if httpError != nil { 63 | writeError(w, httpError.StatusCode, ErrorResp{Message: httpError.Message}) 64 | return 65 | } 66 | handler(w, r, ps) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cmd/server/logger.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/urfave/negroni" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type LoggerMiddleware struct { 12 | logger *zap.Logger 13 | } 14 | 15 | func NewLoggerMiddleware(l *zap.Logger) (*LoggerMiddleware, error) { 16 | mw := &LoggerMiddleware{} 17 | 18 | if l == nil { 19 | logger, err := zap.NewDevelopment() 20 | if err != nil { 21 | return nil, err 22 | } 23 | mw.logger = logger 24 | } else { 25 | mw.logger = l 26 | } 27 | 28 | return mw, nil 29 | } 30 | 31 | func (mw *LoggerMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 32 | s := time.Now() 33 | next(w, r) 34 | res := w.(negroni.ResponseWriter) 35 | 36 | mw.logger.Info("completed request", 37 | zap.Duration("duration", time.Since(s)), 38 | zap.Int("status", res.Status()), 39 | zap.String("remote", r.RemoteAddr), 40 | zap.String("path", r.RequestURI), 41 | ) 42 | } 43 | -------------------------------------------------------------------------------- /cmd/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net" 9 | "net/http" 10 | "os" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/iotaledger/giota" 15 | "github.com/iotaledger/sandbox/auth" 16 | "github.com/iotaledger/sandbox/job" 17 | 18 | "github.com/DataDog/datadog-go/statsd" 19 | "go.uber.org/zap" 20 | 21 | "cloud.google.com/go/pubsub" 22 | "github.com/didip/tollbooth" 23 | "github.com/julienschmidt/httprouter" 24 | "github.com/rs/cors" 25 | uuid "github.com/satori/go.uuid" 26 | "github.com/urfave/negroni" 27 | "golang.org/x/net/context" 28 | "google.golang.org/api/option" 29 | ) 30 | 31 | type App struct { 32 | iriClient *giota.API 33 | router *httprouter.Router 34 | cmdLimiter *CmdLimiter 35 | jobMaxAge time.Duration 36 | 37 | logger *zap.Logger 38 | stats *statsd.Client 39 | 40 | incomingJobsTopic *pubsub.Topic 41 | finishedJobsSubscription *pubsub.Subscription 42 | jobStore job.JobStore 43 | } 44 | 45 | const ( 46 | V1BasePath = "/api/v1" 47 | ) 48 | 49 | type ErrorResp struct { 50 | Message string `json:"message"` 51 | } 52 | 53 | func writeError(w http.ResponseWriter, code int, e ErrorResp) { 54 | w.Header().Set("Content-Type", "application/json") 55 | w.WriteHeader(code) 56 | err := json.NewEncoder(w).Encode(e) 57 | if err != nil { 58 | // Oh god, bail out. 59 | w.Write([]byte(err.Error())) 60 | } 61 | } 62 | 63 | func (app *App) PostCommandsGetNodeInfo(w http.ResponseWriter, b []byte, _ httprouter.Params) { 64 | gnir := &giota.GetNodeInfoRequest{} 65 | err := json.Unmarshal(b, gnir) 66 | if err != nil { 67 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 68 | return 69 | } 70 | 71 | if gnir.Command != "getNodeInfo" { 72 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + gnir.Command}) 73 | return 74 | } 75 | 76 | gni, err := app.iriClient.GetNodeInfo() 77 | if err != nil { 78 | app.logger.Error("iri client", zap.String("callee", "GetNodeInfo"), zap.Error(err)) 79 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 80 | return 81 | } 82 | 83 | err = json.NewEncoder(w).Encode(gni) 84 | if err != nil { 85 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 86 | return 87 | } 88 | } 89 | 90 | func (app *App) PostCommandsGetTips(w http.ResponseWriter, b []byte, _ httprouter.Params) { 91 | gtr := &giota.GetTipsRequest{} 92 | err := json.Unmarshal(b, gtr) 93 | if err != nil { 94 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 95 | return 96 | } 97 | 98 | if gtr.Command != "getTips" { 99 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + gtr.Command}) 100 | return 101 | } 102 | 103 | gt, err := app.iriClient.GetTips() 104 | if err != nil { 105 | app.logger.Error("iri client", zap.String("callee", "GetTips"), zap.Error(err)) 106 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 107 | return 108 | } 109 | 110 | err = json.NewEncoder(w).Encode(gt) 111 | if err != nil { 112 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 113 | return 114 | } 115 | } 116 | 117 | func (app *App) PostCommandsFindTransactions(w http.ResponseWriter, b []byte, _ httprouter.Params) { 118 | ftr := &giota.FindTransactionsRequest{} 119 | err := json.Unmarshal(b, ftr) 120 | if err != nil { 121 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 122 | return 123 | } 124 | 125 | if ftr.Command != "findTransactions" { 126 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + ftr.Command}) 127 | return 128 | } 129 | 130 | ft, err := app.iriClient.FindTransactions(ftr) 131 | if err != nil { 132 | app.logger.Error("iri client", zap.String("callee", "FindTransactions"), zap.Error(err)) 133 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 134 | return 135 | } 136 | 137 | err = json.NewEncoder(w).Encode(ft) 138 | if err != nil { 139 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 140 | return 141 | } 142 | } 143 | 144 | func (app *App) PostCommandsGetTrytes(w http.ResponseWriter, b []byte, _ httprouter.Params) { 145 | gtr := &giota.GetTrytesRequest{} 146 | err := json.Unmarshal(b, gtr) 147 | if err != nil { 148 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 149 | return 150 | } 151 | 152 | if gtr.Command != "getTrytes" { 153 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + gtr.Command}) 154 | return 155 | } 156 | 157 | gt, err := app.iriClient.GetTrytes(gtr.Hashes) 158 | if err != nil { 159 | app.logger.Error("iri client", zap.String("callee", "GetTrytes"), zap.Error(err)) 160 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 161 | return 162 | } 163 | 164 | err = json.NewEncoder(w).Encode(gt) 165 | if err != nil { 166 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 167 | return 168 | } 169 | } 170 | 171 | func (app *App) PostCommandsGetInclusionStates(w http.ResponseWriter, b []byte, _ httprouter.Params) { 172 | gisr := &giota.GetInclusionStatesRequest{} 173 | err := json.Unmarshal(b, gisr) 174 | if err != nil { 175 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 176 | return 177 | } 178 | 179 | if gisr.Command != "getInclusionStates" { 180 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + gisr.Command}) 181 | return 182 | } 183 | 184 | gis, err := app.iriClient.GetInclusionStates(gisr.Transactions, gisr.Tips) 185 | if err != nil { 186 | app.logger.Error("iri client", zap.String("callee", "GetInclusionStates"), zap.Error(err)) 187 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 188 | return 189 | } 190 | 191 | err = json.NewEncoder(w).Encode(gis) 192 | if err != nil { 193 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 194 | return 195 | } 196 | } 197 | 198 | func (app *App) PostCommandsGetBalances(w http.ResponseWriter, b []byte, _ httprouter.Params) { 199 | gbr := &giota.GetBalancesRequest{} 200 | err := json.Unmarshal(b, gbr) 201 | if err != nil { 202 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 203 | return 204 | } 205 | 206 | if gbr.Command != "getBalances" { 207 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + gbr.Command}) 208 | return 209 | } 210 | 211 | gb, err := app.iriClient.GetBalances(gbr.Addresses, gbr.Threshold) 212 | if err != nil { 213 | app.logger.Error("iri client", zap.String("callee", "GetBalances"), zap.Error(err)) 214 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 215 | return 216 | } 217 | 218 | err = json.NewEncoder(w).Encode(gb) 219 | if err != nil { 220 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 221 | return 222 | } 223 | } 224 | 225 | func (app *App) PostCommandsGetTransactionsToApprove(w http.ResponseWriter, b []byte, _ httprouter.Params) { 226 | gttar := &giota.GetTransactionsToApproveRequest{} 227 | err := json.Unmarshal(b, gttar) 228 | if err != nil { 229 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 230 | return 231 | } 232 | 233 | if gttar.Command != "getTransactionsToApprove" { 234 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + gttar.Command}) 235 | return 236 | } 237 | 238 | gtta, err := app.iriClient.GetTransactionsToApprove(gttar.Depth) 239 | if err != nil { 240 | app.logger.Error("iri client", zap.String("callee", "GetTransactionsToApprove"), zap.Error(err)) 241 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 242 | return 243 | } 244 | 245 | err = json.NewEncoder(w).Encode(gtta) 246 | if err != nil { 247 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 248 | return 249 | } 250 | } 251 | 252 | func validTrytesSlice(ts []giota.Trytes) bool { 253 | for _, t := range ts { 254 | if t.IsValid() != nil { 255 | return false 256 | } 257 | } 258 | return true 259 | } 260 | 261 | func (app *App) PostCommandsAttachToTangle(w http.ResponseWriter, b []byte, _ httprouter.Params) { 262 | attr := &giota.AttachToTangleRequest{} 263 | err := json.Unmarshal(b, attr) 264 | if err != nil { 265 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 266 | return 267 | } 268 | 269 | if attr.Command != "attachToTangle" { 270 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + attr.Command}) 271 | return 272 | } 273 | 274 | if len(attr.Trytes) < 1 { 275 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid trytes"}) 276 | return 277 | } 278 | 279 | j := job.NewIRIJob(attr.Command) 280 | j.AttachToTangleRequest = attr 281 | id, err := app.jobStore.InsertJob(j) 282 | if err != nil { 283 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 284 | return 285 | } 286 | 287 | jb, err := json.Marshal(j) 288 | if err != nil { 289 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 290 | return 291 | } 292 | 293 | ctx := context.Background() 294 | r := app.incomingJobsTopic.Publish(ctx, &pubsub.Message{ 295 | Data: jb, 296 | }) 297 | 298 | go func() { 299 | _, err := r.Get(ctx) 300 | if err != nil { 301 | app.logger.Error("publish pubsub message", zap.Error(err)) 302 | j.Error = &job.JobError{Message: "unable to add job to queue"} 303 | j.Status = job.JobStatusFailed 304 | _, err := app.jobStore.UpdateJob(id, j) 305 | if err != nil { 306 | app.logger.Error("update jobStore", zap.Error(err)) 307 | } 308 | } 309 | }() 310 | 311 | w.WriteHeader(http.StatusAccepted) 312 | w.Header().Set("Link", V1BasePath+"/jobs/"+id) 313 | err = json.NewEncoder(w).Encode(j) 314 | if err != nil { 315 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 316 | return 317 | } 318 | 319 | return 320 | } 321 | 322 | func (app *App) PostCommandsBroadcastTransactions(w http.ResponseWriter, b []byte, _ httprouter.Params) { 323 | btr := &giota.BroadcastTransactionsRequest{} 324 | err := json.Unmarshal(b, btr) 325 | if err != nil { 326 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 327 | return 328 | } 329 | 330 | if btr.Command != "broadcastTransactions" { 331 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + btr.Command}) 332 | return 333 | } 334 | 335 | err = app.iriClient.BroadcastTransactions(btr.Trytes) 336 | if err != nil { 337 | app.logger.Error("iri client", zap.String("callee", "BroadcastTransactions"), zap.Error(err)) 338 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 339 | return 340 | } 341 | 342 | err = json.NewEncoder(w).Encode(struct{}{}) 343 | if err != nil { 344 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 345 | return 346 | } 347 | } 348 | 349 | func (app *App) PostCommandsStoreTransactions(w http.ResponseWriter, b []byte, _ httprouter.Params) { 350 | str := &giota.StoreTransactionsRequest{} 351 | err := json.Unmarshal(b, str) 352 | if err != nil { 353 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 354 | return 355 | } 356 | 357 | if str.Command != "storeTransactions" { 358 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + str.Command}) 359 | return 360 | } 361 | 362 | err = app.iriClient.StoreTransactions(str.Trytes) 363 | if err != nil { 364 | app.logger.Error("iri client", zap.String("callee", "StoreTransactions"), zap.Error(err)) 365 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: "failed to talk to IRI"}) 366 | return 367 | } 368 | 369 | err = json.NewEncoder(w).Encode(struct{}{}) 370 | if err != nil { 371 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 372 | return 373 | } 374 | } 375 | 376 | type PostCommand struct { 377 | Command string `json:"command"` 378 | } 379 | 380 | func (app *App) PostCommands(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 381 | lr := io.LimitReader(r.Body, 8388608) // 2^23 bytes 382 | b, err := ioutil.ReadAll(lr) 383 | if err != nil { 384 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 385 | return 386 | } 387 | 388 | cmd := &PostCommand{} 389 | err = json.Unmarshal(b, cmd) 390 | if err != nil { 391 | writeError(w, http.StatusBadRequest, ErrorResp{Message: err.Error()}) 392 | return 393 | } 394 | 395 | auth := r.Context().Value("authorization") 396 | if auth == nil || !auth.(bool) { // If we're not authenticated, then we're subjected to rate limiting. 397 | limitReached := app.cmdLimiter.Limit(cmd.Command, r) 398 | if limitReached != nil { 399 | writeError(w, limitReached.StatusCode, ErrorResp{Message: limitReached.Message}) 400 | return 401 | } 402 | } 403 | 404 | switch cmd.Command { 405 | default: 406 | app.stats.Incr("request.command.unknown", nil, 1) 407 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid command: " + cmd.Command}) 408 | return 409 | case "getNodeInfo": 410 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 411 | app.PostCommandsGetNodeInfo(w, b, nil) 412 | case "getTips": 413 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 414 | app.PostCommandsGetTips(w, b, nil) 415 | case "findTransactions": 416 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 417 | app.PostCommandsFindTransactions(w, b, nil) 418 | case "getTrytes": 419 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 420 | app.PostCommandsGetTrytes(w, b, nil) 421 | case "getInclusionStates": 422 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 423 | app.PostCommandsGetInclusionStates(w, b, nil) 424 | case "getBalances": 425 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 426 | app.PostCommandsGetBalances(w, b, nil) 427 | case "getTransactionsToApprove": 428 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 429 | app.PostCommandsGetTransactionsToApprove(w, b, nil) 430 | case "broadcastTransactions": 431 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 432 | app.PostCommandsBroadcastTransactions(w, b, nil) 433 | case "storeTransactions": 434 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 435 | app.PostCommandsStoreTransactions(w, b, nil) 436 | case "attachToTangle": 437 | app.stats.Incr("request.command."+cmd.Command, nil, 1) 438 | app.PostCommandsAttachToTangle(w, b, nil) 439 | } 440 | } 441 | 442 | func (app *App) GetJobsID(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 443 | id := uuid.FromStringOrNil(ps.ByName("id")) 444 | if id == uuid.Nil { 445 | writeError(w, http.StatusBadRequest, ErrorResp{Message: "invalid id"}) 446 | return 447 | } 448 | 449 | app.stats.Incr("request.jobs", nil, 1) 450 | 451 | j, err := app.jobStore.SelectJob(id.String()) 452 | if err == job.ErrJobNotFound { 453 | writeError(w, http.StatusNotFound, ErrorResp{Message: err.Error()}) 454 | return 455 | } else if err != nil { 456 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 457 | return 458 | } 459 | 460 | err = json.NewEncoder(w).Encode(j) 461 | if err != nil { 462 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 463 | return 464 | } 465 | } 466 | 467 | type APIStatus struct { 468 | IRIReachable bool `json:"iriReachable"` 469 | JobStats struct { 470 | Since int64 `json:"since"` 471 | Processed int64 `json:"processed"` 472 | FailureRate float64 `json:"failureRate"` 473 | } `json:"jobStats"` 474 | } 475 | 476 | func (app *App) GetAPIStatus(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 477 | app.stats.Incr("request.status", nil, 1) 478 | 479 | _, err := app.iriClient.GetNodeInfo() 480 | s := &APIStatus{} 481 | t := time.Now().Add(-1 * time.Hour) 482 | count, rate := app.jobStore.JobFailureRate(-1 * time.Hour) 483 | s.JobStats.Since = t.Unix() 484 | s.JobStats.Processed = count 485 | s.JobStats.FailureRate = rate 486 | s.IRIReachable = err == nil 487 | 488 | err = json.NewEncoder(w).Encode(s) 489 | if err != nil { 490 | writeError(w, http.StatusInternalServerError, ErrorResp{Message: err.Error()}) 491 | return 492 | } 493 | } 494 | 495 | // PullFinishedJobs sets up the receiver function for the finished jobs pubsub 496 | // subscription. 497 | func (app *App) PullFinishedJobs() error { 498 | ctx := context.Background() 499 | err := app.finishedJobsSubscription.Receive(ctx, func(ctx context.Context, m *pubsub.Message) { 500 | app.logger.Debug("got new finished job") 501 | j := &job.IRIJob{} 502 | err := json.Unmarshal(m.Data, j) 503 | if err != nil { 504 | app.logger.Error("unmarshal finished job", zap.Error(err)) 505 | m.Ack() 506 | return 507 | } 508 | 509 | _, err = app.jobStore.UpdateJob(j.ID, j) 510 | if err != nil { 511 | app.logger.Error("updating job store with finished job", zap.Error(err)) 512 | } 513 | 514 | m.Ack() 515 | }) 516 | app.logger.Info("finished job receiver started", zap.Error(err)) 517 | return err 518 | } 519 | 520 | func (app *App) TimeoutJobs() { 521 | t := time.NewTicker(app.jobMaxAge) 522 | for _ = range t.C { 523 | app.jobStore.TimeoutJobs(app.jobMaxAge) 524 | } 525 | } 526 | 527 | // This gets the pubsub topics/subscriptions into the proper state, i.e. checks 528 | // if all of them are available and if not creates them. 529 | func (app *App) initPubSub(credPath string) { 530 | ctx := context.Background() 531 | var psClient *pubsub.Client 532 | 533 | if credPath != "" { 534 | pc, err := pubsub.NewClient(ctx, googleProjectID, option.WithServiceAccountFile(credPath)) 535 | if err != nil { 536 | app.logger.Fatal("pubsub client", zap.Error(err)) 537 | } 538 | psClient = pc 539 | } else { 540 | pc, err := pubsub.NewClient(ctx, googleProjectID) 541 | if err != nil { 542 | app.logger.Fatal("pubsub client", zap.Error(err)) 543 | } 544 | psClient = pc 545 | } 546 | 547 | app.incomingJobsTopic = psClient.Topic(incomingJobsTopicName) 548 | ok, err := app.incomingJobsTopic.Exists(ctx) 549 | if err != nil { 550 | app.logger.Fatal("pubsub topic", zap.Error(err), zap.String("name", incomingJobsTopicName)) 551 | } else if !ok { 552 | t, err := psClient.CreateTopic(ctx, incomingJobsTopicName) 553 | if err != nil { 554 | app.logger.Fatal("pubsub topic", zap.Error(err), zap.String("name", incomingJobsTopicName)) 555 | } 556 | app.incomingJobsTopic = t 557 | } 558 | 559 | incSub := psClient.Subscription(incomingJobsSubscriptionName) 560 | ok, err = incSub.Exists(ctx) 561 | if err != nil { 562 | app.logger.Fatal("pubsub subscription", zap.Error(err), zap.String("name", incomingJobsSubscriptionName)) 563 | } else if !ok { 564 | subConfig := pubsub.SubscriptionConfig{ 565 | Topic: app.incomingJobsTopic, 566 | AckDeadline: 120 * time.Second, 567 | } 568 | _, err := psClient.CreateSubscription(ctx, incomingJobsSubscriptionName, subConfig) 569 | if err != nil { 570 | app.logger.Fatal("pubsub subscription", zap.Error(err), zap.String("name", incomingJobsSubscriptionName)) 571 | } 572 | } 573 | 574 | finTopic := psClient.Topic(finishedJobsTopicName) 575 | ok, err = finTopic.Exists(ctx) 576 | if err != nil { 577 | app.logger.Fatal("pubsub topic", zap.Error(err), zap.String("name", finishedJobsTopicName)) 578 | } else if !ok { 579 | _, err := psClient.CreateTopic(ctx, finishedJobsTopicName) 580 | if err != nil { 581 | app.logger.Fatal("pubsub topic", zap.Error(err), zap.String("name", finishedJobsTopicName)) 582 | } 583 | } 584 | 585 | app.finishedJobsSubscription = psClient.Subscription(finishedJobsSubscriptionName) 586 | ok, err = app.finishedJobsSubscription.Exists(ctx) 587 | if err != nil { 588 | app.logger.Fatal("pubsub subscription", zap.Error(err), zap.String("name", finishedJobsSubscriptionName)) 589 | } else if !ok { 590 | subConfig := pubsub.SubscriptionConfig{ 591 | Topic: finTopic, 592 | AckDeadline: 120 * time.Second, 593 | } 594 | s, err := psClient.CreateSubscription(ctx, finishedJobsSubscriptionName, subConfig) 595 | if err != nil { 596 | app.logger.Fatal("pubsub subscription", zap.Error(err), zap.String("name", finishedJobsSubscriptionName)) 597 | } 598 | app.finishedJobsSubscription = s 599 | } 600 | } 601 | 602 | var ( 603 | incomingJobsTopicName = os.Getenv("INCOMING_JOBS_TOPIC") 604 | finishedJobsTopicName = os.Getenv("FINISHED_JOBS_TOPIC") 605 | incomingJobsSubscriptionName = os.Getenv("INCOMING_JOBS_SUBSCRIPTION") 606 | finishedJobsSubscriptionName = os.Getenv("FINISHED_JOBS_SUBSCRIPTION") 607 | googleProjectID = os.Getenv("GOOGLE_PROJECT_ID") 608 | ) 609 | 610 | func main() { 611 | app := &App{} 612 | 613 | if os.Getenv("DEBUG") == "1" { 614 | logger, err := zap.NewDevelopment() 615 | if err != nil { 616 | log.Fatalf("failed to initialize logger: %s", err) 617 | } 618 | app.logger = logger 619 | } else { 620 | logger, err := zap.NewProduction() 621 | if err != nil { 622 | log.Fatalf("failed to initialize logger: %s", err) 623 | } 624 | app.logger = logger 625 | } 626 | 627 | sh := os.Getenv("STATSD_URI") 628 | if sh == "" { 629 | sh = "127.0.0.1:8125" 630 | } 631 | c, err := statsd.NewBuffered(sh, 100) 632 | if err != nil { 633 | app.logger.Fatal("failed to create statsink", zap.Error(err)) 634 | } 635 | app.stats = c 636 | 637 | // Transport for the client that talks to the IRI instance(s). 638 | tr := &http.Transport{ 639 | DialContext: (&net.Dialer{ 640 | Timeout: 10 * time.Second, 641 | KeepAlive: 30 * time.Second, 642 | }).DialContext, 643 | MaxIdleConns: 100, 644 | IdleConnTimeout: 90 * time.Second, 645 | TLSHandshakeTimeout: 10 * time.Second, 646 | ExpectContinueTimeout: 1 * time.Second, 647 | } 648 | client := &http.Client{ 649 | Transport: tr, 650 | } 651 | 652 | app.iriClient = giota.NewAPI(os.Getenv("IRI_URI"), client) 653 | 654 | credPath := os.Getenv("GOOGLE_CREDENTIALS_FILE") 655 | app.initPubSub(credPath) 656 | 657 | jobMaxAge, err := time.ParseDuration(os.Getenv("JOB_MAX_AGE")) 658 | if err == nil { 659 | app.jobMaxAge = jobMaxAge 660 | } else { 661 | app.jobMaxAge = 5 * time.Minute 662 | } 663 | 664 | js, err := job.NewGCloudDataStore(googleProjectID, credPath) 665 | if err != nil { 666 | app.logger.Fatal("init job store", zap.Error(err)) 667 | } 668 | app.jobStore = js 669 | 670 | r := httprouter.New() 671 | r.POST("/api/v1/commands", app.PostCommands) 672 | r.GET("/api/v1/jobs/:id", app.GetJobsID) 673 | r.GET("/api/v1/status", app.GetAPIStatus) 674 | 675 | // XXX: Make this configurable. 676 | app.cmdLimiter = NewCmdLimiter(map[string]int64{"attachToTangle": 1}, 5) 677 | 678 | n := negroni.New() 679 | 680 | recov := negroni.NewRecovery() 681 | recov.PrintStack = false 682 | n.Use(recov) 683 | lm, err := NewLoggerMiddleware(app.logger) 684 | if err != nil { 685 | app.logger.Fatal("init logger middleware", zap.Error(err)) 686 | } 687 | n.Use(lm) 688 | n.Use(ContentTypeEnforcer("application/json", "application/x-www-form-urlencoded")) 689 | 690 | as, err := auth.NewGCloudDataStore(googleProjectID, credPath) 691 | if err != nil { 692 | app.logger.Fatal("init auth store", zap.Error(err)) 693 | } 694 | 695 | auth := NewAuthMiddleware(as) 696 | n.Use(auth) 697 | 698 | hardLimit, err := strconv.ParseInt(os.Getenv("REQUESTS_PER_MINUTE"), 10, 64) 699 | if err == nil && hardLimit > 0 { 700 | limiter := tollbooth.NewLimiter(hardLimit, time.Minute, nil) 701 | n.Use(LimitHandler(limiter)) 702 | } 703 | 704 | co := cors.New(cors.Options{ 705 | AllowedOrigins: []string{"*"}, 706 | AllowedMethods: []string{"HEAD", "GET", "POST", "PUT", "PATCH", "DELETE"}, 707 | AllowedHeaders: []string{"Authorization", "Content-Type", "X-IOTA-API-Version"}, 708 | AllowCredentials: true, 709 | }) 710 | n.Use(co) 711 | n.UseHandler(r) 712 | 713 | listenAddr := os.Getenv("LISTEN_ADDRESS") 714 | if listenAddr == "" { 715 | listenAddr = "0.0.0.0:8080" 716 | } 717 | 718 | srv := &http.Server{ 719 | Handler: n, 720 | Addr: listenAddr, 721 | ReadTimeout: 10 * time.Second, 722 | WriteTimeout: 10 * time.Second, 723 | } 724 | 725 | go app.PullFinishedJobs() 726 | go app.TimeoutJobs() 727 | app.logger.Info("starting listener") 728 | app.logger.Fatal("ListenAndServe", zap.Error(srv.ListenAndServe())) 729 | } 730 | -------------------------------------------------------------------------------- /cmd/worker/README.md: -------------------------------------------------------------------------------- 1 | # Worker 2 | 3 | This is the worker that, in its current form, consumes IRI jobs off of 4 | an Amazon SQS queue `INCOMING_QUEUE_NAME` and then puts the finished job into 5 | `FINISHED_QUEUE_NAME`. 6 | 7 | At this point the only command supported is `attachToTangle`, which does the 8 | proof-of-work via the executable at `CCURL_PATH`. 9 | 10 | ## environment variables 11 | 12 | required: 13 | 14 | - `AWS_ACCESS_KEY_ID` 15 | - `AWS_SECRET_ACCESS_KEY` 16 | - `AWS_REGION` 17 | - `INCOMING_QUEUE_NAME` 18 | - `FINISHED_QUEUE_NAME` 19 | - `CCURL_PATH` 20 | 21 | optional: 22 | 23 | - `DEBUG` set to `1` for debug logging 24 | -------------------------------------------------------------------------------- /cmd/worker/worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/iotaledger/sandbox/job" 13 | 14 | "cloud.google.com/go/pubsub" 15 | "github.com/iotaledger/giota" 16 | "go.uber.org/zap" 17 | "golang.org/x/net/context" 18 | "google.golang.org/api/option" 19 | ) 20 | 21 | func (app *App) failJob(j *job.IRIJob, errMsg string) { 22 | j.Error = &job.JobError{Message: errMsg} 23 | j.Status = job.JobStatusFailed 24 | j.FinishedAt = time.Now().Unix() 25 | 26 | jb, err := json.Marshal(j) 27 | if err != nil { 28 | app.logger.Error("marshal job", zap.Error(err)) 29 | return 30 | } 31 | 32 | ctx := context.Background() 33 | r := app.finishedJobsTopic.Publish(ctx, &pubsub.Message{ 34 | Data: jb, 35 | }) 36 | 37 | go func() { 38 | _, err := r.Get(ctx) 39 | if err != nil { 40 | app.logger.Error("publish pubsub message", zap.Error(err)) 41 | } 42 | }() 43 | app.logger.Debug("failJob enqueued", zap.String("errMsg", errMsg), zap.Any("job", j)) 44 | } 45 | 46 | func (app *App) processTransactions(ctx context.Context, attachReq *giota.AttachToTangleRequest) ([]giota.Transaction, error) { 47 | ctx.Err() 48 | 49 | outTxs := []giota.Transaction{} 50 | var prevTxHash *giota.Trytes = nil 51 | 52 | for _, ts := range attachReq.Trytes { 53 | trits := ts.Trytes().Trits() 54 | if prevTxHash == nil { 55 | copy(trits[giota.TrunkTransactionTrinaryOffset:giota.TrunkTransactionTrinaryOffset+giota.TrunkTransactionTrinarySize], attachReq.TrunkTransaction.Trits()) 56 | copy(trits[giota.BranchTransactionTrinaryOffset:giota.BranchTransactionTrinaryOffset+giota.BranchTransactionTrinarySize], attachReq.BranchTransaction.Trits()) 57 | } else { 58 | copy(trits[giota.TrunkTransactionTrinaryOffset:giota.TrunkTransactionTrinaryOffset+giota.TrunkTransactionTrinarySize], prevTxHash.Trits()) 59 | copy(trits[giota.BranchTransactionTrinaryOffset:giota.BranchTransactionTrinaryOffset+giota.BranchTransactionTrinarySize], attachReq.TrunkTransaction.Trits()) 60 | } 61 | 62 | s := time.Now() 63 | 64 | cmd := exec.CommandContext(ctx, app.ccurlPath, strconv.Itoa(app.minWeightMagnitude), string(trits.Trytes())) 65 | out, err := cmd.Output() 66 | if err != nil { 67 | app.logger.Error("ccurl", zap.Error(err)) 68 | if err := ctx.Err(); err == context.DeadlineExceeded { 69 | return nil, fmt.Errorf("job exceeded time quota") 70 | } 71 | return nil, err 72 | } 73 | 74 | outS := string(out) 75 | outTx, err := giota.NewTransaction(giota.Trytes(outS)) 76 | if err != nil { 77 | return outTxs, fmt.Errorf("invalid transaction after ccurl: %s", err) 78 | } 79 | 80 | h := outTx.Hash() 81 | prevTxHash = &h 82 | app.logger.Info("runtime", zap.Duration("ccurl", time.Since(s))) 83 | outTxs = append(outTxs, *outTx) 84 | } 85 | 86 | return outTxs, nil 87 | } 88 | 89 | func (app *App) HandleAttachToTangle(ctx context.Context, j *job.IRIJob) error { 90 | if j.AttachToTangleRequest == nil { 91 | return fmt.Errorf("no attachToTangleRequest supplied") 92 | } 93 | 94 | if len(j.AttachToTangleRequest.Trytes) < 1 { 95 | return fmt.Errorf("no trytes supplied for job") 96 | } 97 | 98 | outTxs, err := app.processTransactions(ctx, j.AttachToTangleRequest) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | j.AttachToTangleRespose = &giota.AttachToTangleResponse{Trytes: outTxs} 104 | j.Status = job.JobStatusFinished 105 | j.FinishedAt = time.Now().Unix() 106 | 107 | jb, err := json.Marshal(j) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | ctx = context.Background() 113 | r := app.finishedJobsTopic.Publish(ctx, &pubsub.Message{ 114 | Data: jb, 115 | }) 116 | 117 | go func() { 118 | _, err := r.Get(ctx) 119 | if err != nil { 120 | app.logger.Error("publish pubsub message", zap.Error(err)) 121 | return 122 | } 123 | app.logger.Debug("published finished job") 124 | }() 125 | 126 | return nil 127 | } 128 | 129 | func (app *App) Worker() error { 130 | ctx := context.Background() 131 | err := app.incomingJobsSubscription.Receive(ctx, func(ctx context.Context, m *pubsub.Message) { 132 | app.logger.Debug("got new incoming job") 133 | j := &job.IRIJob{} 134 | err := json.Unmarshal(m.Data, j) 135 | if err != nil { 136 | app.logger.Error("unmarshal job", zap.Error(err)) 137 | } 138 | 139 | j.StartedAt = time.Now().Unix() 140 | //app.logger.Debug("new job", zap.Any("job", j)) 141 | switch j.Command { 142 | default: 143 | app.failJob(j, fmt.Sprintf("unknown command %q", j.Command)) 144 | case "attachToTangle": 145 | ctx := context.Background() 146 | err := app.HandleAttachToTangle(ctx, j) 147 | if err != nil { 148 | app.failJob(j, err.Error()) 149 | app.logger.Error("attach to tangle", zap.Error(err)) 150 | return 151 | } 152 | } 153 | m.Ack() 154 | }) 155 | return err 156 | } 157 | 158 | type App struct { 159 | ccurlPath string 160 | ccurlTimeout time.Duration 161 | minWeightMagnitude int 162 | 163 | logger *zap.Logger 164 | finishedJobsTopic *pubsub.Topic 165 | incomingJobsSubscription *pubsub.Subscription 166 | } 167 | 168 | var ( 169 | finishedJobsTopicName = os.Getenv("FINISHED_JOBS_TOPIC") 170 | incomingJobsSubscriptionName = os.Getenv("INCOMING_JOBS_SUBSCRIPTION") 171 | googleProjectID = os.Getenv("GOOGLE_PROJECT_ID") 172 | ) 173 | 174 | func (app *App) initPubSub(credPath string) { 175 | ctx := context.Background() 176 | var psClient *pubsub.Client 177 | 178 | if credPath != "" { 179 | pc, err := pubsub.NewClient(ctx, googleProjectID, option.WithServiceAccountFile(credPath)) 180 | if err != nil { 181 | app.logger.Fatal("pubsub client", zap.Error(err)) 182 | } 183 | psClient = pc 184 | } else { 185 | pc, err := pubsub.NewClient(ctx, googleProjectID) 186 | if err != nil { 187 | app.logger.Fatal("pubsub client", zap.Error(err)) 188 | } 189 | psClient = pc 190 | } 191 | 192 | app.finishedJobsTopic = psClient.Topic(finishedJobsTopicName) 193 | ok, err := app.finishedJobsTopic.Exists(ctx) 194 | if err != nil { 195 | app.logger.Fatal("pubsub topic", zap.Error(err), zap.String("name", finishedJobsTopicName)) 196 | } else if !ok { 197 | app.logger.Fatal("pubsub topic does not exist", zap.String("name", finishedJobsTopicName)) 198 | } 199 | 200 | app.incomingJobsSubscription = psClient.Subscription(incomingJobsSubscriptionName) 201 | ok, err = app.incomingJobsSubscription.Exists(ctx) 202 | if err != nil { 203 | app.logger.Fatal("pubsub subscription", zap.Error(err), zap.String("name", incomingJobsSubscriptionName)) 204 | } else if !ok { 205 | app.logger.Fatal("pubsub subscription does not exist", zap.String("name", incomingJobsSubscriptionName)) 206 | } 207 | } 208 | 209 | func main() { 210 | app := &App{} 211 | 212 | if os.Getenv("DEBUG") == "1" { 213 | logger, err := zap.NewDevelopment() 214 | if err != nil { 215 | log.Fatalf("failed to initialize logger: %s", err) 216 | } 217 | app.logger = logger 218 | } else { 219 | logger, err := zap.NewProduction() 220 | if err != nil { 221 | log.Fatalf("failed to initialize logger: %s", err) 222 | } 223 | app.logger = logger 224 | } 225 | 226 | // First setup ccurl 227 | app.ccurlPath = os.Getenv("CCURL_PATH") 228 | if app.ccurlPath == "" { 229 | app.logger.Fatal("$CCURL_PATH not set") 230 | } 231 | 232 | mwm, err := strconv.Atoi(os.Getenv("MIN_WEIGHT_MAGNITUDE")) 233 | if err != nil { 234 | app.logger.Fatal("$MIN_WEIGHT_MAGNITUDE is not a valid number") 235 | } 236 | app.minWeightMagnitude = mwm 237 | 238 | app.ccurlTimeout = 120 * time.Second 239 | to, err := strconv.Atoi(os.Getenv("CCURL_TIMEOUT")) 240 | if err == nil { 241 | app.ccurlTimeout = time.Duration(to) * time.Second 242 | } 243 | 244 | app.initPubSub(os.Getenv("GOOGLE_CREDENTIALS_FILE")) 245 | 246 | // TODO: add graceful shutdown 247 | app.logger.Info("starting worker") 248 | app.Worker() 249 | } 250 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | sand: 2 | build: . 3 | ports: 4 | - "80:8080" 5 | - "443:443" 6 | volumes: 7 | - .:/code 8 | environment: 9 | - IRI_URI=http://85.93.93.110:14265 10 | - PUBSUB_EMULATOR_HOST=pubsub:8085 11 | - DATASTORE_EMULATOR_HOST=datastore:8432 12 | - DATASTORE_PROJECT_ID=iota-sandbox 13 | - INCOMING_JOBS_TOPIC=sandbox-incoming 14 | - INCOMING_JOBS_SUBSCRIPTION=sandbox-incoming-sub 15 | - FINISHED_JOBS_TOPIC=sandbox-finished 16 | - FINISHED_JOBS_SUBSCRIPTION=sandbox-finished-sub 17 | - DEBUG=1 18 | restart: always 19 | links: 20 | - pubsub 21 | - datastore 22 | worker: 23 | build: . 24 | dockerfile: Dockerfile.worker 25 | volumes: 26 | - .:/code 27 | environment: 28 | - DEBUG=1 29 | - PUBSUB_EMULATOR_HOST=pubsub:8085 30 | - INCOMING_JOBS_TOPIC=sandbox-incoming 31 | - INCOMING_JOBS_SUBSCRIPTION=sandbox-incoming-sub 32 | - FINISHED_JOBS_TOPIC=sandbox-finished 33 | - FINISHED_JOBS_SUBSCRIPTION=sandbox-finished-sub 34 | - MIN_WEIGHT_MAGNITUDE=13 35 | - CCURL_PATH=/opt/ccurl 36 | restart: always 37 | privileged: true 38 | links: 39 | - pubsub 40 | pubsub: 41 | image: knarz/pubsub-emulator 42 | datastore: 43 | image: knarz/datastore-emulator 44 | #proxy: 45 | #build: ./proxy 46 | #ports: 47 | #- "14265:14265" 48 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: f4d6115cc2ce196225160b89ce5f45cc0f42385893aa94ed9306949b3bdf2de9 2 | updated: 2017-10-14T19:37:34.022226173+02:00 3 | imports: 4 | - name: cloud.google.com/go 5 | version: ba2534604a6c2b8854d680e50f9132cade92d02f 6 | subpackages: 7 | - compute/metadata 8 | - datastore 9 | - iam 10 | - internal 11 | - internal/atomiccache 12 | - internal/fields 13 | - internal/version 14 | - pubsub 15 | - pubsub/apiv1 16 | - name: github.com/DataDog/datadog-go 17 | version: c74bd0589c83817c93e4eff39ccae69d6c46df9b 18 | subpackages: 19 | - statsd 20 | - name: github.com/didip/tollbooth 21 | version: b80d9db9bfe02236089a2056fa894bcf8773f539 22 | subpackages: 23 | - errors 24 | - libstring 25 | - limiter 26 | - name: github.com/golang/protobuf 27 | version: 130e6b02ab059e7b717a096f397c5b60111cae74 28 | subpackages: 29 | - proto 30 | - protoc-gen-go/descriptor 31 | - ptypes 32 | - ptypes/any 33 | - ptypes/duration 34 | - ptypes/empty 35 | - ptypes/struct 36 | - ptypes/timestamp 37 | - ptypes/wrappers 38 | - name: github.com/googleapis/gax-go 39 | version: 317e0006254c44a0ac427cc52a0e083ff0b9622f 40 | - name: github.com/inconshreveable/mousetrap 41 | version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 42 | - name: github.com/iotaledger/giota 43 | version: cc41675ff9ac3193925e0d0b841a850c4691484d 44 | - name: github.com/julienschmidt/httprouter 45 | version: 975b5c4c7c21c0e3d2764200bf2aa8e34657ae6e 46 | - name: github.com/oleiade/lane 47 | version: 3053869314bb02cb983dc2205da8ea2abe46fa96 48 | - name: github.com/patrickmn/go-cache 49 | version: a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0 50 | - name: github.com/rs/cors 51 | version: eabcc6af4bbe5ad3a949d36450326a2b0b9894b8 52 | - name: github.com/satori/go.uuid 53 | version: 5bf94b69c6b68ee1b541973bb8e1144db23a194b 54 | - name: github.com/spf13/cobra 55 | version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b 56 | - name: github.com/spf13/pflag 57 | version: a9789e855c7696159b7db0db7f440b449edf2b31 58 | - name: github.com/urfave/negroni 59 | version: 5bc66cf1ad89af58511e07e108a31f219ed61012 60 | - name: go.uber.org/atomic 61 | version: 4e336646b2ef9fc6e47be8e21594178f98e5ebcf 62 | - name: go.uber.org/multierr 63 | version: 3c4937480c32f4c13a875a1829af76c98ca3d40a 64 | - name: go.uber.org/zap 65 | version: 35aad584952c3e7020db7b839f6b102de6271f89 66 | subpackages: 67 | - buffer 68 | - internal/bufferpool 69 | - internal/color 70 | - internal/exit 71 | - zapcore 72 | - name: golang.org/x/net 73 | version: a04bdaca5b32abe1c069418fb7088ae607de5bd0 74 | subpackages: 75 | - context 76 | - context/ctxhttp 77 | - http2 78 | - http2/hpack 79 | - idna 80 | - internal/timeseries 81 | - lex/httplex 82 | - trace 83 | - name: golang.org/x/oauth2 84 | version: bb50c06baba3d0c76f9d125c0719093e315b5b44 85 | subpackages: 86 | - google 87 | - internal 88 | - jws 89 | - jwt 90 | - name: golang.org/x/sync 91 | version: 8e0aa688b654ef28caa72506fa5ec8dba9fc7690 92 | subpackages: 93 | - errgroup 94 | - semaphore 95 | - name: golang.org/x/text 96 | version: c01e4764d870b77f8abe5096ee19ad20d80e8075 97 | subpackages: 98 | - secure/bidirule 99 | - transform 100 | - unicode/bidi 101 | - unicode/norm 102 | - name: golang.org/x/time 103 | version: 6dc17368e09b0e8634d71cac8168d853e869a0c7 104 | subpackages: 105 | - rate 106 | - name: google.golang.org/api 107 | version: 9e2045da407b8bc1df9bc5c14d4d1a7e7e182e05 108 | subpackages: 109 | - googleapi/transport 110 | - internal 111 | - iterator 112 | - option 113 | - support/bundler 114 | - transport 115 | - transport/grpc 116 | - transport/http 117 | - name: google.golang.org/appengine 118 | version: a2e0dc829727a4f957a7428b1f322805cfc1f362 119 | subpackages: 120 | - internal 121 | - internal/app_identity 122 | - internal/base 123 | - internal/datastore 124 | - internal/log 125 | - internal/modules 126 | - internal/remote_api 127 | - internal/socket 128 | - internal/urlfetch 129 | - socket 130 | - urlfetch 131 | - name: google.golang.org/genproto 132 | version: f676e0f3ac6395ff1a529ae59a6670878a8371a6 133 | subpackages: 134 | - googleapis/api/annotations 135 | - googleapis/datastore/v1 136 | - googleapis/iam/v1 137 | - googleapis/pubsub/v1 138 | - googleapis/rpc/status 139 | - googleapis/type/latlng 140 | - protobuf/field_mask 141 | - name: google.golang.org/grpc 142 | version: c209cdff16049a924bef4aa5c6f7588f0b42b2fa 143 | subpackages: 144 | - balancer 145 | - codes 146 | - connectivity 147 | - credentials 148 | - credentials/oauth 149 | - grpclb/grpc_lb_v1/messages 150 | - grpclog 151 | - internal 152 | - keepalive 153 | - metadata 154 | - naming 155 | - peer 156 | - resolver 157 | - stats 158 | - status 159 | - tap 160 | - transport 161 | - name: gopkg.in/mgo.v2 162 | version: 3f83fa5005286a7fe593b055f0d7771a7dce4655 163 | subpackages: 164 | - bson 165 | - internal/json 166 | - internal/sasl 167 | - internal/scram 168 | - name: leb.io/hashland 169 | version: 07375b562deaa8d6891f9618a04e94a0b98e2ee7 170 | subpackages: 171 | - keccakpg 172 | testImports: [] 173 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/iotaledger/sandbox 2 | import: 3 | - package: github.com/iotaledger/giota 4 | - package: github.com/julienschmidt/httprouter 5 | - package: github.com/satori/go.uuid 6 | - package: go.uber.org/zap 7 | - package: github.com/didip/tollbooth 8 | - package: cloud.google.com/go/datastore 9 | - package: cloud.google.com/go/pubsub 10 | - package: github.com/urfave/negroni 11 | - package: github.com/oleiade/lane 12 | - package: github.com/rs/cors 13 | -------------------------------------------------------------------------------- /job/job.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/iotaledger/giota" 10 | uuid "github.com/satori/go.uuid" 11 | ) 12 | 13 | type JobStatus string 14 | 15 | const ( 16 | JobStatusQueued = "QUEUED" 17 | JobStatusRunning = "RUNNING" 18 | JobStatusFailed = "FAILED" 19 | JobStatusAborted = "ABORTED" 20 | JobStatusFinished = "FINISHED" 21 | ) 22 | 23 | type IRIJob struct { 24 | ID string `json:"id"` 25 | Status JobStatus `json:"status"` 26 | CreatedAt int64 `json:"createdAt" bson:"createdAt"` 27 | StartedAt int64 `json:"startedAt,omitempty" bson:"startedAt"` 28 | FinishedAt int64 `json:"finishedAt,omitempty" bson:"finishedAt"` 29 | Command string `json:"command"` 30 | AttachToTangleRequest *giota.AttachToTangleRequest `json:"attachToTangleRequest,omitempty" datastore:",noindex"` 31 | AttachToTangleRespose *giota.AttachToTangleResponse `json:"attachToTangleResponse,omitempty" datastore:",noindex"` 32 | Error *JobError `json:"error,omitempty"` 33 | } 34 | 35 | func (ij *IRIJob) UnmarshalJSON(data []byte) error { 36 | var aux struct { 37 | ID string `json:"id"` 38 | Status JobStatus `json:"status"` 39 | CreatedAt int64 `json:"createdAt"` 40 | StartedAt int64 `json:"startedAt,omitempty"` 41 | FinishedAt int64 `json:"finishedAt,omitempty"` 42 | Command string `json:"command"` 43 | AttachToTangleRequest *giota.AttachToTangleRequest `json:"attachToTangleRequest,omitempty" datastore:",noindex"` 44 | AttachToTangleRespose *giota.AttachToTangleResponse `json:"attachToTangleResponse,omitempty" datastore:",noindex"` 45 | Error *JobError `json:"error,omitempty"` 46 | } 47 | 48 | dec := json.NewDecoder(bytes.NewReader(data)) 49 | if err := dec.Decode(&aux); err != nil { 50 | return err 51 | } 52 | 53 | switch aux.Command { 54 | case "attachToTangle": 55 | if aux.AttachToTangleRequest == nil { 56 | return fmt.Errorf("attachToTangleRequest object missing") 57 | } 58 | default: 59 | return fmt.Errorf("no command supplied") 60 | } 61 | 62 | ij.ID = aux.ID 63 | ij.Status = aux.Status 64 | ij.CreatedAt = aux.CreatedAt 65 | ij.StartedAt = aux.StartedAt 66 | ij.FinishedAt = aux.FinishedAt 67 | ij.Command = aux.Command 68 | ij.AttachToTangleRequest = aux.AttachToTangleRequest 69 | ij.AttachToTangleRespose = aux.AttachToTangleRespose 70 | ij.Error = aux.Error 71 | 72 | return nil 73 | } 74 | 75 | func NewIRIJob(cmd string) *IRIJob { 76 | return &IRIJob{ 77 | ID: uuid.NewV4().String(), 78 | Status: JobStatusQueued, 79 | CreatedAt: time.Now().Unix(), 80 | Command: cmd, 81 | } 82 | } 83 | 84 | type JobError struct { 85 | Message string `json:"message"` 86 | } 87 | 88 | func (j *JobError) String() string { 89 | return j.Message 90 | } 91 | 92 | func (j *JobError) Error() string { 93 | return j.Message 94 | } 95 | -------------------------------------------------------------------------------- /job/jobstore.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | //uuid "github.com/satori/go.uuid" 10 | "gopkg.in/mgo.v2" 11 | "gopkg.in/mgo.v2/bson" 12 | 13 | "cloud.google.com/go/datastore" 14 | "google.golang.org/api/option" 15 | ) 16 | 17 | type JobStore interface { 18 | InsertJob(*IRIJob) (string, error) 19 | SelectJob(string) (*IRIJob, error) 20 | UpdateJob(string, *IRIJob) (*IRIJob, error) 21 | TimeoutJobs(time.Duration) error 22 | JobFailureRate(time.Duration) (int64, float64) 23 | } 24 | 25 | var ( 26 | ErrJobNotFound = errors.New("could not find job") 27 | ) 28 | 29 | type MemoryStore struct { 30 | mut *sync.Mutex 31 | db map[string]*IRIJob 32 | } 33 | 34 | func NewMemoryStore() *MemoryStore { 35 | return &MemoryStore{ 36 | mut: &sync.Mutex{}, 37 | db: make(map[string]*IRIJob), 38 | } 39 | } 40 | 41 | func (ms *MemoryStore) InsertJob(it *IRIJob) (string, error) { 42 | ms.mut.Lock() 43 | ms.db[it.ID] = it 44 | ms.mut.Unlock() 45 | 46 | return it.ID, nil 47 | } 48 | 49 | func (ms *MemoryStore) SelectJob(id string) (*IRIJob, error) { 50 | ms.mut.Lock() 51 | defer ms.mut.Unlock() 52 | 53 | if ms.db[id] == nil { 54 | return nil, ErrJobNotFound 55 | } 56 | 57 | return ms.db[id], nil 58 | } 59 | 60 | func (ms *MemoryStore) UpdateJob(id string, it *IRIJob) (*IRIJob, error) { 61 | ms.mut.Lock() 62 | defer ms.mut.Unlock() 63 | 64 | ms.db[id] = it 65 | 66 | return ms.db[id], nil 67 | } 68 | 69 | func (ms *MemoryStore) TimeoutJobs(d time.Duration) error { 70 | ms.mut.Lock() 71 | defer ms.mut.Unlock() 72 | t := time.Now().Add(-d).Unix() 73 | 74 | for k, _ := range ms.db { 75 | if ms.db[k].CreatedAt <= t && ms.db[k].Status == JobStatusQueued { 76 | ms.db[k].Status = JobStatusFailed 77 | ms.db[k].Error = &JobError{Message: "timed out"} 78 | } 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func (ms *MemoryStore) JobFailureRate(d time.Duration) (int64, float64) { 85 | ms.mut.Lock() 86 | defer ms.mut.Unlock() 87 | t := time.Now().Add(-d).Unix() 88 | failed := 0.0 89 | finished := 0.0 90 | count := int64(0) 91 | 92 | for k, _ := range ms.db { 93 | if ms.db[k].CreatedAt >= t { 94 | switch ms.db[k].Status { 95 | case JobStatusFinished: 96 | finished += 1.0 97 | case JobStatusFailed: 98 | failed += 1.0 99 | } 100 | count += 1 101 | } 102 | } 103 | 104 | if failed+finished == 0.0 { 105 | return count, 0.0 106 | } 107 | 108 | return count, failed / (failed + finished) 109 | } 110 | 111 | type GCloudDataStore struct { 112 | client *datastore.Client 113 | } 114 | 115 | func NewGCloudDataStore(projectID, credPath string) (*GCloudDataStore, error) { 116 | ctx := context.Background() 117 | var dsClient *datastore.Client 118 | if credPath != "" { 119 | c, err := datastore.NewClient(ctx, projectID, option.WithServiceAccountFile(credPath)) 120 | if err != nil { 121 | return nil, err 122 | } 123 | dsClient = c 124 | } else { 125 | c, err := datastore.NewClient(ctx, projectID) 126 | if err != nil { 127 | return nil, err 128 | } 129 | dsClient = c 130 | } 131 | 132 | g := &GCloudDataStore{client: dsClient} 133 | return g, nil 134 | } 135 | 136 | func (g *GCloudDataStore) InsertJob(it *IRIJob) (string, error) { 137 | ctx := context.Background() 138 | 139 | k := datastore.NameKey("Job", it.ID, nil) 140 | if _, err := g.client.Put(ctx, k, it); err != nil { 141 | return "", err 142 | } 143 | 144 | return it.ID, nil 145 | } 146 | 147 | func (g *GCloudDataStore) SelectJob(id string) (*IRIJob, error) { 148 | ctx := context.Background() 149 | 150 | it := &IRIJob{} 151 | k := datastore.NameKey("Job", id, nil) 152 | if err := g.client.Get(ctx, k, it); err != nil { 153 | if err == datastore.ErrNoSuchEntity { 154 | return nil, ErrJobNotFound 155 | } 156 | 157 | return nil, err 158 | } 159 | 160 | return it, nil 161 | } 162 | 163 | func (g *GCloudDataStore) UpdateJob(id string, it *IRIJob) (*IRIJob, error) { 164 | if _, err := g.InsertJob(it); err != nil { 165 | return nil, err 166 | } 167 | 168 | return it, nil 169 | } 170 | 171 | func (g *GCloudDataStore) TimeoutJobs(d time.Duration) error { 172 | ctx := context.Background() 173 | t := time.Now().Add(-d).Unix() 174 | 175 | q := datastore.NewQuery("Job").Filter("CreatedAt <=", t).Filter("Status =", JobStatusQueued) 176 | var ents []IRIJob 177 | if _, err := g.client.GetAll(ctx, q, &ents); err != nil { 178 | return err 179 | } 180 | 181 | for i := range ents { 182 | ents[i].Status = JobStatusFailed 183 | ents[i].Error = &JobError{Message: "timed out"} 184 | 185 | _, _ = g.UpdateJob(ents[i].ID, &ents[i]) 186 | } 187 | 188 | return nil 189 | } 190 | 191 | func (g *GCloudDataStore) JobFailureRate(d time.Duration) (int64, float64) { 192 | ctx := context.Background() 193 | t := time.Now().Add(-d).Unix() 194 | 195 | failed := 0.0 196 | finished := 0.0 197 | count := int64(0) 198 | 199 | qFail := datastore.NewQuery("Job").Filter("createdAt >=", t).Filter("jobStatus =", JobStatusFailed) 200 | qFin := datastore.NewQuery("Job").Filter("createdAt >=", t).Filter("jobStatus =", JobStatusFinished) 201 | qCount := datastore.NewQuery("Job").Filter("createdAt >=", t) 202 | 203 | if cFail, err := g.client.Count(ctx, qFail); err == nil { 204 | failed = float64(cFail) 205 | } 206 | 207 | if cFin, err := g.client.Count(ctx, qFin); err == nil { 208 | finished = float64(cFin) 209 | } 210 | 211 | if c, err := g.client.Count(ctx, qCount); err == nil { 212 | count = int64(c) 213 | } 214 | 215 | if failed+finished == 0.0 { 216 | return count, 0.0 217 | } 218 | 219 | return count, failed / (failed + finished) 220 | } 221 | 222 | type MongoStore struct { 223 | session *mgo.Session 224 | collection *mgo.Collection 225 | } 226 | 227 | func NewMongoStore(uri string, db string) (*MongoStore, error) { 228 | session, err := mgo.Dial(uri) 229 | if err != nil { 230 | return nil, err 231 | } 232 | session.SetSafe(&mgo.Safe{}) 233 | c := session.DB(db).C("jobs") 234 | 235 | c.EnsureIndex(mgo.Index{Key: []string{"id"}, Unique: true, DropDups: false}) 236 | 237 | m := &MongoStore{session: session, collection: c} 238 | 239 | return m, nil 240 | } 241 | 242 | func (ms *MongoStore) InsertJob(it *IRIJob) (string, error) { 243 | err := ms.collection.Insert(it) 244 | if err != nil { 245 | return "", err 246 | } 247 | return it.ID, nil 248 | } 249 | 250 | func (ms *MongoStore) SelectJob(id string) (*IRIJob, error) { 251 | var r IRIJob 252 | err := ms.collection.Find(bson.M{"id": id}).One(&r) 253 | switch { 254 | case err == mgo.ErrNotFound: 255 | return nil, ErrJobNotFound 256 | case err != nil: 257 | return nil, err 258 | default: 259 | return &r, nil 260 | } 261 | } 262 | 263 | func (ms *MongoStore) UpdateJob(id string, it *IRIJob) (*IRIJob, error) { 264 | err := ms.collection.Update(bson.M{"id": id}, it) 265 | if err != nil { 266 | return nil, err 267 | } 268 | return it, nil 269 | } 270 | 271 | func (ms *MongoStore) TimeoutJobs(d time.Duration) error { 272 | t := time.Now().Add(-d).Unix() 273 | _, err := ms.collection.UpdateAll( 274 | bson.M{ 275 | "createdAt": bson.M{"$lte": t}, 276 | "status": JobStatusQueued, 277 | }, 278 | bson.M{ 279 | "$set": bson.M{ 280 | "status": JobStatusFailed, 281 | "error": bson.M{"message": "timed out"}, 282 | }, 283 | }, 284 | ) 285 | 286 | return err 287 | } 288 | 289 | func (ms *MongoStore) JobFailureRate(d time.Duration) (int64, float64) { 290 | t := time.Now().Add(d).Unix() 291 | failed := 0.0 292 | finished := 0.0 293 | count := int64(0) 294 | 295 | p := ms.collection.Pipe( 296 | []bson.M{ 297 | {"$match": bson.M{"createdAt": bson.M{"$gte": t}}}, 298 | {"$group": bson.M{"_id": "$status", "counts": bson.M{"$sum": 1}}}, 299 | }, 300 | ) 301 | if p == nil { 302 | return 0, 0.0 // ? 303 | } 304 | 305 | var s struct { 306 | Status string `bson:"_id"` 307 | Counts int64 `bson:"counts"` 308 | } 309 | 310 | it := p.Iter() 311 | 312 | for it.Next(&s) { 313 | switch s.Status { 314 | case JobStatusFailed: 315 | failed = float64(s.Counts) 316 | case JobStatusFinished: 317 | finished = float64(s.Counts) 318 | } 319 | count += s.Counts 320 | } 321 | 322 | if failed+finished == 0.0 { 323 | return count, 0.0 324 | } 325 | 326 | return count, failed / (failed + finished) 327 | } 328 | -------------------------------------------------------------------------------- /proxy-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: haproxy 5 | labels: 6 | app: haproxy 7 | tier: backend 8 | role: standalone 9 | spec: 10 | ports: 11 | # the port that this service should serve on 12 | - port: 14265 13 | protocol: TCP 14 | targetPort: 14265 15 | selector: 16 | app: haproxy 17 | tier: backend 18 | role: standalone 19 | --- 20 | apiVersion: extensions/v1beta1 21 | kind: Deployment 22 | metadata: 23 | name: haproxy 24 | spec: 25 | replicas: 2 26 | template: 27 | metadata: 28 | labels: 29 | app: haproxy 30 | name: haproxy 31 | spec: 32 | containers: 33 | - image: haproxy:alpine 34 | imagePullPolicy: Always 35 | name: haproxy 36 | ports: 37 | - containerPort: 14265 38 | name: iotaport 39 | protocol: TCP 40 | volumeMounts: 41 | - name: config 42 | mountPath: /usr/local/etc/haproxy 43 | readOnly: true 44 | volumes: 45 | name: config 46 | secret: 47 | secretName: configs 48 | items: 49 | key: haproxy 50 | path: haproxy.cfg 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /sandbox-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: sandbox 5 | labels: 6 | app: sandbox 7 | tier: frontend 8 | role: standalone 9 | spec: 10 | ports: 11 | - port: 80 12 | name: http 13 | targetPort: 8080 14 | type: LoadBalancer 15 | selector: 16 | app: sandbox 17 | tier: frontend 18 | role: standalone 19 | --- 20 | apiVersion: extensions/v1beta1 21 | kind: Deployment 22 | metadata: 23 | name: sandbox 24 | spec: 25 | replicas: 2 26 | template: 27 | metadata: 28 | labels: 29 | app: sandbox 30 | tier: frontend 31 | role: standalone 32 | spec: 33 | containers: 34 | - env: 35 | - name: FOO 36 | value: BAR 37 | - name: INCOMING_QUEUE_NAME 38 | value: sandbox-incoming 39 | - name: FINISHED_QUEUE_NAME 40 | value: sandbox-finished 41 | name: sandbox 42 | image: iotaledger/sandbox 43 | resources: 44 | requests: 45 | #cpu: 500m 46 | memory: 512Mi 47 | ports: 48 | - containerPort: 8080 49 | -------------------------------------------------------------------------------- /stats-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: dd-agent 5 | labels: 6 | app: dd-agent 7 | tier: backend 8 | role: standalone 9 | spec: 10 | ports: 11 | # the port that this service should serve on 12 | - port: 8125 13 | protocol: UDP 14 | targetPort: 8125 15 | selector: 16 | app: dd-agent 17 | tier: backend 18 | role: standalone 19 | --- 20 | apiVersion: extensions/v1beta1 21 | kind: DaemonSet 22 | metadata: 23 | name: dd-agent 24 | spec: 25 | template: 26 | metadata: 27 | labels: 28 | app: dd-agent 29 | name: dd-agent 30 | spec: 31 | containers: 32 | - image: datadog/docker-dd-agent:kubernetes 33 | imagePullPolicy: Always 34 | name: dd-agent 35 | ports: 36 | - containerPort: 8125 37 | name: dogstatsdport 38 | protocol: UDP 39 | env: 40 | - name: API_KEY 41 | valueFrom: 42 | secretKeyRef: 43 | name: api-keys 44 | key: datadog 45 | - name: KUBERNETES 46 | value: "YES" 47 | volumeMounts: 48 | - name: dockersocket 49 | mountPath: /var/run/docker.sock 50 | - name: procdir 51 | mountPath: /host/proc 52 | readOnly: true 53 | - name: cgroups 54 | mountPath: /host/sys/fs/cgroup 55 | readOnly: true 56 | volumes: 57 | - hostPath: 58 | path: /var/run/docker.sock 59 | name: dockersocket 60 | - hostPath: 61 | path: /proc 62 | name: procdir 63 | - hostPath: 64 | path: /sys/fs/cgroup 65 | name: cgroups 66 | --------------------------------------------------------------------------------