├── .gitignore ├── Makefile ├── README.md ├── api ├── auth.go ├── cors.go ├── query.go ├── setup.go ├── utils.go └── version.go ├── cmd └── joe │ ├── doc.go │ ├── main.go │ ├── new_user.go │ ├── setup.go │ └── vars.go ├── doc ├── markdown.go └── templates │ ├── compiled.go │ └── doc.md ├── go.mod ├── go.sum ├── joe.service └── models ├── cache.go ├── query.go ├── results.go ├── setup.go ├── user.go └── view.go /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .idea 3 | test-unit 4 | test-unit.pub 5 | build 6 | queries 7 | users 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: assets 2 | @mkdir -p build 3 | @go build -o build/joe cmd/joe/*.go 4 | @ls -la build/joe 5 | 6 | assets: 7 | @rm -rf doc/templates/compiled.go 8 | @go-bindata -o doc/templates/compiled.go -pkg templates doc/templates/ 9 | 10 | install: 11 | @cp build/joe /usr/local/bin/ 12 | 13 | clean: 14 | @rm -rf build -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Joe 2 | 3 | Joe is a tool to automatically build a REST API, its documentation and charts around SQL queries and their resulting 4 | data. 5 | 6 | In a way it is an anti-[ORM](https://en.wikipedia.org/wiki/Object-relational_mapping): its purpose is to help backend engineers versioning 7 | , annotating, exposing and charting the queries that can't be implemented or aren't worth implementing in the backend 8 | main business logic and that they would normally keep on .txt or .sql files. 9 | 10 | ## How to Install 11 | 12 | ```sh 13 | go get -u github.com/evilsocket/joe/cmd/joe 14 | ``` 15 | 16 | ## Example 17 | 18 | First create an `/etc/joe/joe.conf` configuration file with the access credentials for the database: 19 | 20 | ```conf 21 | # CHANGE THIS: use a complex secret for the JWT token generation 22 | API_SECRET=02zygnJs5e0bBLJjaHCinWTjfRdheTYO 23 | 24 | DB_HOST=joe-mysql 25 | DB_DRIVER=mysql 26 | DB_USER=joe 27 | DB_PASSWORD=joe 28 | DB_NAME=joe 29 | DB_PORT=3306 30 | ``` 31 | 32 | Then create the `admin` user (this command will generate the file `/etc/joe/users/admin.yml`): 33 | 34 | ```sh 35 | sudo mkdir -p /etc/joe/users 36 | sudo joe -new-user admin -token-ttl 6 # JWT tokens for this user expire after 6 hours 37 | ``` 38 | 39 | For query and chart examples [you can check this repository](https://github.com/evilsocket/pwngrid-queries-joe). Once 40 | you have your `/etc/joe/queries` folder with your queries, you can: 41 | 42 | Generate markdown documentation: 43 | 44 | ```sh 45 | joe -doc /path/to/document.md 46 | ``` 47 | 48 | Start the joe server: 49 | 50 | ```sh 51 | joe -conf /etc/joe/joe.conf -data /etc/joe/queries -users /etc/joe/users 52 | ``` 53 | 54 | ## Why the name Joe? 55 | 56 | The software is a very generic middleware that doesn't have a very specific business logic but adapts to the queries 57 | and the data, so one of the most generic and short names looked like a good idea. You can also think about "Joe" as 58 | that coworker you can always ask to run a query on the backend for you. 59 | 60 | ## License 61 | 62 | `joe` is made with ♥ by [@evilsocket](https://twitter.com/evilsocket) and it is released under the GPL3 license. -------------------------------------------------------------------------------- /api/auth.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/evilsocket/islazy/log" 7 | "github.com/evilsocket/joe/models" 8 | "net/http" 9 | "os" 10 | "time" 11 | 12 | "github.com/dgrijalva/jwt-go" 13 | ) 14 | 15 | var ( 16 | ErrTokenClaims = errors.New("can't extract claims from jwt token") 17 | ErrTokenInvalid = errors.New("jwt token not valid") 18 | ErrTokenExpired = errors.New("jwt token expired") 19 | ErrTokenIncomplete = errors.New("jwt token is missing required fields") 20 | ErrTokenUnauthorized = errors.New("jwt token authorized field is false (?!)") 21 | ) 22 | 23 | func validateToken(header string) (jwt.MapClaims, error) { 24 | token, err := jwt.Parse(header, func(token *jwt.Token) (interface{}, error) { 25 | if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { 26 | return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) 27 | } 28 | return []byte(os.Getenv("API_SECRET")), nil 29 | }) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | claims, ok := token.Claims.(jwt.MapClaims) 35 | if !ok { 36 | return nil, ErrTokenClaims 37 | } else if !token.Valid { 38 | return nil, ErrTokenInvalid 39 | } 40 | 41 | required := []string{ 42 | "expires_at", 43 | "username", 44 | "authorized", 45 | } 46 | for _, req := range required { 47 | if _, found := claims[req]; !found { 48 | return nil, ErrTokenIncomplete 49 | } 50 | } 51 | 52 | log.Debug("%+v", claims) 53 | 54 | if expiresAt, err := time.Parse(time.RFC3339, claims["expires_at"].(string)); err != nil { 55 | return nil, ErrTokenExpired 56 | } else if expiresAt.Before(time.Now()) { 57 | return nil, ErrTokenExpired 58 | } else if claims["authorized"].(bool) != true { 59 | return nil, ErrTokenUnauthorized 60 | } 61 | return claims, err 62 | } 63 | 64 | func getUser(r *http.Request) *models.User { 65 | client := clientIP(r) 66 | tokenHeader := reqToken(r) 67 | if tokenHeader == "" { 68 | log.Debug("unauthenticated request from %s", client) 69 | return nil 70 | } 71 | 72 | claims, err := validateToken(tokenHeader) 73 | if err != nil { 74 | log.Debug("token error for %s: %v", client, err) 75 | return nil 76 | } 77 | 78 | if u, found := models.Users.Load(claims["username"].(string)); found { 79 | return u.(*models.User) 80 | } 81 | 82 | return nil 83 | } 84 | 85 | // GET|POST /api/v1/auth 86 | func (api *API) Authenticate(w http.ResponseWriter, r *http.Request) { 87 | client := clientIP(r) 88 | params := parseParameters(r) 89 | if username, found := params["user"]; !found { 90 | ERROR(w, http.StatusBadRequest, ErrEmpty) 91 | } else if password, found := params["pass"]; !found { 92 | ERROR(w, http.StatusBadRequest, ErrEmpty) 93 | } else if u, found := models.Users.Load(username); !found { 94 | log.Warning("%s: user %s not found", client, username) 95 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 96 | } else if user := u.(*models.User); user.ValidPassword(password.(string)) == false { 97 | log.Warning("%s: invalid password", client) 98 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 99 | } else { 100 | claims := jwt.MapClaims{} 101 | claims["authorized"] = true 102 | claims["username"] = username 103 | claims["expires_at"] = time.Now().Add(time.Hour * time.Duration(user.TokenTTL)).Format(time.RFC3339) 104 | token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) 105 | if token, err := token.SignedString([]byte(os.Getenv("API_SECRET"))); err != nil { 106 | log.Error("error creating token for %s: %v", client, err) 107 | ERROR(w, http.StatusInternalServerError, ErrEmpty) 108 | } else { 109 | log.Info("%s authenticated successfully", client) 110 | JSON(w, http.StatusOK, map[string]string{ 111 | "token": token, 112 | }) 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /api/cors.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/go-chi/cors" 5 | "net/http" 6 | ) 7 | 8 | func CORS(next http.Handler) http.Handler { 9 | cors := cors.New(cors.Options{ 10 | AllowedOrigins: []string{"*"}, 11 | AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, 12 | AllowedHeaders: []string{"Accept", "Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization"}, 13 | AllowCredentials: true, 14 | MaxAge: 300, 15 | }) 16 | return cors.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | w.Header().Add("X-Frame-Options", "DENY") 18 | w.Header().Add("X-Content-Type-Options", "nosniff") 19 | w.Header().Add("X-XSS-Protection", "1; mode=block") 20 | w.Header().Add("Referrer-Policy", "same-origin") 21 | next.ServeHTTP(w, r) 22 | })) 23 | } 24 | -------------------------------------------------------------------------------- /api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/evilsocket/joe/models" 5 | "github.com/wcharczuk/go-chart" 6 | "net/http" 7 | ) 8 | 9 | // GET /api/v1/queries/ 10 | func (api *API) ListQueries(w http.ResponseWriter, r *http.Request) { 11 | user := getUser(r) 12 | if user == nil { 13 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 14 | return 15 | } 16 | 17 | queries := make([]*models.Query, 0) 18 | models.Queries.Range(func(key, value interface{}) bool { 19 | query := value.(*models.Query) 20 | if query.Authorized(user) { 21 | queries = append(queries, query) 22 | } 23 | return true 24 | }) 25 | 26 | JSON(w, http.StatusOK, queries) 27 | } 28 | 29 | // GET /api/v1/query//view 30 | func (api *API) ShowQuery(w http.ResponseWriter, r *http.Request) { 31 | user := getUser(r) 32 | name, _ := parseName("name", r) 33 | if q := models.FindQuery(name); q == nil { 34 | ERROR(w, http.StatusNotFound, ErrEmpty) 35 | } else if q.Authorized(user) == false { 36 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 37 | } else { 38 | JSON(w, http.StatusOK, q) 39 | } 40 | } 41 | 42 | // GET|POST /api/v1/query/ 43 | func (api *API) RunQuery(w http.ResponseWriter, r *http.Request) { 44 | user := getUser(r) 45 | name, ext := parseName("name", r) 46 | if q := models.FindQuery(name); q == nil { 47 | ERROR(w, http.StatusNotFound, ErrEmpty) 48 | } else if q.Authorized(user) == false { 49 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 50 | } else { 51 | params := parseParameters(r) 52 | if rows, err := q.Query(params); err != nil { 53 | ERROR(w, http.StatusBadRequest, err) 54 | } else if ext == "csv" { 55 | CSV(w, http.StatusOK, rows) 56 | } else if ext == "json" { 57 | JSON(w, http.StatusOK, rows) 58 | } else { 59 | ERROR(w, http.StatusNotFound, ErrEmpty) 60 | } 61 | } 62 | } 63 | 64 | // GET|POST /api/v1/query//explain 65 | func (api *API) ExplainQuery(w http.ResponseWriter, r *http.Request) { 66 | user := getUser(r) 67 | name, _ := parseName("name", r) 68 | if q := models.FindQuery(name); q == nil { 69 | ERROR(w, http.StatusNotFound, ErrEmpty) 70 | } else if q.Authorized(user) == false { 71 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 72 | } else { 73 | params := parseParameters(r) 74 | if rows, err := q.Explain(params); err != nil { 75 | ERROR(w, http.StatusBadRequest, err) 76 | } else { 77 | JSON(w, http.StatusOK, rows) 78 | } 79 | } 80 | } 81 | 82 | // GET|POST /api/v1/query// 83 | func (api *API) RunView(w http.ResponseWriter, r *http.Request) { 84 | user := getUser(r) 85 | name, _ := parseName("name", r) 86 | if query := models.FindQuery(name); query == nil { 87 | ERROR(w, http.StatusNotFound, ErrEmpty) 88 | } else if query.Authorized(user) == false { 89 | ERROR(w, http.StatusUnauthorized, ErrEmpty) 90 | } else { 91 | viewName, viewExt := parseName("view_name", r) 92 | view := query.View(viewName) 93 | if view == nil { 94 | ERROR(w, http.StatusNotFound, ErrEmpty) 95 | return 96 | } 97 | 98 | params := parseParameters(r) 99 | rows, err := query.Query(params) 100 | if err != nil { 101 | ERROR(w, http.StatusBadRequest, err) 102 | return 103 | } 104 | 105 | graph := view.Call(rows) 106 | 107 | if viewExt == "png" { 108 | w.Header().Set("Content-Type", "image/png") 109 | w.WriteHeader(http.StatusOK) 110 | graph.Render(chart.PNG, w) 111 | } else if viewExt == "svg" { 112 | w.Header().Set("Content-Type", "image/svg+xml") 113 | w.WriteHeader(http.StatusOK) 114 | graph.Render(chart.SVG, w) 115 | } else { 116 | ERROR(w, http.StatusNotFound, ErrEmpty) 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /api/setup.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/evilsocket/islazy/log" 5 | "github.com/evilsocket/islazy/tui" 6 | "github.com/evilsocket/joe/models" 7 | "github.com/go-chi/chi" 8 | "github.com/go-chi/chi/middleware" 9 | "net/http" 10 | ) 11 | 12 | type API struct { 13 | Router *chi.Mux 14 | } 15 | 16 | func Setup() (err error, api *API) { 17 | api = &API{ 18 | Router: chi.NewRouter(), 19 | } 20 | 21 | api.Router.Use(CORS) 22 | 23 | api.Router.Use(middleware.DefaultCompress) 24 | 25 | api.Router.Route("/api", func(r chi.Router) { 26 | r.Route("/v1", func(r chi.Router) { 27 | // GET|POST /api/v1/auth 28 | r.Get("/auth", api.Authenticate) 29 | r.Post("/auth", api.Authenticate) 30 | 31 | // GET /api/v1/queries/ 32 | r.Get("/queries", api.ListQueries) 33 | 34 | // GET /api/v1/query//view 35 | r.Get("/query/{name:.+}/view", api.ShowQuery) 36 | 37 | // GET /api/v1/query/ 38 | r.Get("/query/{name:.+}", api.RunQuery) 39 | // POST /api/v1/query/ 40 | r.Post("/query/{name:.+}", api.RunQuery) 41 | 42 | // POST /api/v1/query//explain 43 | r.Post("/query/{name:.+}/explain", api.ExplainQuery) 44 | // GET /api/v1/query//explain 45 | r.Get("/query/{name:.+}/explain", api.ExplainQuery) 46 | 47 | // GET /api/v1/query// 48 | r.Get("/query/{name:.+}/{view_name:.+}", api.RunView) 49 | // POST /api/v1/query// 50 | r.Post("/query/{name:.+}/{view_name:.+}", api.RunView) 51 | }) 52 | }) 53 | 54 | return 55 | } 56 | 57 | func (api *API) Run(addr string) { 58 | log.Info("joe api v%s starting on %s ...", Version, addr) 59 | 60 | models.Queries.Range(func(key, value interface{}) bool { 61 | q := value.(*models.Query) 62 | log.Debug(" %s", tui.Dim(q.Expression)) 63 | log.Debug(" http://%s/api/v1/query/%s(.json|csv)(/explain?)", addr, key) 64 | for name, _ := range q.Views { 65 | log.Debug(" http://%s/api/v1/query/%s/%s(.png|svg)", addr, key, name) 66 | } 67 | return true 68 | }) 69 | 70 | log.Fatal("%v", http.ListenAndServe(addr, api.Router)) 71 | } 72 | -------------------------------------------------------------------------------- /api/utils.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "github.com/evilsocket/islazy/log" 10 | "github.com/evilsocket/joe/models" 11 | "github.com/go-chi/chi" 12 | "net/http" 13 | "strconv" 14 | "strings" 15 | ) 16 | 17 | var ( 18 | ErrEmpty = errors.New("") 19 | ErrUnauthorized = errors.New("unauthorized") 20 | ) 21 | 22 | func clientIP(r *http.Request) string { 23 | address := strings.Split(r.RemoteAddr, ":")[0] 24 | if forwardedFor := r.Header.Get("X-Forwarded-For"); forwardedFor != "" { 25 | address = forwardedFor 26 | } 27 | // https://support.cloudflare.com/hc/en-us/articles/206776727-What-is-True-Client-IP- 28 | if trueClient := r.Header.Get("True-Client-IP"); trueClient != "" { 29 | address = trueClient 30 | } 31 | // handle multiple IPs case 32 | return strings.Trim(strings.Split(address, ",")[0], " ") 33 | } 34 | 35 | func reqToken(r *http.Request) string { 36 | keys := r.URL.Query() 37 | token := keys.Get("token") 38 | if token != "" { 39 | return token 40 | } 41 | bearerToken := r.Header.Get("Authorization") 42 | if parts := strings.Split(bearerToken, " "); len(parts) == 2 { 43 | return parts[1] 44 | } 45 | return "" 46 | } 47 | 48 | func pageNum(r *http.Request) (int, error) { 49 | pageParam := r.URL.Query().Get("p") 50 | if pageParam == "" { 51 | pageParam = "1" 52 | } 53 | return strconv.Atoi(pageParam) 54 | } 55 | 56 | func parseName(paramName string, r *http.Request) (name, ext string) { 57 | ext = "json" 58 | name = chi.URLParam(r, paramName) 59 | parts := strings.Split(name, ".") 60 | if numParts := len(parts); numParts > 1 { 61 | ext = parts[numParts-1] 62 | name = strings.Join(parts[:numParts-1], ".") 63 | } 64 | return 65 | } 66 | 67 | func parseParameters(r *http.Request) map[string]interface{} { 68 | params := make(map[string]interface{}) 69 | 70 | // from POST 71 | if err := r.ParseForm(); err == nil { 72 | for key, values := range r.PostForm { 73 | params[key] = values[0] 74 | } 75 | } else { 76 | log.Warning("error parsing form: %v", err) 77 | } 78 | 79 | // from GET 80 | for name, values := range r.URL.Query() { 81 | params[name] = values[0] 82 | } 83 | 84 | delete(params, "token") 85 | 86 | return params 87 | } 88 | 89 | func CSV(w http.ResponseWriter, statusCode int, rows *models.Results) { 90 | buf := bytes.Buffer{} 91 | wr := csv.NewWriter(&buf) 92 | 93 | if err := wr.Write(rows.ColumnNames); err != nil { 94 | log.Error("error sending response: %v", err) 95 | return 96 | } 97 | 98 | for _, row := range rows.Rows { 99 | values := make([]string, rows.NumColumns) 100 | for idx, col := range rows.ColumnNames { 101 | values[idx] = fmt.Sprintf("%v", row[col]) 102 | } 103 | 104 | if err := wr.Write(values); err != nil { 105 | log.Error("error sending response: %v", err) 106 | return 107 | } 108 | } 109 | wr.Flush() 110 | 111 | w.Header().Set("Content-Type", "text/csv") 112 | w.WriteHeader(statusCode) 113 | 114 | if sent, err := w.Write(buf.Bytes()); err != nil { 115 | log.Error("error sending response: %v", err) 116 | } else { 117 | log.Debug("sent %d bytes of csv response", sent) 118 | } 119 | } 120 | 121 | func JSON(w http.ResponseWriter, statusCode int, data interface{}) { 122 | js, err := json.Marshal(data) 123 | if err != nil { 124 | http.Error(w, err.Error(), http.StatusInternalServerError) 125 | return 126 | } 127 | 128 | w.Header().Set("Content-Type", "application/json") 129 | w.WriteHeader(statusCode) 130 | 131 | if sent, err := w.Write(js); err != nil { 132 | log.Error("error sending response: %v", err) 133 | } else { 134 | log.Debug("sent %d bytes of json response", sent) 135 | } 136 | } 137 | 138 | func ERROR(w http.ResponseWriter, statusCode int, err error) { 139 | if err == nil { 140 | JSON(w, http.StatusBadRequest, nil) 141 | } else if err == ErrEmpty { 142 | w.WriteHeader(statusCode) 143 | } else { 144 | JSON(w, statusCode, struct { 145 | Error string `json:"error"` 146 | }{ 147 | Error: err.Error(), 148 | }) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /api/version.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | const ( 4 | Version = "1.0.0" 5 | ) 6 | -------------------------------------------------------------------------------- /cmd/joe/doc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/evilsocket/islazy/log" 5 | "github.com/evilsocket/joe/doc" 6 | ) 7 | 8 | func makeDoc() { 9 | if docFormat == "markdown" { 10 | if err := doc.ToMarkdown(address, docOutput); err != nil { 11 | log.Fatal("%v", err) 12 | } 13 | } else { 14 | log.Fatal("documentation format '%s' not supported", docFormat) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /cmd/joe/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/evilsocket/islazy/log" 7 | "github.com/evilsocket/joe/api" 8 | "github.com/evilsocket/joe/models" 9 | ) 10 | 11 | func main() { 12 | flag.Parse() 13 | 14 | setup() 15 | defer cleanup() 16 | 17 | if ver { 18 | fmt.Println(api.Version) 19 | return 20 | } else if newUser != "" { 21 | addNewUser() 22 | return 23 | } 24 | 25 | compileViews := docOutput == "" 26 | 27 | if err := models.Setup(confFile, dataPath, usersPath, compileViews); err != nil { 28 | log.Fatal("%v", err) 29 | } 30 | 31 | if docOutput != "" { 32 | makeDoc() 33 | return 34 | } 35 | 36 | if err, server := api.Setup(); err != nil { 37 | log.Fatal("%v", err) 38 | } else { 39 | server.Run(address) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /cmd/joe/new_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/evilsocket/islazy/fs" 5 | "github.com/evilsocket/joe/models" 6 | "golang.org/x/crypto/ssh/terminal" 7 | "path" 8 | "fmt" 9 | "syscall" 10 | ) 11 | 12 | func addNewUser() { 13 | userFile := path.Join(usersPath, fmt.Sprintf("%s.yml", newUser)) 14 | if fs.Exists(userFile) { 15 | fmt.Printf("%s already exists.\n", userFile) 16 | return 17 | } 18 | 19 | fmt.Print("Enter Password: ") 20 | raw, err := terminal.ReadPassword(int(syscall.Stdin)) 21 | if err != nil { 22 | panic(err) 23 | } 24 | 25 | password := string(raw) 26 | user := models.User { 27 | Username: newUser, 28 | Password: password, 29 | TokenTTL: tokenTTL, 30 | } 31 | 32 | if err := models.SaveUser(user, userFile); err != nil { 33 | fmt.Printf("error: %v\n", err) 34 | return 35 | } 36 | 37 | fmt.Printf("Saved to %s\n", userFile) 38 | } 39 | -------------------------------------------------------------------------------- /cmd/joe/setup.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/evilsocket/islazy/log" 5 | "github.com/evilsocket/joe/models" 6 | ) 7 | 8 | func setup() { 9 | if debug { 10 | log.Level = log.DEBUG 11 | } else { 12 | log.Level = log.INFO 13 | } 14 | log.OnFatal = log.ExitOnFatal 15 | } 16 | 17 | func cleanup() { 18 | models.Cleanup() 19 | } 20 | -------------------------------------------------------------------------------- /cmd/joe/vars.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/evilsocket/islazy/log" 6 | ) 7 | 8 | var ( 9 | debug = false 10 | ver = false 11 | address = "127.0.0.1:8080" 12 | confFile = "/etc/joe/joe.conf" 13 | usersPath = "/etc/joe/users" 14 | dataPath = "/etc/joe/queries" 15 | 16 | newUser = "" 17 | tokenTTL = 24 18 | 19 | docOutput = "" 20 | docFormat = "markdown" 21 | ) 22 | 23 | func init() { 24 | flag.BoolVar(&debug, "debug", debug, "Enable debug logs.") 25 | flag.StringVar(&log.Output, "log", log.Output, "Log file path or empty for standard output.") 26 | flag.StringVar(&address, "address", address, "API address.") 27 | flag.StringVar(&confFile, "conf", confFile, "Configuration file.") 28 | flag.StringVar(&usersPath, "users", usersPath, "Path containing user credentials in YML.") 29 | flag.StringVar(&dataPath, "data", dataPath, "Data path.") 30 | 31 | flag.BoolVar(&ver, "version", ver, "Print version and exit.") 32 | 33 | flag.StringVar(&newUser, "new-user", newUser, "Create a new user with the provided username.") 34 | flag.IntVar(&tokenTTL, "token-ttl", tokenTTL, "How many hours a JWT token for this user is valid.") 35 | 36 | flag.StringVar(&docOutput, "doc", docOutput, "Generate the API documentation to this file.") 37 | flag.StringVar(&docFormat, "format", docFormat, "Format of the generated documentation.") 38 | } 39 | -------------------------------------------------------------------------------- /doc/markdown.go: -------------------------------------------------------------------------------- 1 | package doc 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/evilsocket/islazy/log" 7 | "github.com/evilsocket/joe/api" 8 | "github.com/evilsocket/joe/doc/templates" 9 | "github.com/evilsocket/joe/models" 10 | "os" 11 | "text/template" 12 | ) 13 | 14 | var funcMap = template.FuncMap{ 15 | "json": func(arg interface{}) string { 16 | if raw, err := json.MarshalIndent(arg, "", " "); err != nil { 17 | return fmt.Sprintf("template error: %v", err) 18 | } else { 19 | return string(raw) 20 | } 21 | }, 22 | } 23 | 24 | func tpl(name string) (*template.Template, error) { 25 | raw, err := templates.Asset(name) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | t, err := template.New(name).Funcs(funcMap).Parse(string(raw)) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return t, nil 36 | } 37 | 38 | func ToMarkdown(address, fileName string) (err error) { 39 | log.Info("generating markdown documentation for %d queries to %s ...", models.NumQueries, fileName) 40 | t, err := tpl("doc/templates/doc.md") 41 | if err != nil { 42 | return 43 | } 44 | 45 | queries := make([]*models.Query, 0) 46 | models.Queries.Range(func(key, value interface{}) bool { 47 | queries = append(queries, value.(*models.Query)) 48 | return true 49 | }) 50 | 51 | out, err := os.Create(fileName) 52 | if err != nil { 53 | return 54 | } 55 | defer out.Close() 56 | 57 | return t.Execute(out, map[string]interface{}{ 58 | "Address": address, 59 | "Version": api.Version, 60 | "Queries": queries, 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /doc/templates/compiled.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "fmt" 7 | "io" 8 | "strings" 9 | ) 10 | 11 | func bindata_read(data []byte, name string) ([]byte, error) { 12 | gz, err := gzip.NewReader(bytes.NewBuffer(data)) 13 | if err != nil { 14 | return nil, fmt.Errorf("Read %q: %v", name, err) 15 | } 16 | 17 | var buf bytes.Buffer 18 | _, err = io.Copy(&buf, gz) 19 | gz.Close() 20 | 21 | if err != nil { 22 | return nil, fmt.Errorf("Read %q: %v", name, err) 23 | } 24 | 25 | return buf.Bytes(), nil 26 | } 27 | 28 | var _doc_templates_doc_md = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\x6d\x6f\xda\x48\x10\xfe\xee\x5f\x31\x72\xaa\x53\xa2\x4b\x16\x48\x9a\xf6\x8a\xd4\x0f\x69\x43\x73\xa9\x52\xe0\x80\xf6\x5e\xaa\x8a\x2c\xf6\x00\xdb\x98\x5d\x77\x77\x4d\xea\x33\xfe\xef\xa7\x59\x1b\x83\x49\x9b\xea\xfa\xa2\xfb\x72\x12\x12\xde\xd9\xd9\x99\xd9\xe7\x99\x97\xdd\xdb\x83\x97\x0a\xe1\xb7\x04\xb5\x40\xe3\x79\xa3\xb9\x30\x10\xaa\x20\x59\xa0\xb4\x30\xe7\x06\x26\x88\x12\x78\x62\xd5\x82\x5b\x11\xf0\x28\x4a\x61\x86\x12\x35\xb7\x18\xc2\xad\xb0\x73\x78\xfb\x5e\x21\x2c\xb3\x8c\xbd\x41\x6d\x84\x92\x79\xfe\x6e\x7f\x6e\x6d\x6c\xda\x8d\xc6\x4c\xd8\x79\x32\x61\x81\x5a\x34\x70\x29\x22\xa3\x82\x1b\xb4\x8d\xf7\x0a\x0f\x98\xe7\xed\xed\xed\xc1\x45\x67\xb4\xea\xf7\x86\x23\x68\xf0\x58\x34\x96\xad\x06\x4f\xec\x1c\xc0\xf3\xce\x12\x3b\x47\x49\x2e\x2d\x82\x55\x60\xe7\x08\x67\xfd\xcb\xc2\x65\x62\x50\x4b\xbe\x40\xe0\x32\x84\x98\x1b\x73\xab\x74\x08\x42\x82\xd2\x21\x6a\x52\x9f\xa1\x05\x0e\x2f\x7f\x1f\x81\x55\x37\x28\x99\xe7\xad\xfa\x5c\xf3\x05\x5a\xd4\xab\x73\x9c\xf2\x24\xb2\x2b\x6f\x75\x74\x44\x3f\x6f\x05\xd7\x64\xf2\x1a\x56\x30\x96\x4a\xe2\x18\x9c\x8c\x2c\xd7\x64\x14\xf2\x1e\x0c\xf0\x43\x82\xc6\x7a\x1e\x00\x40\x90\xe8\x08\x8e\x8e\x42\x6e\x39\xf8\x64\xe4\x29\x0f\x17\x42\xfe\x44\x67\x8b\x4f\x1f\x08\x8d\x76\xa3\x91\x65\xc0\xce\xc2\x50\xa3\x31\x90\xe7\xdb\x17\xae\x0c\x9b\x58\x49\x83\x9e\x77\x7d\x7d\xfd\xde\x28\xe9\x65\x1e\x80\xef\x6e\xe0\xb7\xc1\x67\x8c\xf9\x5e\x4e\x9b\x15\x78\x15\x6e\x1f\x0a\x0a\x1b\x9e\x77\xe1\xae\x1e\x09\x63\x41\x4d\x1d\x6e\xe5\x1e\xd8\x39\xb7\xc0\x35\x52\xd0\x1a\xa5\x8d\x52\x88\x14\x0f\xd1\x41\x47\x8a\x26\x35\x16\x17\x0e\x55\xa7\x4b\x32\xba\x13\x04\x5c\x02\x0f\x02\x34\x86\x7d\x1e\x84\x5f\xc1\x27\xd6\x94\x16\x7f\x73\x2b\x94\x6c\xc3\x33\xe4\x1a\x35\x30\x56\x90\xc0\xbe\x00\xc5\x87\x75\x1a\x7e\x06\x8d\xb7\x1e\x40\xe6\x1c\xfa\x81\x46\xca\xc0\x31\xb7\x04\xcc\x71\xb3\xf5\xe4\xa8\xd5\x3a\x6a\x3e\x19\xb5\x9a\xed\xd3\xc7\xed\x66\x8b\x3d\xfe\xe5\xe1\xc9\x49\xeb\xf8\xf4\xf4\xe7\x66\xab\xdd\x6c\xfa\x87\xc5\xc1\x24\x0e\xbf\x7c\xf0\xe1\x93\x87\xc7\x27\x3b\x07\x29\xdf\xe8\x88\x55\xf1\x38\x8e\x78\x8a\xda\xac\xb7\x02\x1e\xcc\x69\xaf\x88\x8d\x18\x4b\x63\x5a\xb7\x0e\xd7\x82\x1b\x4c\x8d\xdf\x86\xb7\xe5\x1a\xc0\x8f\xc4\x42\x58\xbf\x5c\xbf\xab\x14\xad\x8d\xfc\x36\x9c\x3c\x6a\x36\x9d\x24\x2f\x3d\x84\x68\x02\x2d\x62\x42\x95\x62\x18\xa9\x18\xca\x18\x60\x92\x96\xcc\x40\xac\x84\xb4\x86\xf9\xd5\x19\x97\xe5\x66\x3b\xb0\xc2\x6b\x1b\x8e\x4f\x6b\xe6\x09\xf8\x94\x0c\x0f\x3b\x57\x9d\xe7\x23\x80\x84\x6d\x70\x02\x6e\x80\x07\x56\x2c\x71\xcc\xed\x21\x24\x8c\x90\xa0\xff\xa9\x90\x33\xd4\xb1\x16\xd2\x89\x03\x95\x48\xab\xd3\x43\x78\xde\x7b\xdd\x1d\xed\x73\x26\xc2\x03\x38\x1b\x82\x44\x7b\xab\xf4\x8d\x81\x17\x83\xde\x2b\x48\xa4\xb0\x06\x12\xb8\xec\x76\x3b\x03\x78\xd9\xbb\xec\x96\xe1\x8f\x8b\xf0\x81\x43\xaf\x0b\x09\x13\x21\x3c\x05\xce\x48\x7d\x2c\x42\xb8\x18\xf4\x5e\xf7\xe1\xd9\x9f\xc5\x4e\x6f\x70\xde\x19\xd0\xaa\xb2\x7d\xde\x19\x3e\x87\xab\xcb\x57\x97\x23\xc8\xdc\x25\xf3\x35\x0c\x4b\x81\xb7\x35\x0c\x26\x5c\x9b\x1d\x22\xc7\x24\x63\x33\xe5\xd7\x50\x29\xe2\xda\xa2\xcd\x2f\xea\xd9\xad\xde\x79\xa5\x22\x63\x6c\x5d\x93\x59\x06\x0f\x78\x99\xd5\xed\xa7\xdb\x19\x4e\x7b\x9a\xcb\x19\x02\x2b\x7b\x2d\xc9\x3e\x55\xc3\x69\x23\xcb\x58\x97\x2f\x30\xcf\x1b\x14\x3a\xf5\xc2\xe1\x5c\xdd\x82\x90\x53\xa5\x17\xae\xb0\x80\x4f\x54\x52\x54\x67\xa5\xec\x8a\x3c\xbd\x53\x9d\x59\x26\xa6\xc0\xa8\x2c\x49\x24\x34\x86\x14\xcd\xd7\xd6\x6c\x75\xbb\x7a\xcd\xee\xc6\xec\x65\x19\x46\x06\xb7\x1d\x7d\x95\x0d\x19\x96\x28\x7d\xb2\x35\x66\xf4\x07\x2c\xaf\x37\xc4\xfa\x34\xd9\xb1\xbc\xcf\xe8\xcc\x8a\x05\x66\x79\x40\xd8\xb0\xf3\x4d\x5d\x15\x24\x11\x5a\xd5\xa0\x70\xcc\xdd\x37\x37\xd6\xa4\x3e\x28\x4a\xe2\x41\x88\x53\x47\x7c\xcd\xc2\x0a\xae\xb3\xcc\x69\xe4\x39\x4d\x13\xe7\x84\x54\xf3\x3c\xcb\xd6\xff\x05\x5e\xc5\xa0\x29\x6f\x0e\xab\x0a\x83\x5d\x2c\xfe\x23\x72\x1d\x7a\x59\xe6\x52\x38\x1d\x5a\x2d\xe4\xac\x08\xee\x1b\xc8\x66\x25\x99\xdf\x9f\xec\x06\x7e\x8c\x23\x2e\xa4\xe7\x0d\xd0\x26\x5a\x82\x46\x43\xfd\x10\xa6\x4a\x03\x97\xd0\xf9\xa3\x7f\x75\x76\xd9\x05\x15\xd3\x6b\x86\xca\x4a\xc9\x9d\x9a\x5a\x70\x21\xab\xc2\xfa\x3f\x39\xee\xaf\xda\x12\xee\xef\x9c\x1f\xf7\x99\xbd\x3f\x65\xbc\xf5\x64\x2e\x87\xbd\x4c\xa2\x88\x1a\xb6\x8f\x1f\x31\x18\x5b\xe1\xc6\xf9\xf1\xe3\x47\x27\xc7\x8f\x4e\x9c\x5c\x26\x8b\xb1\xc6\x40\xe9\xd0\xac\x87\xb7\xbf\x59\x17\x43\xa0\x9a\x22\x9d\x8f\x56\x73\x1a\x23\x7f\xa1\x56\x50\xcc\xd5\x6a\x8a\x4f\x45\x64\x51\x63\xb8\xe5\xd5\xc9\x45\xb8\xfb\x2a\xd8\xd5\xb8\xc1\x74\x1c\xb9\xd7\x5e\x4d\x1c\x73\x6d\x05\xb1\x64\xee\xec\x28\x63\xc4\x24\xc2\x71\xf9\xc4\xa8\x6d\x6a\x9c\xde\x11\xa9\xdb\x3b\x6a\x06\x23\x0c\xec\xb8\x7c\xb5\xf8\xc3\xcb\x57\xfd\xab\xce\xe6\x36\x96\x4f\x22\xdc\x3d\x53\x2a\x93\xac\x18\x9a\x1e\xcd\xc4\xa2\x3c\xc1\x4d\x42\x47\x24\x91\xe8\xb2\xde\x7d\x38\xda\xca\x1d\x57\x06\xe6\x6e\x45\x6c\x34\x0a\xaa\x9d\xc6\x16\xf5\x35\x95\x5a\x7a\xbb\x91\xbb\x93\xef\x9b\x2a\xa4\xa1\xd2\x2d\x2a\x91\x3e\x5f\x88\xa8\x08\xec\x0d\xbd\x0f\x48\xf5\x73\xd3\x78\x73\x91\x3c\xa7\xe5\xda\x50\x9e\xb3\xfd\x58\xce\x56\x66\x39\x3b\xa8\x7a\x0c\x87\x7e\xf7\x02\x94\x86\xe1\x9b\x0b\xd0\x18\x6b\x34\x28\x6d\xd9\x5e\xa6\xc0\xa1\x66\x00\x82\x39\xd7\xd6\x35\xa4\xa2\xf1\x6c\xfb\xaa\x37\x9e\x1a\x68\x5f\xd7\x79\x76\x4c\xfc\xe0\xd6\xf3\x09\x82\x7e\x4c\xff\xb9\x87\x9f\x58\xce\xd6\xfb\xdf\xda\x8e\xfe\xbd\x97\x02\x99\xdd\x8f\x7f\x02\x00\x00\xff\xff\x5a\x17\xc7\x6f\x72\x0f\x00\x00") 29 | 30 | func doc_templates_doc_md() ([]byte, error) { 31 | return bindata_read( 32 | _doc_templates_doc_md, 33 | "doc/templates/doc.md", 34 | ) 35 | } 36 | 37 | // Asset loads and returns the asset for the given name. 38 | // It returns an error if the asset could not be found or 39 | // could not be loaded. 40 | func Asset(name string) ([]byte, error) { 41 | cannonicalName := strings.Replace(name, "\\", "/", -1) 42 | if f, ok := _bindata[cannonicalName]; ok { 43 | return f() 44 | } 45 | return nil, fmt.Errorf("Asset %s not found", name) 46 | } 47 | 48 | // AssetNames returns the names of the assets. 49 | func AssetNames() []string { 50 | names := make([]string, 0, len(_bindata)) 51 | for name := range _bindata { 52 | names = append(names, name) 53 | } 54 | return names 55 | } 56 | 57 | // _bindata is a table, holding each asset generator, mapped to its name. 58 | var _bindata = map[string]func() ([]byte, error){ 59 | "doc/templates/doc.md": doc_templates_doc_md, 60 | } 61 | // AssetDir returns the file names below a certain 62 | // directory embedded in the file by go-bindata. 63 | // For example if you run go-bindata on data/... and data contains the 64 | // following hierarchy: 65 | // data/ 66 | // foo.txt 67 | // img/ 68 | // a.png 69 | // b.png 70 | // then AssetDir("data") would return []string{"foo.txt", "img"} 71 | // AssetDir("data/img") would return []string{"a.png", "b.png"} 72 | // AssetDir("foo.txt") and AssetDir("notexist") would return an error 73 | // AssetDir("") will return []string{"data"}. 74 | func AssetDir(name string) ([]string, error) { 75 | node := _bintree 76 | if len(name) != 0 { 77 | cannonicalName := strings.Replace(name, "\\", "/", -1) 78 | pathList := strings.Split(cannonicalName, "/") 79 | for _, p := range pathList { 80 | node = node.Children[p] 81 | if node == nil { 82 | return nil, fmt.Errorf("Asset %s not found", name) 83 | } 84 | } 85 | } 86 | if node.Func != nil { 87 | return nil, fmt.Errorf("Asset %s not found", name) 88 | } 89 | rv := make([]string, 0, len(node.Children)) 90 | for name := range node.Children { 91 | rv = append(rv, name) 92 | } 93 | return rv, nil 94 | } 95 | 96 | type _bintree_t struct { 97 | Func func() ([]byte, error) 98 | Children map[string]*_bintree_t 99 | } 100 | var _bintree = &_bintree_t{nil, map[string]*_bintree_t{ 101 | "doc": &_bintree_t{nil, map[string]*_bintree_t{ 102 | "templates": &_bintree_t{nil, map[string]*_bintree_t{ 103 | "doc.md": &_bintree_t{doc_templates_doc_md, map[string]*_bintree_t{ 104 | }}, 105 | }}, 106 | }}, 107 | }} 108 | -------------------------------------------------------------------------------- /doc/templates/doc.md: -------------------------------------------------------------------------------- 1 | ## Joe Queries 2 | 3 | This document has been automatically generated with [joe v{{.Version}}](https://github.com/evilsocket/joe). 4 | 5 | ### GET|POST /api/v1/auth 6 | 7 | Authenticate to the API with username and password in order to get a JWT token. 8 | 9 | |Parameter|Default| 10 | |--|--| 11 | | `user` | _none_ | 12 | | `pass` | _none_ | 13 | 14 | #### Request 15 | 16 | curl --data "user=admin&pass=admin" http://{{ .Address }}/api/v1/auth 17 | 18 | #### Response 19 | 20 | ```json 21 | { 22 | "token": "..." 23 | } 24 | ``` 25 | 26 | ### GET /api/v1/queries/ 27 | 28 | Get a list of the queries that are currently loaded in the system and that the user can access. 29 | 30 | #### Request 31 | 32 | curl -H "Authorization: Bearer ..token.." http://{{ .Address }}/api/v1/queries 33 | 34 | #### Response 35 | 36 | ```json 37 | [ 38 | { 39 | "created_at": "2019-11-09T10:57:01.784331255+01:00", 40 | "updated_at": "2019-11-09T10:57:01.784494235+01:00", 41 | "name": "top_players", 42 | "cache": { 43 | "type": 1, 44 | "keys": [ 45 | "limit" 46 | ], 47 | "ttl": 3600 48 | }, 49 | "description": "Top players by access points.", 50 | "defaults": { 51 | "limit": 25 52 | }, 53 | "query": "SELECT u.updated_at as active_at, u.name, u.fingerprint, u.country, COUNT(a.id) AS networks FROM units u INNER JOIN access_points a ON u.id = a.unit_id GROUP BY u.id ORDER BY networks DESC LIMIT {limit}", 54 | "views": { 55 | "bars": "top_players_bars.go" 56 | }, 57 | "access": [ 58 | "admin" 59 | ] 60 | }, 61 | ... 62 | } 63 | ``` 64 | 65 | {{ $address := .Address }} 66 | 67 | {{range .Queries}} 68 | 69 | ### GET /api/v1/query/{{.Name}}/view 70 | 71 | Show information about the {{.Name}} query. 72 | 73 | #### Request 74 | 75 | {{if .AuthRequired }} 76 | curl -H "Authorization: Bearer ..token.." http://{{ $address }}/api/v1/query/{{.Name}}/view 77 | {{else}} 78 | curl http://{{ $address }}/api/v1/query/{{.Name}}/view 79 | {{end}} 80 | 81 | #### Response 82 | 83 | ```json 84 | {{json .}} 85 | ``` 86 | 87 | ### GET|POST /api/v1/query/{{.Name}}(.json|.csv) 88 | 89 | {{.Description}} 90 | 91 | {{if .Parameters }} 92 | |Parameter|Default| 93 | |--|--| 94 | {{range $name, $def := .Parameters }}| `{{$name}}` | {{if $def}}{{$def}}{{else}}_none_{{end}} | 95 | {{end}} 96 | {{end}} 97 | 98 | #### Request 99 | 100 | {{if .AuthRequired }} 101 | curl -H "Authorization: Bearer ..token.." http://{{ $address }}/api/v1/query/{{.Name}}.json{{.QueryString}} 102 | {{else}} 103 | curl http://{{ $address }}/api/v1/query/{{.Name}}.json 104 | {{end}} 105 | 106 | #### Response 107 | 108 | ```json 109 | {{json .}} 110 | ``` 111 | 112 | ### GET|POST /api/v1/query/{{.Name}}/explain 113 | 114 | Return results for an EXPLAIN operation on the {{.Name}} main query. 115 | 116 | {{if .Parameters }} 117 | |Parameter|Default| 118 | |--|--| 119 | {{range $name, $def := .Parameters }}| `{{$name}}` | {{if $def}}{{$def}}{{else}}_none_{{end}} | 120 | {{end}} 121 | {{end}} 122 | 123 | #### Request 124 | 125 | {{if .AuthRequired }} 126 | curl -H "Authorization: Bearer ..token.." http://{{ $address }}/api/v1/query/{{.Name}}/explain{{.QueryString}} 127 | {{else}} 128 | curl http://{{ $address }}/api/v1/query/{{.Name}}/explain{{.QueryString}} 129 | {{end}} 130 | 131 | #### Response 132 | 133 | ```json 134 | { 135 | "cached_at": null, 136 | "exec_time": 2763263, 137 | "num_records": 1, 138 | "records": [ 139 | { 140 | "Extra": "Zero limit", 141 | "filtered": null, 142 | "id": 1, 143 | "key": null, 144 | "key_len": null, 145 | "partitions": null, 146 | "possible_keys": null, 147 | "ref": null, 148 | "rows": null, 149 | "select_type": "SIMPLE", 150 | "table": null, 151 | "type": null 152 | } 153 | ] 154 | } 155 | ``` 156 | 157 | {{ $queryName := .Name }} 158 | {{ $queryParams := .Parameters }} 159 | {{ $queryString := .QueryString }} 160 | {{ $queryAuthRequired := .AuthRequired }} 161 | 162 | {{range $viewName, $viewFile := .Views }} 163 | ### GET /api/v1/query/{{$queryName}}/{{$viewName}}.(png|svg) 164 | 165 | Return a PNG or SVG representation of a {{$viewName}} chart for the {{$queryName}} query. 166 | 167 | {{if $queryParams }} 168 | |Parameter|Default| 169 | |--|--| 170 | {{range $name, $def := $queryParams }}| `{{$name}}` | {{if $def}}{{$def}}{{else}}_none_{{end}} | 171 | {{end}} 172 | {{end}} 173 | 174 | #### Request 175 | 176 | {{if $queryAuthRequired }} 177 | curl -H "Authorization: Bearer ..token.." http://{{ $address }}/api/v1/query/{{$queryName}}/{{$viewName}}.png{{$queryString}} 178 | {{else}} 179 | curl http://{{ $address }}/api/v1/query/{{$queryName}}/{{$viewName}}.png{{$queryString}} 180 | {{end}} 181 | 182 | {{end}} 183 | 184 | {{end}} -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/evilsocket/joe 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/dgrijalva/jwt-go v3.2.0+incompatible 7 | github.com/evilsocket/islazy v1.10.6 8 | github.com/go-chi/chi v4.0.2+incompatible 9 | github.com/go-chi/cors v1.0.0 10 | github.com/go-sql-driver/mysql v1.4.1 11 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect 12 | github.com/joho/godotenv v1.3.0 13 | github.com/jteeuwen/go-bindata v3.0.7+incompatible // indirect 14 | github.com/lib/pq v1.2.0 15 | github.com/wcharczuk/go-chart v2.0.2-0.20190910040548-3a7bc5543113+incompatible 16 | golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4 // indirect 17 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect 18 | gopkg.in/djherbis/times.v1 v1.2.0 19 | gopkg.in/yaml.v2 v2.2.5 20 | ) 21 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 2 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 3 | github.com/evilsocket/islazy v1.10.4 h1:Z5373Kn5Gh2EWch1Tb/Qxb6vyQ7lw704bmKi7sY4Ecs= 4 | github.com/evilsocket/islazy v1.10.4/go.mod h1:OrwQGYg3DuZvXUfmH+KIZDjwTCbrjy48T24TUpGqVVw= 5 | github.com/evilsocket/islazy v1.10.6 h1:MFq000a1ByoumoJWlytqg0qon0KlBeUfPsDjY0hK0bo= 6 | github.com/evilsocket/islazy v1.10.6/go.mod h1:OrwQGYg3DuZvXUfmH+KIZDjwTCbrjy48T24TUpGqVVw= 7 | github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= 8 | github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= 9 | github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0= 10 | github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I= 11 | github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= 12 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 13 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= 14 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= 15 | github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= 16 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 17 | github.com/jteeuwen/go-bindata v3.0.7+incompatible h1:91Uy4d9SYVr1kyTJ15wJsog+esAZZl7JmEfTkwmhJts= 18 | github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs= 19 | github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= 20 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 21 | github.com/wcharczuk/go-chart v2.0.1+incompatible h1:0pz39ZAycJFF7ju/1mepnk26RLVLBCWz1STcD3doU0A= 22 | github.com/wcharczuk/go-chart v2.0.1+incompatible/go.mod h1:PF5tmL4EIx/7Wf+hEkpCqYi5He4u90sw+0+6FhrryuE= 23 | github.com/wcharczuk/go-chart v2.0.2-0.20190910040548-3a7bc5543113+incompatible h1:Bz/7IMIv+MmCANT7drP8zyxHH5xHC0/+smWpcJCCk4M= 24 | github.com/wcharczuk/go-chart v2.0.2-0.20190910040548-3a7bc5543113+incompatible/go.mod h1:PF5tmL4EIx/7Wf+hEkpCqYi5He4u90sw+0+6FhrryuE= 25 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 26 | golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4 h1:PDpCLFAH/YIX0QpHPf2eO7L4rC2OOirBrKtXTLLiNTY= 27 | golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 28 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= 29 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 30 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 31 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 32 | golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= 33 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 34 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 35 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 36 | gopkg.in/djherbis/times.v1 v1.2.0 h1:UCvDKl1L/fmBygl2Y7hubXCnY7t4Yj46ZrBFNUipFbM= 37 | gopkg.in/djherbis/times.v1 v1.2.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= 38 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= 39 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 40 | -------------------------------------------------------------------------------- /joe.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=joe service 3 | Documentation=https://github.com/evilsocket/joe 4 | Wants=network.target 5 | After=network.target 6 | 7 | [Service] 8 | Type=simple 9 | PermissionsStartOnly=true 10 | ExecStart=/usr/local/bin/joe -log /var/log/joe.log 11 | Restart=always 12 | RestartSec=30 13 | 14 | [Install] 15 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /models/cache.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "fmt" 7 | "github.com/evilsocket/islazy/log" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | type CachePolicyType int 13 | 14 | const ( 15 | None CachePolicyType = iota 16 | ByKey 17 | ByTime 18 | ) 19 | 20 | type Cached struct { 21 | At time.Time 22 | Data interface{} 23 | } 24 | 25 | type CachePolicy struct { 26 | Type CachePolicyType `yaml:"type" json:"type"` 27 | Keys []string `yaml:"keys" json:"keys"` 28 | TTL int `yaml:"ttl" json:"ttl"` 29 | 30 | cache sync.Map 31 | } 32 | 33 | func (c *CachePolicy) KeyFor(params map[string]interface{}) string { 34 | // results are cached regardless of the parameters for a given amount of time 35 | if c.Type == ByTime { 36 | return "time" 37 | } 38 | 39 | // results are cached by a key made of combining the value of the given parameters 40 | hash := sha256.New() 41 | for _, key := range c.Keys { 42 | what := "" 43 | if v, found := params[key]; found { 44 | what = fmt.Sprintf("%s:%s", key, v) 45 | } else { 46 | what = fmt.Sprintf("%s:", key) 47 | } 48 | hash.Write([]byte(what)) 49 | } 50 | return hex.EncodeToString(hash.Sum(nil)) 51 | } 52 | 53 | func (c *CachePolicy) Get(params map[string]interface{}) *Cached { 54 | // cache disabled 55 | if c.Type == None { 56 | return nil 57 | } 58 | key := c.KeyFor(params) 59 | if obj, found := c.cache.Load(key); !found { 60 | log.Debug("cache[%s] miss", key) 61 | // miss 62 | return nil 63 | } else if cached := obj.(*Cached); time.Since(cached.At).Seconds() > float64(c.TTL) { 64 | log.Debug("cache[%s] expired", key) 65 | // expired 66 | c.cache.Delete(key) 67 | return nil 68 | } else { 69 | log.Debug("cache[%s] hit", key) 70 | // found 71 | return cached 72 | } 73 | } 74 | 75 | func (c *CachePolicy) Set(params map[string]interface{}, data interface{}) { 76 | // cache disabled 77 | if c.Type == None { 78 | return 79 | } 80 | 81 | key := c.KeyFor(params) 82 | entry := &Cached{ 83 | At: time.Now(), 84 | Data: data, 85 | } 86 | 87 | log.Debug("cache[%s] = %v", key, entry) 88 | c.cache.Store(key, entry) 89 | } 90 | -------------------------------------------------------------------------------- /models/query.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/evilsocket/islazy/log" 7 | _ "github.com/go-sql-driver/mysql" 8 | "gopkg.in/djherbis/times.v1" 9 | "gopkg.in/yaml.v2" 10 | "io/ioutil" 11 | "path" 12 | "regexp" 13 | "strings" 14 | "sync" 15 | "time" 16 | ) 17 | 18 | var paramParser = regexp.MustCompile("\\{([^}]+)\\}") 19 | 20 | type Query struct { 21 | sync.Mutex 22 | 23 | CreatedAt time.Time `yaml:"-" json:"created_at"` 24 | UpdatedAt time.Time `yaml:"-" json:"updated_at"` 25 | Name string `yaml:"-" json:"name"` 26 | Cache *CachePolicy `yaml:"cache" json:"cache"` 27 | Description string `yaml:"description" json:"description"` 28 | Defaults map[string]interface{} `yaml:"defaults" json:"defaults"` 29 | Expression string `yaml:"query" json:"query"` 30 | Views map[string]string `yaml:"views" json:"views"` 31 | Access []string `yaml:"access" json:"access"` 32 | 33 | Parameters map[string]string `yaml:"-" json:"-"` 34 | 35 | access map[string]*User 36 | views map[string]*View 37 | compiledViews bool 38 | fileName string 39 | statement string 40 | parameters map[string]int 41 | numParams int 42 | prepared *sql.Stmt 43 | explainer *Query 44 | } 45 | 46 | func LoadQuery(fileName string, compileViews bool) (*Query, error) { 47 | log.Debug("loading %s ...", fileName) 48 | 49 | query := &Query{ 50 | fileName: fileName, 51 | Parameters: make(map[string]string), 52 | parameters: make(map[string]int), 53 | access: make(map[string]*User), 54 | views: make(map[string]*View), 55 | compiledViews: compileViews, 56 | } 57 | 58 | if raw, err := ioutil.ReadFile(fileName); err != nil { 59 | return nil, err 60 | } else if err = yaml.Unmarshal(raw, query); err != nil { 61 | return nil, err 62 | } else if err = query.load(); err != nil { 63 | return nil, err 64 | } 65 | return query, nil 66 | } 67 | 68 | func (q *Query) QueryString() string { 69 | parts := []string{} 70 | for name, value := range q.Parameters { 71 | if value == "" { 72 | parts = append(parts, fmt.Sprintf("%s=VALUE", name)) 73 | } else { 74 | parts = append(parts, fmt.Sprintf("%s=%s", name, value)) 75 | } 76 | } 77 | 78 | if len(parts) > 0 { 79 | return fmt.Sprintf("?%s", strings.Join(parts, "&")) 80 | } else { 81 | return "" 82 | } 83 | } 84 | 85 | func (q *Query) prepare() (err error) { 86 | q.statement = q.Expression 87 | for _, match := range paramParser.FindAllStringSubmatch(q.Expression, -1) { 88 | tok, par := match[0], match[1] 89 | if _, found := q.parameters[par]; found { 90 | return fmt.Errorf("token %s has been used more than once", tok) 91 | } else { 92 | if def, found := q.Defaults[par]; found { 93 | q.Parameters[par] = fmt.Sprintf("%v", def) 94 | } else { 95 | q.Parameters[par] = "" 96 | } 97 | q.parameters[par] = q.numParams 98 | q.numParams++ 99 | q.statement = strings.Replace(q.statement, tok, "?", 1) 100 | } 101 | } 102 | 103 | if q.prepared, err = DB.Prepare(q.statement); err != nil { 104 | return fmt.Errorf("error preparing statement for %s: %v", q.Name, err) 105 | } 106 | 107 | return 108 | } 109 | 110 | func (q *Query) load() error { 111 | if t, err := times.Stat(q.fileName); err != nil { 112 | return err 113 | } else if raw, err := ioutil.ReadFile(q.fileName); err != nil { 114 | return err 115 | } else if err = yaml.Unmarshal(raw, q); err != nil { 116 | return err 117 | } else if len(q.Access) == 0 { 118 | return fmt.Errorf("%s doens't declare an access section", q.Name) 119 | } else { 120 | if t.HasBirthTime() { 121 | q.CreatedAt = t.BirthTime() 122 | } else { 123 | q.CreatedAt = time.Now() 124 | } 125 | q.UpdatedAt = t.ModTime() 126 | q.Name = strings.ReplaceAll(path.Base(q.fileName), ".yml", "") 127 | 128 | for _, username := range q.Access { 129 | if u, found := Users.Load(username); !found { 130 | return fmt.Errorf("user %s not found", username) 131 | } else { 132 | q.access[username] = u.(*User) 133 | if username == "anonymous" { 134 | log.Warning("query %s allows anonymous access", q.Name) 135 | } 136 | } 137 | } 138 | 139 | // prepare the main statement 140 | if err := q.prepare(); err != nil { 141 | return err 142 | } 143 | 144 | // prepare the explain statement 145 | explain := fmt.Sprintf("EXPLAIN %s", q.Expression) 146 | q.explainer = &Query{ 147 | Expression: explain, 148 | Cache: &CachePolicy{Type: None}, 149 | statement: explain, 150 | parameters: make(map[string]int), 151 | Parameters: make(map[string]string), 152 | } 153 | 154 | if err := q.explainer.prepare(); err != nil { 155 | return err 156 | } 157 | 158 | // load views 159 | for viewName, viewFileName := range q.Views { 160 | if viewFileName != "" { 161 | if viewFileName[0] != '/' && viewFileName[0] != '.' { 162 | viewFileName = path.Join(path.Dir(q.fileName), viewFileName) 163 | } 164 | } 165 | if view, err := PrepareView(q.Name, viewName, viewFileName, q.compiledViews); err != nil { 166 | return fmt.Errorf("%s: %v", viewName, err) 167 | } else { 168 | q.views[viewName] = view 169 | } 170 | } 171 | 172 | log.Debug("loaded %v", q) 173 | } 174 | return nil 175 | } 176 | 177 | func (q *Query) toQueryArgs(params map[string]interface{}) ([]interface{}, error) { 178 | // assign statement parameters 179 | args := make([]interface{}, q.numParams) 180 | for name, value := range params { 181 | if order, found := q.parameters[name]; !found { 182 | return nil, fmt.Errorf("unknown parameter '%s'", name) 183 | } else { 184 | args[order] = value 185 | } 186 | } 187 | 188 | // assign missing 189 | for name, defValue := range q.Defaults { 190 | if _, found := params[name]; !found { 191 | if order, found := q.parameters[name]; !found { 192 | return nil, fmt.Errorf("unknown parameter '%s'", name) 193 | } else { 194 | args[order] = defValue 195 | } 196 | } 197 | } 198 | 199 | return args, nil 200 | } 201 | 202 | func (q *Query) AuthRequired() bool { 203 | // does allow anonymous access? 204 | if _, found := q.access["anonymous"]; found { 205 | return false 206 | } 207 | return true 208 | } 209 | 210 | func (q *Query) Authorized(user *User) bool { 211 | if !q.AuthRequired() { 212 | return true 213 | } 214 | // sanity check 215 | if user == nil { 216 | return false 217 | } 218 | // check if included 219 | if _, found := q.access[user.Username]; found { 220 | return true 221 | } 222 | return false 223 | } 224 | 225 | func (q *Query) Query(params map[string]interface{}) (*Results, error) { 226 | log.Debug("running '%s' with %s", q.statement, params) 227 | 228 | begin := time.Now() 229 | 230 | if cached := q.Cache.Get(params); cached != nil { 231 | rows := cached.Data.(*Results) 232 | rows.CachedAt = &cached.At 233 | rows.ExecutionTime = time.Since(begin) 234 | return rows, nil 235 | } 236 | 237 | q.Lock() 238 | defer q.Unlock() 239 | 240 | args, err := q.toQueryArgs(params) 241 | if err != nil { 242 | return nil, err 243 | } 244 | 245 | dbRows, err := q.prepared.Query(args...) 246 | if err != nil { 247 | return nil, err 248 | } 249 | defer dbRows.Close() 250 | 251 | rows := &Results{ 252 | Rows: make([]Row, 0), 253 | } 254 | 255 | if rows.ColumnNames, err = dbRows.Columns(); err != nil { 256 | return nil, err 257 | } else { 258 | rows.NumColumns = len(rows.ColumnNames) 259 | 260 | for dbRows.Next() { 261 | columnValues := make([]interface{}, rows.NumColumns) 262 | for idx, _ := range columnValues { 263 | var dummy interface{} 264 | columnValues[idx] = &dummy 265 | } 266 | 267 | if err := dbRows.Scan(columnValues...); err != nil { 268 | return nil, err 269 | } 270 | 271 | row := make(Row) 272 | for idx, name := range rows.ColumnNames { 273 | v := *(columnValues[idx].(*interface{})) 274 | if rawBytes, ok := v.([]uint8); ok { 275 | row[name] = string(rawBytes) 276 | } else { 277 | row[name] = v 278 | } 279 | } 280 | 281 | rows.Rows = append(rows.Rows, row) 282 | rows.NumRows++ 283 | } 284 | } 285 | 286 | rows.ExecutionTime = time.Since(begin) 287 | 288 | q.Cache.Set(params, rows) 289 | 290 | return rows, nil 291 | } 292 | 293 | func (q *Query) View(name string) *View { 294 | if v, found := q.views[name]; found { 295 | return v 296 | } 297 | return nil 298 | } 299 | 300 | func (q *Query) Explain(params map[string]interface{}) (*Results, error) { 301 | return q.explainer.Query(params) 302 | } 303 | -------------------------------------------------------------------------------- /models/results.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "time" 4 | 5 | type Row map[string]interface{} 6 | 7 | type Results struct { 8 | CachedAt *time.Time `json:"cached_at"` 9 | ExecutionTime time.Duration `json:"exec_time"` 10 | ColumnNames []string `json:"-"` 11 | NumColumns int `json:"-"` 12 | NumRows int `json:"num_records"` 13 | Rows []Row `json:"records"` 14 | } -------------------------------------------------------------------------------- /models/setup.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/evilsocket/islazy/async" 7 | "github.com/evilsocket/islazy/fs" 8 | "github.com/evilsocket/islazy/log" 9 | "github.com/joho/godotenv" 10 | "os" 11 | "sync" 12 | "sync/atomic" 13 | 14 | _ "github.com/go-sql-driver/mysql" 15 | _ "github.com/lib/pq" 16 | ) 17 | 18 | var ( 19 | DB = (*sql.DB)(nil) 20 | Queries = sync.Map{} 21 | NumQueries = uint64(0) 22 | Users = sync.Map{} 23 | NumUsers = 0 24 | ) 25 | 26 | func FindQuery(name string) *Query { 27 | if q, found := Queries.Load(name); found { 28 | return q.(*Query) 29 | } 30 | return nil 31 | } 32 | 33 | func Cleanup() { 34 | if DB != nil { 35 | if err := DB.Close(); err != nil { 36 | fmt.Println(err) 37 | } 38 | DB = nil 39 | } 40 | } 41 | 42 | func Setup(confFile, dataPath, usersPath string, compileViews bool) (err error) { 43 | defer func() { 44 | log.Debug("users:%d queries:%d", NumUsers, NumQueries) 45 | }() 46 | 47 | if err := godotenv.Load(confFile); err != nil { 48 | return fmt.Errorf("error while loading %s: %v", confFile, err) 49 | } 50 | 51 | dbDriver := os.Getenv("DB_DRIVER") 52 | dbHost := os.Getenv("DB_HOST") 53 | dbPort := os.Getenv("DB_PORT") 54 | dbUsername := os.Getenv("DB_USER") 55 | dbPassword := os.Getenv("DB_PASSWORD") 56 | dbName := os.Getenv("DB_NAME") 57 | dbURL := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbUsername, dbPassword, dbHost, dbPort, dbName) 58 | 59 | log.Debug("connecting to %s database at %s ...", dbDriver, dbURL) 60 | 61 | if DB, err = sql.Open(dbDriver, dbURL); err != nil { 62 | return 63 | } else if err = DB.Ping(); err != nil { 64 | return 65 | } 66 | 67 | log.Debug("loading users from %s ...", usersPath) 68 | err = fs.Glob(usersPath, "*.yml", func(fileName string) error { 69 | if user, err := LoadUser(fileName); err != nil { 70 | return fmt.Errorf("error while loading %s: %v", fileName, err) 71 | } else { 72 | Users.Store(user.Username, user) 73 | NumUsers++ 74 | } 75 | return nil 76 | }) 77 | if err != nil { 78 | return err 79 | } 80 | Users.Store("anonymous", &User{}) 81 | 82 | // since each query must be prepared and might potentially require compilation of 83 | // its view files, run this in a workers queue in order to parallelize 84 | queue := async.NewQueue(0, func(arg async.Job) { 85 | fileName := arg.(string) 86 | if query, err := LoadQuery(fileName, compileViews); err != nil { 87 | log.Error("error while loading %s: %v", fileName, err) 88 | } else { 89 | Queries.Store(query.Name, query) 90 | atomic.AddUint64(&NumQueries, 1) 91 | } 92 | }) 93 | 94 | log.Debug("loading data from %s ...", dataPath) 95 | err = fs.Glob(dataPath, "*.yml", func(fileName string) error { 96 | queue.Add(async.Job(fileName)) 97 | return nil 98 | }) 99 | queue.WaitDone() 100 | return err 101 | } 102 | -------------------------------------------------------------------------------- /models/user.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "github.com/evilsocket/islazy/log" 7 | "gopkg.in/yaml.v2" 8 | "io/ioutil" 9 | "os" 10 | ) 11 | 12 | type User struct { 13 | Username string `yaml:"username"` 14 | Password string `yaml:"password"` 15 | TokenTTL int `yaml:"token_ttl"` 16 | } 17 | 18 | func SaveUser(user User, fileName string) error { 19 | hash := sha256.New() 20 | hash.Write([]byte(user.Password)) 21 | user.Password = hex.EncodeToString(hash.Sum(nil)) 22 | 23 | if raw, err := yaml.Marshal(&user); err != nil { 24 | return err 25 | } else if err := ioutil.WriteFile(fileName, raw, os.ModePerm); err != nil { 26 | return err 27 | } 28 | return nil 29 | } 30 | 31 | func LoadUser(fileName string) (*User, error) { 32 | log.Debug("loading %s ...", fileName) 33 | 34 | user := &User{} 35 | 36 | if raw, err := ioutil.ReadFile(fileName); err != nil { 37 | return nil, err 38 | } else if err = yaml.Unmarshal(raw, user); err != nil { 39 | return nil, err 40 | } 41 | 42 | return user, nil 43 | } 44 | 45 | func (u *User) ValidPassword(password string) bool { 46 | hash := sha256.New() 47 | hash.Write([]byte(password)) 48 | return hex.EncodeToString(hash.Sum(nil)) == u.Password 49 | } 50 | -------------------------------------------------------------------------------- /models/view.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "fmt" 5 | "github.com/evilsocket/islazy/log" 6 | "github.com/wcharczuk/go-chart" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "path" 11 | "path/filepath" 12 | "plugin" 13 | "strings" 14 | ) 15 | 16 | type Chart interface { 17 | Render(rp chart.RendererProvider, w io.Writer) error 18 | } 19 | 20 | type View struct { 21 | Name string 22 | SourceFileName string 23 | NativeFileName string 24 | 25 | plugin *plugin.Plugin 26 | cb func(*Results) Chart 27 | } 28 | 29 | func PrepareView(queryName, viewName, viewFileName string, compile bool) (view *View, err error) { 30 | view = &View{ 31 | Name: viewName, 32 | } 33 | 34 | if viewFileName, err = filepath.Abs(viewFileName); err != nil { 35 | return 36 | } 37 | basePath := path.Dir(viewFileName) 38 | 39 | if strings.HasSuffix(viewFileName, ".go") { 40 | view.SourceFileName = viewFileName 41 | view.NativeFileName = path.Join(basePath, fmt.Sprintf( 42 | "%s_%s.so", 43 | queryName, 44 | viewName)) 45 | 46 | if compile { 47 | goPath, err := exec.LookPath("go") 48 | if err != nil { 49 | return nil, fmt.Errorf("go not found, can't compile %s", viewFileName) 50 | } 51 | 52 | log.Info("compiling %s ...", viewFileName) 53 | 54 | cmdLine := fmt.Sprintf("%s build -buildmode=plugin -o '%s' '%s'", 55 | goPath, 56 | view.NativeFileName, 57 | view.SourceFileName) 58 | 59 | log.Debug("%s", cmdLine) 60 | 61 | cmd := exec.Command("sh", "-c", cmdLine) 62 | cmd.Stdout = os.Stdout 63 | cmd.Stderr = os.Stderr 64 | cmd.Env = os.Environ() 65 | 66 | if err := cmd.Run(); err != nil { 67 | return nil, err 68 | } 69 | } 70 | } else { 71 | view.SourceFileName = viewFileName 72 | view.NativeFileName = viewFileName 73 | } 74 | 75 | if compile { 76 | log.Debug("loading view %s ...", view.NativeFileName) 77 | 78 | if view.plugin, err = plugin.Open(view.NativeFileName); err != nil { 79 | return nil, err 80 | } 81 | 82 | f, err := view.plugin.Lookup("View") 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | var ok bool 88 | if view.cb, ok = f.(func(*Results) Chart); !ok { 89 | return nil, fmt.Errorf("can't cast %+v to func(*Results) Chart", f) 90 | } 91 | } 92 | 93 | return view, nil 94 | } 95 | 96 | func (v *View) Call(res *Results) Chart { 97 | return v.cb(res) 98 | } 99 | --------------------------------------------------------------------------------