Hello {{.Name}} Logout
`) 189 | t.Execute(w, user) 190 | } else { 191 | t, _ := template.New("loggedOut").Parse(`Come on in - Log in with GitHub
`) 192 | t.Execute(w, "") 193 | } 194 | } 195 | 196 | func anotherPageTest(w http.ResponseWriter, r *http.Request) { 197 | isLoggedIn, user, err := isLoggedIn(r) 198 | if err != nil { 199 | http.Error(w, err.Error(), http.StatusInternalServerError) 200 | } 201 | 202 | if isLoggedIn { 203 | t, _ := template.New("another").Parse(`Hello {{.Name}} This is another page. Logout
`) 204 | t.Execute(w, user) 205 | } else { 206 | t, _ := template.New("anotherOut").Parse(`You're not logged in, you should go here
`) 207 | t.Execute(w, "") 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /api/auth_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequired 2 | 3 | package api 4 | 5 | import ( 6 | "encoding/json" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | 12 | "github.com/microscaling/microbadger/database" 13 | ) 14 | 15 | func TestMe(t *testing.T) { 16 | var err error 17 | var res *http.Response 18 | var req *http.Request 19 | var user database.User 20 | 21 | db = getDatabase(t) 22 | emptyDatabase(db) 23 | addThings(db) 24 | addUser(db) 25 | sessionStore = NewTestStore() 26 | 27 | ts := httptest.NewServer(muxRoutes()) 28 | defer ts.Close() 29 | 30 | // If not logged in, /v1/me should return nothing 31 | res, err = http.Get(ts.URL + "/v1/me") 32 | if err != nil { 33 | t.Fatalf("Failed to send request") 34 | } 35 | 36 | body, err := ioutil.ReadAll(res.Body) 37 | res.Body.Close() 38 | if err != nil { 39 | t.Errorf("Error getting body. %v", err) 40 | } 41 | 42 | if res.StatusCode != 200 { 43 | t.Errorf("Failed with status code %d", res.StatusCode) 44 | } 45 | 46 | if string(body) != "{}" { 47 | t.Errorf("Body unexpectedly not empty: %s", body) 48 | } 49 | 50 | // Now try again but this time log in 51 | req, err = http.NewRequest("GET", ts.URL+"/v1/me", nil) 52 | logIn(req) 53 | res, err = http.DefaultClient.Do(req) 54 | if err != nil { 55 | t.Fatalf("Failed to send request %v", err) 56 | } 57 | 58 | body, err = ioutil.ReadAll(res.Body) 59 | res.Body.Close() 60 | if err != nil { 61 | t.Errorf("Error getting body. %v", err) 62 | } 63 | 64 | err = json.Unmarshal(body, &user) 65 | if err != nil { 66 | t.Errorf("Error unmarshalling user. %v", err) 67 | } 68 | 69 | if (user.Email != "myname@myaddress.com") || (user.Name != "myuser") { 70 | t.Errorf("Didn't get the user we expected %#v", user) 71 | } 72 | 73 | // Now log out again 74 | req, err = http.NewRequest("GET", ts.URL+"/v1/me", nil) 75 | logOut(req) 76 | res, err = http.DefaultClient.Do(req) 77 | if err != nil { 78 | t.Fatalf("Failed to send request %v", err) 79 | } 80 | 81 | body, err = ioutil.ReadAll(res.Body) 82 | res.Body.Close() 83 | if err != nil { 84 | t.Errorf("Error getting body. %v", err) 85 | } 86 | 87 | if res.StatusCode != 200 { 88 | t.Errorf("Failed with status code %d", res.StatusCode) 89 | } 90 | 91 | if string(body) != "{}" { 92 | t.Errorf("Body unexpectedly not empty: %s", body) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /api/badge.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "time" 9 | 10 | "code.cloudfoundry.org/bytefmt" 11 | "github.com/gorilla/mux" 12 | 13 | "github.com/microscaling/microbadger/database" 14 | "github.com/microscaling/microbadger/inspector" 15 | ) 16 | 17 | func handleGetImageBadge(w http.ResponseWriter, r *http.Request) { 18 | var labelValue, badgeSVG string 19 | var imageVersion database.ImageVersion 20 | 21 | var image, org, tag, badgeType string 22 | var ok bool 23 | var license *database.License 24 | var vcs *database.VersionControl 25 | 26 | vars := mux.Vars(r) 27 | image = vars["image"] 28 | if org, ok = vars["org"]; !ok { 29 | org = "library" 30 | } 31 | 32 | badgeType = vars["badgeType"] 33 | tag = vars["tag"] 34 | 35 | w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") 36 | w.Header().Set("Expires", time.Now().Format(http.TimeFormat)) 37 | 38 | log.Debugf("Badge type: %s - image: %s - tag: %s", badgeType, org+"/"+image, tag) 39 | 40 | img, err := db.GetImage(org + "/" + image) 41 | if err != nil || img.Status == "MISSING" || img.IsPrivate { 42 | log.Infof("Image %s missing for badge", img.Name) 43 | badgeType = "imagemissing" 44 | } else { 45 | if tag == "" { 46 | imageVersion, err = db.GetImageVersionBySHA(img.Latest, img.Name, false) 47 | if err != nil { 48 | // Error as there should always be a latest SHA. 49 | log.Errorf("Missing latest version for %s: %v", img.Name, err) 50 | w.WriteHeader(http.StatusInternalServerError) 51 | return 52 | } 53 | } else { 54 | imageVersion, err = db.GetImageVersionByTag(img.Name, tag) 55 | if err != nil { 56 | log.Infof("Missing image version for %s tag %s : %v", img.Name, tag, err) 57 | badgeType = "tagmissing" 58 | } 59 | } 60 | 61 | if !strings.Contains(badgeType, "missing") { 62 | _, license, vcs = inspector.ParseLabels(&imageVersion) 63 | } 64 | } 65 | 66 | switch badgeType { 67 | case "image": 68 | badgeSVG = generateImageBadge(imageVersion) 69 | 70 | case "commit": 71 | if vcs == nil { 72 | labelValue = "not given" 73 | } else { 74 | labelValue = formatLabel(vcs.Commit, 7) 75 | } 76 | badgeSVG = generateBadge(badgeType, labelValue) 77 | 78 | case "version": 79 | // Use a tag if it was specified 80 | if tag != "" { 81 | log.Debugf("Specified tag is %s", tag) 82 | labelValue = formatLabel(tag, 10) 83 | } else { 84 | labelValue = formatLabel(getLongestTag(&imageVersion), 10) 85 | } 86 | badgeSVG = generateBadge(badgeType, labelValue) 87 | 88 | case "license": 89 | if license == nil { 90 | labelValue = "not given" 91 | } else { 92 | labelValue = formatLabel(license.Code, 10) 93 | } 94 | badgeSVG = generateBadge(badgeType, labelValue) 95 | 96 | case "imagemissing": 97 | badgeSVG = generateBadge("Image", "not found") 98 | 99 | case "tagmissing": 100 | badgeSVG = generateBadge(formatLabel(tag, 7), "not found") 101 | 102 | default: 103 | log.Infof("Bad badge type for badge %s", badgeType) 104 | w.WriteHeader(http.StatusBadRequest) 105 | return 106 | } 107 | 108 | w.Write([]byte(badgeSVG)) 109 | 110 | ga.ImageView("imageView", "imageView", "badge", badgeType, r) 111 | } 112 | 113 | func formatLabel(input string, length int) string { 114 | var result string 115 | 116 | if len(input) > length { 117 | lastChar := length - 1 118 | 119 | result = input[0:lastChar] + constEllipsis 120 | } else { 121 | result = input 122 | } 123 | 124 | return result 125 | } 126 | 127 | func generateImageBadge(latest database.ImageVersion) (badgeSVG string) { 128 | size := fmt.Sprintf("%sB", bytefmt.ByteSize(uint64(latest.DownloadSize))) 129 | layers := fmt.Sprintf("%d layers", latest.LayerCount) 130 | 131 | badgeSVG = "" 132 | 133 | badgeSVG = strings.Replace(badgeSVG, "SIZE", size, 2) 134 | badgeSVG = strings.Replace(badgeSVG, "LAYERS", layers, 2) 135 | 136 | return badgeSVG 137 | } 138 | 139 | func generateBadge(label string, value string) (badgeSVG string) { 140 | if len(value) > 7 { 141 | badgeSVG = "" 142 | } else { 143 | badgeSVG = "" 144 | } 145 | 146 | badgeSVG = strings.Replace(badgeSVG, "LABEL", label, 2) 147 | badgeSVG = strings.Replace(badgeSVG, "VALUE", value, 2) 148 | 149 | return badgeSVG 150 | } 151 | 152 | func handleGetBadgeCounts(w http.ResponseWriter, r *http.Request) { 153 | var badgeCounts BadgeCounts 154 | badges, images, err := db.GetBadgesInstalledCount() 155 | if err != nil { 156 | log.Errorf("Couldn't retrieve badge count: %v", err) 157 | } 158 | 159 | badgeCounts.DockerHub.Badges = badges 160 | badgeCounts.DockerHub.Images = images 161 | 162 | bytes, err := json.Marshal(badgeCounts) 163 | if err != nil { 164 | log.Errorf("Error: %v", err) 165 | } 166 | 167 | w.Write([]byte(bytes)) 168 | } 169 | -------------------------------------------------------------------------------- /database/favourite.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import () 4 | 5 | // GetFavourite returns true if this user favourited this image 6 | func (d *PgDB) GetFavourite(user User, image string) (bool, error) { 7 | var fav Favourite 8 | 9 | _, err := d.GetImage(image) 10 | if err != nil { 11 | return false, err 12 | } 13 | 14 | err = d.db.Where(`"user_id" = ? AND image_name = ?`, user.ID, image).First(&fav).Error 15 | if err != nil { 16 | // TODO!! We're kind of rashly assuming that an error here simply means the row doesn't exist 17 | // Return no error, but this just isn't a favourite for this user 18 | return false, nil 19 | } 20 | return true, nil 21 | } 22 | 23 | // PutFavourite saves it 24 | func (d *PgDB) PutFavourite(user User, image string) (fav Favourite, err error) { 25 | 26 | _, err = d.GetImage(image) 27 | if err != nil { 28 | return fav, err 29 | } 30 | 31 | err = d.db.Where("id = ?", user.ID).Find(&user).Error 32 | if err != nil { 33 | return fav, err 34 | } 35 | 36 | fav = Favourite{UserID: user.ID, ImageName: image} 37 | err = d.db.Where(fav).FirstOrCreate(&fav).Error 38 | if err != nil { 39 | log.Debugf("Put Favourite error %v", err) 40 | return fav, err 41 | } 42 | 43 | err = d.db.Save(&fav).Error 44 | if err != nil { 45 | log.Debugf("Put Favourite 2 error %v", err) 46 | } 47 | 48 | return fav, err 49 | } 50 | 51 | // GetFavourites returns a slice of image names that are favourites for this user 52 | func (d *PgDB) GetFavourites(user User) ImageList { 53 | var images []string 54 | 55 | err := d.db.Table("favourites"). 56 | Where(`"user_id" = ?`, user.ID). 57 | Pluck("image_name", &images).Error 58 | if err != nil { 59 | log.Errorf("Error getting favourites images: %v", err) 60 | } 61 | 62 | return ImageList{Images: images} 63 | } 64 | 65 | // DeleteFavourite deletes a favourite, returning an error if it doesn't exist 66 | func (d *PgDB) DeleteFavourite(user User, image string) error { 67 | 68 | err := d.db.Where(`"user_id" = ? and "image_name" = ?`, user.ID, image).Delete(Favourite{}).Error 69 | if err != nil { 70 | log.Debugf("Error Deleting favourites images: %v", err) 71 | } 72 | 73 | return err 74 | } 75 | -------------------------------------------------------------------------------- /database/favourite_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequired 2 | 3 | package database 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/markbates/goth" 9 | ) 10 | 11 | func TestFavourites(t *testing.T) { 12 | var err error 13 | var db PgDB 14 | var isFav bool 15 | 16 | db = getDatabase(t) 17 | emptyDatabase(db) 18 | addThings(db) 19 | 20 | u, err := db.GetOrCreateUser(User{}, goth.User{Provider: "myprov", UserID: "12345", Name: "myname", Email: "me@myaddress.com"}) 21 | if err != nil { 22 | t.Errorf("Error creating user %v", err) 23 | } 24 | 25 | u2, err := db.GetOrCreateUser(User{}, goth.User{Provider: "myprov", UserID: "67890", Name: "anothername", Email: "another@theiraddress.com"}) 26 | if err != nil { 27 | t.Errorf("Error creating user %v", err) 28 | } 29 | 30 | // Check there are no favourites 31 | favs := db.GetFavourites(u) 32 | if favs.Images != nil { 33 | t.Errorf("Unexpected favourites %v", favs) 34 | } 35 | 36 | favs = db.GetFavourites(u2) 37 | if favs.Images != nil { 38 | t.Errorf("Unexpected favourites %v", favs) 39 | } 40 | 41 | // Add, get and delete a favourite 42 | _, err = db.PutFavourite(u, "lizrice/childimage") 43 | if err != nil { 44 | t.Errorf("Failed to create favourite") 45 | } 46 | 47 | isFav, err = db.GetFavourite(u, "lizrice/childimage") 48 | if err != nil { 49 | t.Errorf("Error getting favourite") 50 | } 51 | if !isFav { 52 | t.Errorf("Added favourite doesn't exist") 53 | } 54 | 55 | favs = db.GetFavourites(u) 56 | if (len(favs.Images) != 1) || (favs.Images[0] != "lizrice/childimage") { 57 | t.Errorf("Unexpected favourites %v", favs) 58 | } 59 | 60 | err = db.DeleteFavourite(u, "lizrice/childimage") 61 | if err != nil { 62 | t.Errorf("Failed to create favourite") 63 | } 64 | 65 | // Check we can do this for a second user 66 | favs = db.GetFavourites(u2) 67 | if favs.Images != nil { 68 | t.Errorf("Unexpected favourites %v", favs) 69 | } 70 | 71 | _, err = db.PutFavourite(u2, "lizrice/childimage") 72 | if err != nil { 73 | t.Errorf("Failed to create favourite") 74 | } 75 | 76 | // Check this is now showing up as a favourite for u2 but not u 77 | favs = db.GetFavourites(u2) 78 | if (len(favs.Images) != 1) || (favs.Images[0] != "lizrice/childimage") { 79 | t.Errorf("Unexpected favourites %v", favs) 80 | } 81 | 82 | favs = db.GetFavourites(u) 83 | if favs.Images != nil { 84 | t.Errorf("Unexpected favourites %v", favs) 85 | } 86 | 87 | // Check that we can have more than one favourite for a user 88 | _, err = db.PutFavourite(u2, "lizrice/featured") 89 | if err != nil { 90 | t.Errorf("Failed to create favourite %v", err) 91 | } 92 | 93 | favs = db.GetFavourites(u2) 94 | if len(favs.Images) != 2 { 95 | t.Errorf("Unexpected favourites %v", favs) 96 | } 97 | 98 | // And check that deleting one leaves the other in place 99 | err = db.DeleteFavourite(u2, "lizrice/childimage") 100 | if err != nil { 101 | t.Errorf("Failed to delete favourite") 102 | } 103 | 104 | favs = db.GetFavourites(u2) 105 | if (len(favs.Images) != 1) || (favs.Images[0] != "lizrice/featured") { 106 | t.Errorf("Unexpected favourites %v", favs) 107 | } 108 | 109 | // this image should still be a favourite for u2 110 | isFav, err = db.GetFavourite(u2, "lizrice/featured") 111 | if err != nil { 112 | t.Errorf("Error getting favourite") 113 | } 114 | if !isFav { 115 | t.Errorf("Added favourite doesn't exist") 116 | } 117 | 118 | // but this image should no longer be a favourite for u2 119 | isFav, err = db.GetFavourite(u2, "lizrice/childimage") 120 | if err != nil { 121 | t.Errorf("Error getting favourite: %s", err) 122 | } 123 | if isFav { 124 | t.Errorf("Image is unexpectedly a favourite") 125 | } 126 | 127 | // Error cases 128 | _, err = db.PutFavourite(u, "missing") 129 | if err == nil { 130 | t.Errorf("Shouldn't have been able to create a favourite for an image that doesn't exist") 131 | } 132 | 133 | _, err = db.PutFavourite(User{}, "lizrice/childimage") 134 | if err == nil { 135 | t.Errorf("Shouldn't have been able to create a favourite for a user that doesn't exist") 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /database/image_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequired 2 | 3 | package database 4 | 5 | import ( 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | type searchTestCase struct { 11 | search string 12 | images []string 13 | } 14 | 15 | type pageURLTestCase struct { 16 | image string 17 | url string 18 | } 19 | 20 | func TestImageSearch(t *testing.T) { 21 | var db PgDB 22 | var tests = []searchTestCase{ 23 | {search: "liz", images: []string{"lizrice/featured", "lizrice/childimage"}}, 24 | {search: "rice", images: []string{"lizrice/featured", "lizrice/childimage"}}, 25 | {search: "image", images: []string{"lizrice/childimage"}}, 26 | {search: "lizrice/featured", images: []string{"lizrice/featured"}}, 27 | {search: "micro", images: []string{}}, 28 | {search: "myuser/private", images: []string{}}, 29 | {search: "private", images: []string{}}, 30 | } 31 | 32 | db = getDatabase(t) 33 | emptyDatabase(db) 34 | addThings(db) 35 | 36 | for _, test := range tests { 37 | images, _ := db.ImageSearch(test.search) 38 | 39 | if len(images) > 0 || len(test.images) > 0 { 40 | if !reflect.DeepEqual(images, test.images) { 41 | t.Errorf("Expected search results to be %v but were %v", test.images, images) 42 | } 43 | } 44 | } 45 | } 46 | 47 | func TestFeaturedImages(t *testing.T) { 48 | var db PgDB 49 | 50 | db = getDatabase(t) 51 | emptyDatabase(db) 52 | addThings(db) 53 | 54 | il := db.GetFeaturedImages() 55 | if (il.CurrentPage != 1) || (il.PageCount != 1) { 56 | t.Errorf("ImageList pagination wrong: %v", il) 57 | } 58 | 59 | if len(il.Images) != il.ImageCount { 60 | t.Errorf("Wrong image count %d but %d image names included", il.ImageCount, len(il.Images)) 61 | } 62 | 63 | testImages := []string{"lizrice/featured"} 64 | if !reflect.DeepEqual(il.Images, testImages) { 65 | t.Errorf("Unexpected featured images: %v\n expected: %v", il.Images, testImages) 66 | } 67 | } 68 | 69 | func TestRecentImages(t *testing.T) { 70 | var db PgDB 71 | 72 | db = getDatabase(t) 73 | emptyDatabase(db) 74 | addThings(db) 75 | 76 | il := db.GetRecentImages() 77 | if (il.CurrentPage != 1) || (il.PageCount != 1) { 78 | t.Errorf("ImageList pagination wrong: %v", il) 79 | } 80 | 81 | if len(il.Images) != il.ImageCount { 82 | t.Errorf("Wrong image count %d but %d image names included", il.ImageCount, len(il.Images)) 83 | } 84 | 85 | testImages := []string{"lizrice/childimage", "lizrice/featured"} 86 | if !reflect.DeepEqual(il.Images, testImages) { 87 | t.Errorf("Unexpected recent images: %v\n expected: %v", il.Images, testImages) 88 | } 89 | } 90 | 91 | func TestLabelSchemaImages(t *testing.T) { 92 | var db PgDB 93 | 94 | db = getDatabase(t) 95 | emptyDatabase(db) 96 | addThings(db) 97 | 98 | il := db.GetLabelSchemaImages(1) 99 | if (il.CurrentPage != 1) || (il.PageCount != 1) { 100 | t.Errorf("ImageList pagination wrong: %v", il) 101 | } 102 | 103 | if len(il.Images) != il.ImageCount { 104 | t.Errorf("Wrong image count %d but %d image names included", il.ImageCount, len(il.Images)) 105 | } 106 | 107 | testImages := []string{"lizrice/childimage"} 108 | if !reflect.DeepEqual(il.Images, testImages) { 109 | t.Errorf("Unexpected label schema images: %v\n expected: %v", il.Images, testImages) 110 | } 111 | } 112 | 113 | func TestGetImage(t *testing.T) { 114 | var db PgDB 115 | 116 | db = getDatabase(t) 117 | emptyDatabase(db) 118 | addThings(db) 119 | 120 | img, err := db.GetImage("lizrice/featured") 121 | if err != nil { 122 | t.Errorf("failed to get image") 123 | } 124 | 125 | if img.Name != "lizrice/featured" { 126 | t.Errorf("Unexpected image name") 127 | } 128 | } 129 | 130 | func TestGetImageForUser(t *testing.T) { 131 | var db PgDB 132 | 133 | db = getDatabase(t) 134 | emptyDatabase(db) 135 | addThings(db) 136 | 137 | img, permission, err := db.GetImageForUser("lizrice/featured", nil) 138 | if err != nil { 139 | t.Errorf("failed to get image") 140 | } 141 | 142 | if !permission { 143 | t.Errorf("Should have permission to get public image") 144 | } 145 | 146 | if img.Name != "lizrice/featured" { 147 | t.Errorf("Unexpected image name") 148 | } 149 | 150 | // Shouldn't be able to get a private image unless we have permissions for it t.Logf("At the database level we are allowing access to private images without checking permissions?") 151 | img, permission, err = db.GetImageForUser("myuser/private", nil) 152 | if permission { 153 | t.Errorf("Shouldn't be able to get private image") 154 | } 155 | 156 | // User Image Permission tests in user_test.go check that we can only get images we have permission for 157 | } 158 | 159 | func TestFeatureImage(t *testing.T) { 160 | var db PgDB 161 | 162 | db = getDatabase(t) 163 | emptyDatabase(db) 164 | addThings(db) 165 | 166 | // Can feature and unfeature an image 167 | err := db.FeatureImage("lizrice/childimage", true) 168 | if err != nil { 169 | t.Errorf("Failed to feature image") 170 | } 171 | 172 | // Can feature and unfeature an image 173 | err = db.FeatureImage("lizrice/childimage", false) 174 | if err != nil { 175 | t.Errorf("Failed to unfeature image") 176 | } 177 | 178 | // But not if it's private 179 | err = db.FeatureImage("myuser/private", true) 180 | if err == nil { 181 | t.Errorf("Shouldn't be able to feature private image") 182 | } 183 | } 184 | 185 | func TestGetPageURL(t *testing.T) { 186 | var db PgDB 187 | var tests = []pageURLTestCase{ 188 | {image: "microbadgertest/alpine", url: "https://microbadger.com/registry/docker/images/microbadgertest/alpine"}, 189 | {image: "microscaling/microscaling", url: "https://microbadger.com/images/microscaling/microscaling"}, 190 | {image: "library/alpine", url: "https://microbadger.com/images/alpine"}, 191 | } 192 | 193 | db = getDatabase(t) 194 | emptyDatabase(db) 195 | 196 | db.Exec("INSERT INTO images (name, status, is_private) VALUES('microbadgertest/alpine', 'INSPECTED', true)") 197 | db.Exec("INSERT INTO images (name, status, is_private) VALUES('microscaling/microscaling', 'INSPECTED', false)") 198 | db.Exec("INSERT INTO images (name, status, is_private) VALUES('library/alpine', 'INSPECTED', false)") 199 | 200 | for _, test := range tests { 201 | img, _ := db.GetImage(test.image) 202 | url := db.GetPageURL(img) 203 | 204 | if url != test.url { 205 | t.Errorf("Unexpected image URL. Expected %s but was %s", test.url, url) 206 | } 207 | } 208 | } 209 | 210 | func TestDeleteImage(t *testing.T) { 211 | var d PgDB 212 | var rows int 213 | 214 | d = getDatabase(t) 215 | emptyDatabase(d) 216 | addThings(d) 217 | 218 | var images = []string{"lizrice/childimage", "lizrice/featured", "myuser/private", "public/sitemap"} 219 | 220 | for _, image := range images { 221 | err := d.DeleteImage(image) 222 | if err != nil { 223 | t.Errorf("Unexpected error deleting image %s - %v", image, err) 224 | } 225 | } 226 | 227 | d.db.Table("images").Count(&rows) 228 | if rows != 0 { 229 | t.Errorf("Found %d unexpected rows in images", rows) 230 | } 231 | 232 | d.db.Table("image_versions").Count(&rows) 233 | if rows != 0 { 234 | t.Errorf("Found %d unexpected rows in image_versions", rows) 235 | } 236 | 237 | d.db.Table("tags").Count(&rows) 238 | if rows != 0 { 239 | t.Errorf("Found %d unexpected rows in tags", rows) 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /database/interface.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "database/sql/driver" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/jinzhu/gorm" 10 | logging "github.com/op/go-logging" 11 | ) 12 | 13 | var ( 14 | log = logging.MustGetLogger("mmdata") 15 | ) 16 | 17 | // TableName for Registry 18 | func (r Registry) TableName() string { 19 | return "registries" 20 | } 21 | 22 | // ImageList is used to send a list of images on the API. 23 | // TODO We should probably move everything to use ImageInfoList 24 | type ImageList struct { 25 | CurrentPage int `json:",omitempty"` 26 | PageCount int `json:",omitempty"` 27 | ImageCount int `json:",omitempty"` 28 | Images []string `json:",omitempty"` 29 | } 30 | 31 | // ImageInfo returns lists of images with pagination 32 | type ImageInfoList struct { 33 | CurrentPage int 34 | PageCount int 35 | ImageCount int 36 | Images []ImageInfo 37 | } 38 | 39 | // ImageInfo has summary info for display 40 | type ImageInfo struct { 41 | ImageName string 42 | Status string 43 | IsPrivate bool 44 | } 45 | 46 | // RegistryList lists the registries 47 | type RegistryList struct { 48 | UserID uint 49 | EnabledImageCount int 50 | Registries []Registry 51 | } 52 | 53 | // Registry is a supported docker registry 54 | type Registry struct { 55 | ID string `gorm:"primary_key"` 56 | Name string 57 | Url string 58 | CredentialsName string `gorm:"-"` 59 | } 60 | 61 | // Image is an image 62 | type Image struct { 63 | // Fields managed by gorm 64 | Name string `gorm:"primary_key" json:"-"` 65 | Status string `json:"-"` 66 | Featured bool `json:"-"` 67 | Latest string `sql:"REFERENCES image_versions(sha) ON DELETE RESTRICT" json:"LatestSHA"` 68 | BadgeCount int `json:"-"` // Number of badges we can generate for this image 69 | AuthToken string `json:"-"` 70 | WebhookURL string `gorm:"column:web_hook_url" json:",omitempty"` 71 | CreatedAt time.Time `json:"-"` // Auto-updated 72 | UpdatedAt time.Time `json:"UpdatedAt"` // Auto-updated - This is used to show when we last inspected the image 73 | Description string `json:",omitempty"` 74 | IsPrivate bool `json:"-"` 75 | IsAutomated bool `json:"-"` 76 | LastUpdated time.Time `json:",omitempty"` // This is what the hub API tells us 77 | BadgesInstalled int `json:"-"` // Number of badges for this image we have found (so far we only look on Docker Hub) 78 | PullCount int 79 | StarCount int 80 | 81 | // Gorm auto-updating doesn't work well as we have our own primary key, so we handle this ourselves 82 | Versions []ImageVersion `gorm:"-" json:",omitempty"` 83 | 84 | // These are json-only fields that are copied from the latest version 85 | // TODO We could move more of these to ImageVersion 86 | ID string `gorm:"-" json:"Id,omitempty"` // SHA from latest version 87 | ImageName string `gorm:"-" json:",omitempty"` // Name but with library removed for base images 88 | ImageURL string `gorm:"-" json:",omitempty"` 89 | Author string `gorm:"-" json:",omitempty"` 90 | LayerCount int `gorm:"-" json:",omitempty"` 91 | DownloadSize int64 `gorm:"-" json:",omitempty"` 92 | Labels map[string]string `gorm:"-" json:",omitempty"` 93 | LatestTag string `gorm:"-" json:"LatestVersion,omitempty"` 94 | } 95 | 96 | // ImageVersion is a version of an image 97 | type ImageVersion struct { 98 | SHA string `gorm:"primary_key"` 99 | Tags []Tag `json:"Tags"` 100 | ImageName string `gorm:"primary_key" sql:"REFERENCES images(name) ON DELETE RESTRICT"` 101 | Author string `` 102 | Labels string `json:"-"` 103 | LabelMap map[string]string `gorm:"-" json:"Labels,omitempty"` 104 | LayerCount int `sql:"DEFAULT:0"` 105 | DownloadSize int64 `sql:"DEFAULT:0"` 106 | Created time.Time // From Registry, tells us when this image version was created 107 | Layers string `json:"-"` 108 | LayersArray []ImageLayer `gorm:"-" json:"Layers,omitempty"` 109 | Manifest string `json:"-"` 110 | Hash string `gorm:"index" json:"-"` // Hash of the layers in this image 111 | Parents []ImageVersion `gorm:"-" json:",omitempty"` 112 | Identical []ImageVersion `gorm:"-" json:",omitempty"` 113 | MicrobadgerURL string `gorm:"-" json:",omitempty"` 114 | 115 | // JSON only fields for data parsed from the labels. 116 | License *License `gorm:"-" json:",omitempty"` 117 | VersionControl *VersionControl `gorm:"-" json:",omitempty"` 118 | } 119 | 120 | // Tag is assigned to an image version 121 | type Tag struct { 122 | Tag string `gorm:"primary_key" json:"tag"` 123 | ImageName string `gorm:"primary_key" json:"-" sql:"REFERENCES images(name) ON DELETE RESTRICT" ` 124 | SHA string `gorm:"index" json:",omitempty" sql:"REFERENCES image_versions(sha) on DELETE RESTRICT"` 125 | } 126 | 127 | // ImageLayer is the detail of layers that make up an image version. TODO!! Consider storing these so we can pull them and store the details 128 | type ImageLayer struct { 129 | BlobSum string `gorm:"-"` // TODO! We need this in API for calculating hashes, but we don't want it travelling on the API 130 | Command string `gorm:"-" json:"Command"` 131 | DownloadSize int64 `gorm:"-" json:"DownloadSize"` 132 | } 133 | 134 | // License is parsed from the org.label-schema.license label. 135 | type License struct { 136 | Code string `json:"Code,omitempty"` 137 | URL string `json:"URL,omitempty"` 138 | } 139 | 140 | // VersionControl is parsed from the org.label-schema.vcs-* labels. 141 | type VersionControl struct { 142 | Type string 143 | URL string 144 | Commit string 145 | } 146 | 147 | // User is a user, and has to refer to potentially multiple authorizations 148 | type User struct { 149 | gorm.Model `json:"-"` 150 | Name string `json:",omitempty"` 151 | Email string `json:",omitempty"` 152 | AvatarURL string `json:",omitempty"` 153 | Auths []UserAuth `json:"-" gorm:"ForeignKey:UserID"` 154 | UserSetting *UserSetting `gorm:"ForeignKey:UserID" json:",omitempty"` 155 | } 156 | 157 | // UserAuth is the identify of this user for an OAuth provider 158 | type UserAuth struct { 159 | UserID uint `gorm:"primary_key"` 160 | 161 | Provider string `gorm:"primary_key"` 162 | NameFromAuth string 163 | IDFromAuth string 164 | NicknameFromAuth string 165 | } 166 | 167 | // UserImagePermission holds permissions for user access to private images 168 | type UserImagePermission struct { 169 | UserID uint `gorm:"primary_key"` 170 | ImageName string `gorm:"primary_key"` 171 | 172 | CreatedAt time.Time `json:"-"` // Auto-updated 173 | UpdatedAt time.Time `json:"-"` // Auto-updated 174 | } 175 | 176 | // UserRegistryCredential holds credentials for accessing private images 177 | type UserRegistryCredential struct { 178 | RegistryID string `gorm:"primary_key"` 179 | UserID uint `gorm:"primary_key"` 180 | 181 | User string 182 | Password string `gorm:"-"` 183 | EncryptedPassword string `json:"-"` 184 | EncryptedKey string `json:"-"` 185 | 186 | CreatedAt time.Time `json:"-"` // Auto-updated 187 | UpdatedAt time.Time `json:"-"` // Auto-updated 188 | } 189 | 190 | // UserSetting holds additional data that is not stored on the session. 191 | type UserSetting struct { 192 | gorm.Model `json:"-"` 193 | UserID uint `gorm:"primary_key" json:"-"` 194 | 195 | NotificationLimit int `json:",omitempty"` 196 | HasPrivateRegistrySupport bool `json:",omitempty"` 197 | } 198 | 199 | // Favourite is an image that a user wanted to keep track of 200 | type Favourite struct { 201 | User User 202 | UserID uint `gorm:"primary_key"` 203 | 204 | ImageName string `gorm:"primary_key" sql:"REFERENCES images(name) ON DELETE RESTRICT"` 205 | } 206 | 207 | type IsFavourite struct { 208 | IsFavourite bool 209 | } 210 | 211 | // Notification is an image that a user wants to be notified when it changes 212 | type Notification struct { 213 | ID uint `json:",omitempty", gorm:"primary_key"` 214 | UserID uint `json:"-" gorm:"ForeignKey:UserID" sql:"REFERENCES users(id) ON DELETE RESTRICT"` 215 | ImageName string `json:",omitempty" gorm:"ForeignKey:ImageName" sql:"REFERENCES images(name) ON DELETE RESTRICT"` 216 | WebhookURL string `json:",omitempty"` 217 | PageURL string `json:",omitempty" gorm: "-"` 218 | HistoryArray []NotificationMessage `json:"History,omitempty", gorm:"-"` 219 | } 220 | 221 | // NotificationMessage is a message sent to a webhook 222 | type NotificationMessage struct { 223 | gorm.Model `json:"-"` 224 | NotificationID uint `json:"-" gorm:"ForeignKey:NotificationID" sql:"REFERENCES notifications(id) ON DELETE RESTRICT"` 225 | ImageName string `json:"-"` 226 | WebhookURL string 227 | Message PostgresJSON `gorm:"type:jsonb"` 228 | Attempts int 229 | StatusCode int 230 | Response string 231 | SentAt time.Time 232 | } 233 | 234 | type NotificationMessageChanges struct { 235 | Text string `json:"text"` 236 | ImageName string `json:"image_name"` 237 | NewTags []Tag `json:"new_tags"` 238 | ChangedTags []Tag `json:"changed_tags"` 239 | DeletedTags []Tag `json:"deleted_tags"` 240 | } 241 | 242 | // NotificationList is a list of notifications with the count and limit for this user 243 | type NotificationList struct { 244 | NotificationCount int `gorm:"-"` 245 | NotificationLimit int `gorm:"-"` 246 | Notifications []NotificationStatus `gorm:"-"` 247 | } 248 | 249 | // NotificationStatus is a notification with its most recently sent message 250 | type NotificationStatus struct { 251 | ID int 252 | ImageName string 253 | WebhookURL string 254 | Message PostgresJSON 255 | StatusCode int 256 | Response string 257 | SentAt time.Time 258 | } 259 | 260 | // IsNotification returns whether a notification exists as well as the current count and limit 261 | type IsNotification struct { 262 | NotificationCount int 263 | NotificationLimit int 264 | Notification Notification 265 | } 266 | 267 | // Implement sql.Scanner interface to save a json.RawMessage to the database 268 | type PostgresJSON struct { 269 | json.RawMessage 270 | } 271 | 272 | func (j PostgresJSON) Value() (driver.Value, error) { 273 | return j.MarshalJSON() 274 | } 275 | 276 | func (j *PostgresJSON) Scan(src interface{}) error { 277 | if data, ok := src.([]byte); ok { 278 | return j.UnmarshalJSON(data) 279 | } 280 | return fmt.Errorf("Type assertion failed - src is type %T", src) 281 | } 282 | -------------------------------------------------------------------------------- /database/notification.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | const ( 8 | constNotificationStatusesSQL = ` 9 | SELECT n.id, n.image_name, n.webhook_url, 10 | COALESCE(nm.message, '{}') AS message, nm.sent_at, nm.response, nm.status_code 11 | FROM notifications n 12 | LEFT OUTER JOIN ( 13 | SELECT notification_id, MAX(id) AS max_id 14 | FROM notification_messages 15 | GROUP BY notification_id 16 | ) nmax ON n.id = nmax.notification_id 17 | LEFT OUTER JOIN notification_messages nm 18 | ON nmax.notification_id = nm.notification_id 19 | AND nmax.max_id = nm.id 20 | AND n.image_name = nm.image_name 21 | WHERE n.user_id = ? ORDER BY n.image_name` 22 | ) 23 | 24 | // GetNotifications gets all image notifications for a user along with the most 25 | // recently sent message 26 | func (d *PgDB) GetNotifications(user User) (list NotificationList, err error) { 27 | var notifications []NotificationStatus 28 | err = d.db.Raw(constNotificationStatusesSQL, user.ID). 29 | Find(¬ifications).Error 30 | if err != nil { 31 | log.Errorf("Error getting notifications: %v", err) 32 | } 33 | 34 | us, err := d.GetUserSetting(user) 35 | if err != nil { 36 | log.Errorf("Error getting user settings: %v", err) 37 | } 38 | 39 | list = NotificationList{ 40 | NotificationCount: len(notifications), 41 | NotificationLimit: us.NotificationLimit, 42 | Notifications: notifications, 43 | } 44 | 45 | return list, err 46 | } 47 | 48 | // GetNotification returns an image notification for this user 49 | func (d *PgDB) GetNotification(user User, id int) (notify Notification, err error) { 50 | err = d.db.Table("notifications"). 51 | Where(`"id" = ? AND "user_id" = ?`, id, user.ID). 52 | First(¬ify).Error 53 | if err != nil { 54 | log.Errorf("Error getting notification %d for user %d: %v", id, user.ID, err) 55 | } 56 | 57 | img, err := d.GetImage(notify.ImageName) 58 | if err != nil { 59 | log.Errorf("Error getting image %s for notification %d: %v", notify.ImageName, id, err) 60 | } 61 | 62 | // Set URL depending on whether image is public or private 63 | notify.PageURL = d.GetPageURL(img) 64 | 65 | return notify, err 66 | } 67 | 68 | // GetNotificationCount returns the number of notifications for a a user 69 | func (d *PgDB) GetNotificationCount(user User) (count int, err error) { 70 | err = d.db.Table("notifications"). 71 | Where("user_id = ?", user.ID).Count(&count).Error 72 | if err != nil { 73 | log.Errorf("Error getting notifications count: %v", err) 74 | } 75 | 76 | return count, err 77 | } 78 | 79 | // GetNotificationHistory returns a slice of the most recent notification messages 80 | func (d *PgDB) GetNotificationHistory(id int, image string, count int) (history []NotificationMessage, err error) { 81 | err = d.db.Table("notification_messages"). 82 | Where(`"notification_id" = ? AND image_name = ?`, id, image). 83 | Order("created_at DESC"). 84 | Limit(count). 85 | Find(&history).Error 86 | if err != nil { 87 | log.Errorf("Error getting history for notification %d - %v", id, err) 88 | } 89 | 90 | return history, err 91 | } 92 | 93 | // CreateNotification creates it 94 | func (d *PgDB) CreateNotification(user User, notify Notification) (Notification, error) { 95 | _, err := d.GetImage(notify.ImageName) 96 | if err != nil { 97 | log.Errorf("Error getting image %s - %v", notify.ImageName, err) 98 | return notify, err 99 | } 100 | 101 | count, err := d.GetNotificationCount(user) 102 | if err != nil { 103 | log.Errorf("Error getting notification count for user - %v", err) 104 | return notify, err 105 | } 106 | 107 | us, err := d.GetUserSetting(user) 108 | if err != nil { 109 | log.Errorf("Error getting user settings: %v", err) 110 | return notify, err 111 | } 112 | 113 | if count >= us.NotificationLimit { 114 | err = errors.New("Failed to create notification as limit is exceeded") 115 | return notify, err 116 | } 117 | 118 | err = d.db.Table("notifications"). 119 | Where(`"user_id" = ? AND "image_name" = ?`, notify.UserID, notify.ImageName). 120 | FirstOrCreate(¬ify).Error 121 | if err != nil { 122 | log.Errorf("Create Notification error %v", err) 123 | return notify, err 124 | } 125 | 126 | err = d.db.Save(¬ify).Error 127 | if err != nil { 128 | log.Errorf("Create Notification error 2: %v", err) 129 | } 130 | 131 | return notify, err 132 | } 133 | 134 | // UpdateNotification updates it 135 | func (d *PgDB) UpdateNotification(user User, id int, input Notification) (Notification, error) { 136 | _, err := d.GetImage(input.ImageName) 137 | if err != nil { 138 | log.Errorf("Error getting image %s - %v", input.ImageName, err) 139 | return input, err 140 | } 141 | 142 | // Get the saved notification. 143 | notify, err := d.GetNotification(user, id) 144 | if err == nil { 145 | // Set fields that need to be updated. 146 | notify.ImageName = input.ImageName 147 | notify.WebhookURL = input.WebhookURL 148 | 149 | err = d.db.Save(¬ify).Error 150 | if err != nil { 151 | log.Errorf("Update Notification error: %v", err) 152 | } 153 | 154 | } else { 155 | log.Errorf("Update Notification error 2: %v", err) 156 | } 157 | 158 | return notify, err 159 | } 160 | 161 | // DeleteNotification deletes a notification, returning an error if it doesn't exist 162 | func (d *PgDB) DeleteNotification(user User, id int) error { 163 | notify := Notification{} 164 | err := d.db.Where(`"id" = ? AND "user_id" = ?`, id, user.ID).Delete(¬ify).Error 165 | if err != nil { 166 | log.Debugf("Error Deleting notification: %v", err) 167 | } 168 | 169 | return err 170 | } 171 | 172 | // CreateNotificationMessage saves whether a notification was sent successfully 173 | func (d *PgDB) CreateNotificationMessage(msg *NotificationMessage) error { 174 | err := d.db.Table("notification_messages"). 175 | Create(&msg).Error 176 | if err != nil { 177 | log.Errorf("Save Notification Message error %v", err) 178 | } 179 | 180 | return err 181 | } 182 | 183 | // GetNotificationMessage saves whether a notification was sent successfully 184 | func (d *PgDB) GetNotificationMessage(id uint) (NotificationMessage, error) { 185 | var nm NotificationMessage 186 | err := d.db.First(&nm, id).Error 187 | if err != nil { 188 | log.Errorf("Failed to get NotificationMessage: %v", err) 189 | } 190 | 191 | return nm, err 192 | } 193 | 194 | // SaveNotificationMessage updates an existing notification message 195 | func (d *PgDB) SaveNotificationMessage(nm *NotificationMessage) error { 196 | err := d.db.Save(nm).Error 197 | if err != nil { 198 | log.Errorf("Failed to save NotificationMessage: %v", err) 199 | } 200 | 201 | return err 202 | } 203 | 204 | // GetNotificationsForImage gets a list of notifications we need to make for this image 205 | func (d *PgDB) GetNotificationsForImage(imageName string) (n []Notification, err error) { 206 | err = d.db.Where("image_name = ?", imageName).Find(&n).Error 207 | return n, err 208 | } 209 | 210 | // GetNotificationForUser returns true and the notification if it exists for this user and image 211 | func (d *PgDB) GetNotificationForUser(user User, image string) (bool, Notification) { 212 | var n Notification 213 | 214 | err := d.db.Where(`"user_id" = ? AND "image_name" = ?`, user.ID, image). 215 | First(&n).Error 216 | return (err == nil), n 217 | } 218 | -------------------------------------------------------------------------------- /database/notification_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequired 2 | 3 | package database 4 | 5 | import ( 6 | "encoding/json" 7 | "reflect" 8 | "testing" 9 | 10 | "github.com/markbates/goth" 11 | ) 12 | 13 | func TestNotifications(t *testing.T) { 14 | var err error 15 | var db PgDB 16 | var imageName = "lizrice/childimage" 17 | var webhookURL = "https://hooks.slack.com/services/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" 18 | var secondWebhookURL = "https://hooks.example.com/test" 19 | var pageURL = "https://microbadger.com/images/lizrice/childimage" 20 | var count int 21 | 22 | db = getDatabase(t) 23 | emptyDatabase(db) 24 | addThings(db) 25 | 26 | u, err := db.GetOrCreateUser(User{}, goth.User{Provider: "myprov", UserID: "12345", Name: "myname", Email: "me@myaddress.com"}) 27 | if err != nil { 28 | t.Errorf("Error creating user %v", err) 29 | } 30 | 31 | // Check there are no notifications 32 | n, _ := db.GetNotifications(u) 33 | if len(n.Notifications) > 0 { 34 | t.Errorf("Unexpected notifications %v", n.Notifications) 35 | } 36 | 37 | count, _ = db.GetNotificationCount(u) 38 | if count != 0 { 39 | t.Errorf("Expected notification count to be 0 but was %d", count) 40 | } 41 | 42 | notify := Notification{UserID: u.ID, 43 | ImageName: imageName, 44 | WebhookURL: webhookURL} 45 | 46 | // Add, get, update, list and delete a notification 47 | notify, err = db.CreateNotification(u, notify) 48 | if err != nil { 49 | t.Errorf("Failed to create notification %v", err) 50 | } 51 | 52 | count, _ = db.GetNotificationCount(u) 53 | if count != 1 { 54 | t.Errorf("Expected notification count to be 1 but was %d", count) 55 | } 56 | 57 | notify, err = db.GetNotification(u, int(notify.ID)) 58 | if err != nil { 59 | t.Errorf("Error getting notification") 60 | } 61 | if notify.WebhookURL != webhookURL { 62 | t.Errorf("Added notification doesn't exist") 63 | } 64 | 65 | if notify.PageURL != pageURL { 66 | t.Errorf("Expected page URL to be %s but was %s", pageURL, notify.PageURL) 67 | } 68 | 69 | isPresent, _ := db.GetNotificationForUser(u, imageName) 70 | if !isPresent { 71 | t.Errorf("Expected notification was not found for this user") 72 | } 73 | 74 | isPresent, _ = db.GetNotificationForUser(u, "lizrice/featured") 75 | if isPresent { 76 | t.Errorf("Unexpected notification was found for this user") 77 | } 78 | 79 | notify.WebhookURL = secondWebhookURL 80 | notify, err = db.UpdateNotification(u, int(notify.ID), notify) 81 | if notify.WebhookURL != secondWebhookURL { 82 | t.Errorf("Notification not updated") 83 | } 84 | 85 | n, _ = db.GetNotifications(u) 86 | 87 | if (len(n.Notifications) != 1) || (n.Notifications[0].WebhookURL != secondWebhookURL) || 88 | (n.Notifications[0].StatusCode != 0) { 89 | t.Errorf("Unexpected notifications %v", n.Notifications) 90 | } 91 | 92 | nmc := getNotificationMessageChanges() 93 | 94 | nmcAsJSON, err := json.Marshal(nmc) 95 | if err != nil { 96 | t.Fatalf("Failed to marshal NMC: %v", err) 97 | } 98 | 99 | msg := NotificationMessage{NotificationID: notify.ID, 100 | ImageName: imageName, 101 | WebhookURL: webhookURL, 102 | Attempts: 1, 103 | StatusCode: 200, 104 | Response: `{"errors":[]}"`, 105 | Message: PostgresJSON{nmcAsJSON}, 106 | } 107 | 108 | err = db.SaveNotificationMessage(&msg) 109 | if err != nil { 110 | t.Errorf("Error saving notification message %v", err) 111 | } 112 | 113 | n, _ = db.GetNotifications(u) 114 | 115 | if (len(n.Notifications) != 1) || (n.Notifications[0].WebhookURL != secondWebhookURL) || 116 | (n.Notifications[0].StatusCode != 200) { 117 | t.Errorf("Unexpected notifications %v", n.Notifications) 118 | } 119 | 120 | err = db.DeleteNotification(u, int(notify.ID)) 121 | if err != nil { 122 | t.Errorf("Failed to delete notification") 123 | } 124 | } 125 | 126 | func TestNotificationLimit(t *testing.T) { 127 | var err error 128 | var db PgDB 129 | 130 | db = getDatabase(t) 131 | emptyDatabase(db) 132 | addThings(db) 133 | 134 | u, err := db.GetOrCreateUser(User{}, goth.User{Provider: "myprov", UserID: "12345", Name: "myname", Email: "me@myaddress.com"}) 135 | if err != nil { 136 | t.Errorf("Error creating user %v", err) 137 | } 138 | 139 | us, _ := db.GetUserSetting(u) 140 | if us.NotificationLimit != 10 { 141 | t.Error("Error default notification limit should be 10") 142 | } 143 | 144 | // Create a notification 145 | notify := Notification{UserID: u.ID, 146 | ImageName: "lizrice/childimage", 147 | WebhookURL: "http://example.com/webhook"} 148 | 149 | notify, err = db.CreateNotification(u, notify) 150 | if err != nil { 151 | t.Errorf("Failed to create notification") 152 | } 153 | 154 | // Set the notification limit to one so we can easily check what happens when we try to exceed it 155 | us.NotificationLimit = 1 156 | err = db.db.Save(us).Error 157 | if err != nil { 158 | log.Errorf("Failed to save UserSettings: %v", err) 159 | } 160 | 161 | notify, err = db.CreateNotification(u, notify) 162 | if err == nil { 163 | t.Errorf("Should have failed to create 2nd notification") 164 | } 165 | } 166 | 167 | func TestNotificationMessages(t *testing.T) { 168 | var err error 169 | var db PgDB 170 | var imageName = "lizrice/childimage" 171 | var webhookURL = "https://hooks.example.com/test" 172 | 173 | db = getDatabase(t) 174 | emptyDatabase(db) 175 | addThings(db) 176 | 177 | u, err := db.GetOrCreateUser(User{}, goth.User{Provider: "myprov", UserID: "12345", Name: "myname", Email: "me@myaddress.com"}) 178 | if err != nil { 179 | t.Errorf("Error creating user %v", err) 180 | } 181 | 182 | // Create a notification 183 | notify := Notification{UserID: u.ID, 184 | ImageName: imageName, 185 | WebhookURL: webhookURL} 186 | 187 | notify, err = db.CreateNotification(u, notify) 188 | if err != nil { 189 | t.Errorf("Failed to create notification") 190 | } 191 | 192 | nmc := getNotificationMessageChanges() 193 | 194 | nmcAsJSON, err := json.Marshal(nmc) 195 | if err != nil { 196 | t.Fatalf("Failed to marshal NMC: %v", err) 197 | } 198 | 199 | msg := NotificationMessage{NotificationID: notify.ID, 200 | ImageName: imageName, 201 | WebhookURL: webhookURL, 202 | Attempts: 1, 203 | StatusCode: 200, 204 | Response: `{"errors":[]}"`, 205 | Message: PostgresJSON{nmcAsJSON}, 206 | } 207 | 208 | err = db.SaveNotificationMessage(&msg) 209 | if err != nil { 210 | t.Errorf("Error saving notification message %v", err) 211 | } 212 | 213 | // Test you can get it back out again 214 | msg2, err := db.GetNotificationMessage(msg.ID) 215 | if err != nil { 216 | t.Errorf("Error getting notification message %v", err) 217 | } 218 | 219 | if msg.ID != msg2.ID { 220 | log.Errorf("Failed to get notification message ID %d", msg.ID) 221 | } 222 | 223 | log.Debugf("Message is %s", string(msg2.Message.RawMessage)) 224 | var nmc2 NotificationMessageChanges 225 | err = json.Unmarshal(msg2.Message.RawMessage, &nmc2) 226 | if err != nil { 227 | t.Fatalf("Failed to unmarshal into NMC: %v", err) 228 | } 229 | 230 | log.Debugf("Got NMC: %v", nmc2) 231 | if !reflect.DeepEqual(nmc, nmc2) { 232 | t.Fatalf("NMCs are not the same\n Got %v\n Exp %v", nmc2, nmc) 233 | } 234 | } 235 | 236 | func getNotificationMessageChanges() NotificationMessageChanges { 237 | nmc := NotificationMessageChanges{ 238 | ImageName: "lizrice/childimage", 239 | NewTags: []Tag{}, 240 | DeletedTags: []Tag{{Tag: "Tagx", SHA: "12345"}}, 241 | ChangedTags: []Tag{}, 242 | } 243 | 244 | return nmc 245 | } 246 | -------------------------------------------------------------------------------- /database/postgres.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/jinzhu/gorm" 9 | _ "github.com/jinzhu/gorm/dialects/postgres" 10 | logging "github.com/op/go-logging" 11 | "github.com/wader/gormstore" 12 | 13 | "github.com/microscaling/microbadger/utils" 14 | ) 15 | 16 | func init() { 17 | utils.InitLogging() 18 | } 19 | 20 | const ( 21 | constRecentImagesDays = 14 22 | constDisplayMaxImages = 5 23 | constImagesPerPage = 100 24 | constDBAttempts = 5 25 | ) 26 | 27 | // PgDB is our postgres database 28 | type PgDB struct { 29 | db *gorm.DB 30 | SessionStore *gormstore.Store 31 | SiteURL string 32 | } 33 | 34 | // Exec does a raw SQL command on the database 35 | func (d *PgDB) Exec(cmd string, params ...interface{}) { 36 | d.db.Exec(cmd, params...) 37 | } 38 | 39 | // GetDB returns a database connection. 40 | func GetDB() (db PgDB, err error) { 41 | host := utils.GetEnvOrDefault("MB_DB_HOST", "postgres") 42 | user := utils.GetEnvOrDefault("MB_DB_USER", "microbadger") 43 | dbname := utils.GetEnvOrDefault("MB_DB_NAME", "microbadger") 44 | password := utils.GetEnvOrDefault("MB_DB_PASSWORD", "microbadger") 45 | 46 | dbLogLevel := logging.GetLevel("mmdata") 47 | db, err = GetPostgres(host, user, dbname, password, (dbLogLevel == logging.DEBUG)) 48 | return db, err 49 | } 50 | 51 | // GetPostgres opens a database connection 52 | func GetPostgres(host string, user string, dbname string, password string, debug bool) (db PgDB, err error) { 53 | params := fmt.Sprintf("host=%s user=%s dbname=%s sslmode=disable password=%s", host, user, dbname, password) 54 | log.Debugf("Opening postgres with params %s", params) 55 | db = PgDB{} 56 | var gormDb *gorm.DB 57 | 58 | attempts := 0 59 | for { 60 | gormDb, err = gorm.Open("postgres", params) 61 | if err == nil { 62 | break 63 | } 64 | 65 | time.Sleep(1 * time.Second) 66 | attempts++ 67 | if attempts <= constDBAttempts { 68 | log.Debugf("Sleep %d trying to connect to database: %v", attempts, err) 69 | } else { 70 | log.Errorf("Error connecting to database: %v", err) 71 | log.Debugf("%s", params) 72 | return db, err 73 | } 74 | } 75 | 76 | if debug { 77 | // log.Info("Database logging on") 78 | db.db = gormDb.Debug() 79 | } else { 80 | // log.Info("Database logging off") 81 | db.db = gormDb 82 | } 83 | 84 | db.db.AutoMigrate(&Registry{}, &Image{}, &ImageVersion{}, &Tag{}, &Favourite{}, &User{}, &UserAuth{}, &UserSetting{}, &UserImagePermission{}, &UserRegistryCredential{}, &Notification{}, &NotificationMessage{}) 85 | 86 | // Session store 87 | db.SessionStore = gormstore.New(db.db, []byte(os.Getenv("MB_SESSION_SECRET"))) 88 | // db cleanup every hour - close quit channel to stop cleanup (We don't do this) 89 | quit := make(chan struct{}) 90 | go db.SessionStore.PeriodicCleanup(1*time.Hour, quit) 91 | 92 | // Initialize the Docker Hub registry if it's not already there 93 | _, dockerMissing := db.GetRegistry("docker") 94 | if dockerMissing != nil { 95 | db.PutRegistry(&Registry{ID: "docker", Name: "Docker Hub", Url: "https://hub.docker.com"}) 96 | } 97 | 98 | // For building URLs 99 | db.SiteURL = utils.GetEnvOrDefault("MB_SITE_URL", "https://microbadger.com") 100 | 101 | return db, err 102 | } 103 | -------------------------------------------------------------------------------- /database/postgres_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequired 2 | 3 | package database 4 | 5 | import ( 6 | "testing" 7 | "time" 8 | 9 | logging "github.com/op/go-logging" 10 | ) 11 | 12 | // Setting up test database for this package 13 | // $ psql -c 'create database microbadger_database_test;' -U postgres 14 | 15 | func getDatabase(t *testing.T) PgDB { 16 | dbLogLevel := logging.GetLevel("mmdata") 17 | 18 | testdb, err := GetPostgres("localhost", "postgres", "microbadger_database_test", "", (dbLogLevel == logging.DEBUG)) 19 | if err != nil { 20 | t.Fatalf("Failed to open test database: %v", err) 21 | } 22 | 23 | return testdb 24 | } 25 | 26 | func emptyDatabase(db PgDB) { 27 | db.Exec("DELETE FROM tags") 28 | db.Exec("DELETE FROM image_versions") 29 | db.Exec("DELETE FROM images") 30 | db.Exec("DELETE FROM favourites") 31 | db.Exec("DELETE FROM notification_messages") 32 | db.Exec("SELECT setval('notifications_id_seq', 1, false)") 33 | db.Exec("DELETE FROM notifications") 34 | db.Exec("DELETE FROM users") 35 | db.Exec("SELECT setval('users_id_seq', 1, false)") 36 | db.Exec("DELETE from user_auths") 37 | db.Exec("DELETE from user_image_permissions") 38 | db.Exec("DELETE from user_registry_credentials") 39 | db.Exec("DELETE from user_settings") 40 | db.Exec("DELETE FROM sessions") 41 | db.Exec("DELETE FROM registries WHERE id <> 'docker'") // don't delete the pre-installed Docker registry 42 | } 43 | 44 | func addThings(db PgDB) { 45 | now := time.Now().UTC() 46 | 47 | db.Exec("INSERT INTO images (name, status, badge_count, created_at, auth_token, pull_count, latest) VALUES('lizrice/childimage', 'INSPECTED', 2, $1, 'lowercase', 10, '10000')", now) 48 | // We wouldn't expect to have any image versions yet for an image in SITEMAP status 49 | db.Exec("INSERT INTO images (name, status, badge_count, created_at, auth_token, pull_count, latest) VALUES('public/sitemap', 'SITEMAP', 2, $1, 'lowercase', 10, '12000')", now) 50 | db.Exec("INSERT INTO images (name, status, badge_count, created_at, featured, auth_token, pull_count, latest) VALUES('lizrice/featured', 'INSPECTED', 2, $1, True, 'mIxeDcAse', 1000, '15000')", now) 51 | db.Exec("INSERT INTO images (name, status, badge_count, created_at, featured, auth_token, pull_count, latest, is_private) VALUES('myuser/private', 'INSPECTED', 2, $1, True, 'mIxeDcAse', 1000, '20000', True)", now) 52 | 53 | db.Exec("INSERT INTO image_versions (image_name, labels, sha) VALUES('lizrice/childimage', 'org.label-schema.name=childimage', '10000')") 54 | db.Exec("INSERT INTO image_versions (image_name, sha) VALUES('lizrice/featured', '15000')") 55 | db.Exec("INSERT INTO image_versions (image_name, labels, sha) VALUES('myuser/private', 'org.label-schema.name=private', '20000')") 56 | } 57 | -------------------------------------------------------------------------------- /database/registry.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | func (d *PgDB) GetRegistry(id string) (reg *Registry, err error) { 4 | // Need to initialize an empty Registry so that we get the table name correctly 5 | reg = &Registry{} 6 | err = d.db.Where("id = ?", id).First(reg).Error 7 | return reg, err 8 | } 9 | 10 | func (d *PgDB) PutRegistry(reg *Registry) error { 11 | err := d.db.FirstOrCreate(reg).Error 12 | return err 13 | } 14 | -------------------------------------------------------------------------------- /database/registry_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequired 2 | 3 | package database 4 | 5 | import ( 6 | "testing" 7 | ) 8 | 9 | func TestRegistry(t *testing.T) { 10 | var err error 11 | var db PgDB 12 | 13 | reg := Registry{ 14 | ID: "test", 15 | Name: "My test registry", 16 | Url: "http://test", 17 | } 18 | 19 | db = getDatabase(t) 20 | 21 | // Before we empty the database, check that we are setting up the default Docker Hub entry correctly 22 | _, err = db.GetRegistry("docker") 23 | if err != nil { 24 | t.Errorf("Didn't initialize the Docker Registry entry: %v", err) 25 | } 26 | 27 | emptyDatabase(db) 28 | 29 | err = db.PutRegistry(®) 30 | if err != nil { 31 | t.Errorf("Error creating registry %v", err) 32 | } 33 | 34 | r, err := db.GetRegistry("test") 35 | if err != nil { 36 | t.Errorf("Error getting registry: %v", err) 37 | } 38 | 39 | if r.Name != reg.Name || r.ID != reg.ID { 40 | t.Errorf("Unexpected: %v ", r) 41 | } 42 | 43 | _, err = db.GetRegistry("notthere") 44 | if err == nil { 45 | t.Errorf("Shouldn't be able to get missing registry") 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | api: 4 | build: . 5 | command: "api" 6 | links: 7 | - nats 8 | - postgres 9 | ports: 10 | - "8080:8080" 11 | env_file: .env 12 | environment: 13 | - NATS_SEND_QUEUE_NAME=$MB_IMAGE_QUEUE_NAME 14 | depends_on: 15 | - nats 16 | - postgres 17 | 18 | inspector: 19 | build: . 20 | command: "inspector" 21 | links: 22 | - nats 23 | - postgres 24 | env_file: .env 25 | environment: 26 | - NATS_RECEIVE_QUEUE_NAME=$MB_IMAGE_QUEUE_NAME 27 | - NATS_SEND_QUEUE_NAME=$MB_SIZE_QUEUE_NAME 28 | depends_on: 29 | - nats 30 | - postgres 31 | 32 | size: 33 | build: . 34 | command: "size" 35 | links: 36 | - nats 37 | - postgres 38 | env_file: .env 39 | environment: 40 | - NATS_RECEIVE_QUEUE_NAME=$MB_SIZE_QUEUE_NAME 41 | depends_on: 42 | - nats 43 | - postgres 44 | 45 | notifier: 46 | build: . 47 | entrypoint: "/notifier" 48 | links: 49 | - nats 50 | - postgres 51 | env_file: .env 52 | depends_on: 53 | - nats 54 | - postgres 55 | 56 | nats: 57 | image: nats:2.1.7-alpine 58 | ports: 59 | - "4222" 60 | 61 | postgres: 62 | build: postgres 63 | ports: 64 | - "5432" 65 | -------------------------------------------------------------------------------- /encryption/encrypt.go: -------------------------------------------------------------------------------- 1 | package encryption 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "crypto/rand" 7 | "encoding/base64" 8 | "errors" 9 | "io" 10 | "os" 11 | 12 | "github.com/op/go-logging" 13 | 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/session" 16 | "github.com/aws/aws-sdk-go/service/kms" 17 | 18 | "github.com/microscaling/microbadger/utils" 19 | ) 20 | 21 | var ( 22 | log = logging.MustGetLogger("mmenc") 23 | ) 24 | 25 | const kmsKeySpec = "AES_256" 26 | 27 | type Service interface { 28 | Encrypt(input string) (encKey string, encVal string, err error) 29 | Decrypt(encKey string, encVal string) (result string, err error) 30 | } 31 | 32 | // EncryptionService for encrypting data using KMS. 33 | type EncryptionService struct { 34 | svc *kms.KMS 35 | keyName string 36 | } 37 | 38 | // NewService opens a new session with KMS. 39 | func NewService() EncryptionService { 40 | r := aws.String(utils.GetEnvOrDefault("AWS_REGION", "us-east-1")) 41 | s := session.New(&aws.Config{Region: r}) 42 | 43 | k := os.Getenv("KMS_ENCRYPTION_KEY_NAME") 44 | 45 | return EncryptionService{ 46 | svc: kms.New(s), 47 | keyName: k, 48 | } 49 | } 50 | 51 | // Encrypt a string using AES 256 bit encryption. The encryption key is 52 | // generated by KMS and an encrypted copy is returned which must also be stored. 53 | // The encrypted value starts with a random IV that is needed to decrypt. 54 | func (e EncryptionService) Encrypt(input string) (encKey string, encVal string, err error) { 55 | log.Debug("Encrypting string") 56 | 57 | if e.keyName == "" { 58 | return "", "", errors.New("Missing encryption key name") 59 | } 60 | 61 | plaintext := []byte(input) 62 | 63 | plaintextKey, encryptedKey, err := e.generateKey() 64 | if err != nil { 65 | log.Errorf("Error generating encryption key - %v", err) 66 | return "", "", err 67 | } 68 | 69 | c, err := aes.NewCipher(plaintextKey) 70 | if err != nil { 71 | log.Errorf("Error creating encryption cipher - %v", err) 72 | return "", "", err 73 | } 74 | 75 | // Clear key as soon as its no longer needed 76 | plaintextKey = nil 77 | 78 | // Generate a random IV 79 | ciphertext := make([]byte, aes.BlockSize+len(plaintext)) 80 | iv := ciphertext[:aes.BlockSize] 81 | if _, err := io.ReadFull(rand.Reader, iv); err != nil { 82 | return "", "", err 83 | } 84 | 85 | stream := cipher.NewCFBEncrypter(c, iv) 86 | stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext) 87 | 88 | // Encode as base64 for storing in the database 89 | encKey = base64.StdEncoding.EncodeToString(encryptedKey) 90 | encVal = base64.StdEncoding.EncodeToString(ciphertext) 91 | 92 | return encKey, encVal, err 93 | } 94 | 95 | // Decrypt a string using AES 256 bit encryption. The encrypted key is first 96 | // decrypted by KMS. The encrypted value starts with a random IV that was 97 | // generated when it was encrypted. 98 | func (e EncryptionService) Decrypt(encKey string, encVal string) (res string, err error) { 99 | log.Debug("Decrypting string") 100 | 101 | if e.keyName == "" { 102 | return "", errors.New("Missing encryption key name") 103 | } 104 | 105 | encryptedKey, err := base64.StdEncoding.DecodeString(encKey) 106 | if err != nil { 107 | log.Errorf("Error decoding encrypted key - %v", err) 108 | return "", err 109 | } 110 | 111 | plaintextKey, err := e.decryptKey(encryptedKey) 112 | if err != nil { 113 | log.Errorf("Error decrypting key using KMS API - %v", err) 114 | return "", err 115 | } 116 | 117 | c, err := aes.NewCipher(plaintextKey) 118 | if err != nil { 119 | log.Errorf("Error creating decryption cipher - %v", err) 120 | return "", err 121 | } 122 | 123 | // Clear key as soon as its no longer needed 124 | plaintextKey = nil 125 | 126 | encryptedVal, err := base64.StdEncoding.DecodeString(encVal) 127 | if err != nil { 128 | log.Errorf("Error decoding encrypted value - %v", err) 129 | return "", err 130 | } 131 | 132 | if len(encVal) < aes.BlockSize { 133 | return "", errors.New("Error cipher text is smaller than block size") 134 | } 135 | 136 | // Separate the IV and the encrypted value 137 | iv := encryptedVal[:aes.BlockSize] 138 | ciphertext := encryptedVal[aes.BlockSize:] 139 | 140 | stream := cipher.NewCFBDecrypter(c, iv) 141 | stream.XORKeyStream(ciphertext, ciphertext) 142 | 143 | return string(ciphertext), err 144 | } 145 | 146 | // Generate an encryption key using the KMS API. The master key is managed by AWS. 147 | func (e EncryptionService) generateKey() (plaintextKey []byte, encKey []byte, err error) { 148 | params := &kms.GenerateDataKeyInput{ 149 | KeyId: aws.String(e.keyName), 150 | KeySpec: aws.String(kmsKeySpec), 151 | } 152 | 153 | output, err := e.svc.GenerateDataKey(params) 154 | if err != nil { 155 | log.Errorf("Error generating data key using KMS API - %v", err) 156 | return nil, nil, err 157 | } 158 | 159 | return output.Plaintext, output.CiphertextBlob, err 160 | } 161 | 162 | // Decrypt an encryption key using the KMS API. 163 | func (e EncryptionService) decryptKey(encryptedKey []byte) ([]byte, error) { 164 | params := &kms.DecryptInput{ 165 | CiphertextBlob: encryptedKey, 166 | } 167 | 168 | output, err := e.svc.Decrypt(params) 169 | if err != nil { 170 | log.Errorf("Error decrypting data key using KMS API - %v", err) 171 | return nil, err 172 | } 173 | 174 | return output.Plaintext, err 175 | } 176 | -------------------------------------------------------------------------------- /encryption/encrypt_test.go: -------------------------------------------------------------------------------- 1 | package encryption 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestEncryptAndDecrypt(t *testing.T) { 8 | es := NewMockService() 9 | tests := []string{ 10 | "Q_Qesb1Z2hA7H94iXu3_buJeQ7416", 11 | } 12 | 13 | for _, val := range tests { 14 | encKey, encVal, err := es.Encrypt(val) 15 | if err != nil { 16 | t.Errorf("Error encrypting string %v", err) 17 | } 18 | 19 | res, err := es.Decrypt(encKey, encVal) 20 | if err != nil { 21 | t.Errorf("Error decrypting string %v", err) 22 | } 23 | 24 | if res != val { 25 | t.Error("Encrypted and decrypted values do not match") 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /encryption/mock.go: -------------------------------------------------------------------------------- 1 | package encryption 2 | 3 | // MockService for tests 4 | type MockService struct{} 5 | 6 | // make sure it satisfies the interface 7 | var _ Service = (*MockService)(nil) 8 | 9 | func NewMockService() MockService { 10 | return MockService{} 11 | } 12 | 13 | // Encrypt on mock returns static data 14 | func (q MockService) Encrypt(input string) (encKey string, encVal string, err error) { 15 | encKey = `AQEDAHithgxYTcdIpj37aYm1VAycoViFcSM2L+KQ42Aw3R0MdAAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMrhUevDKOjuP7L1 16 | XAIBEIA7/F9A1spnmoaehxqU5fi8lBwiZECAvXkSI33YPgJGAsCqmlEAQuirHHp4av4lI7jjvWCIj/nyHxa6Ss8=` 17 | encVal = "+DKd7lg8HsLD+ISl7ZrP0n6XhmrTRCYDVq4Zj9hTrL1JjxAb2fGsp/2DMSPy" 18 | err = nil 19 | 20 | return encKey, encVal, err 21 | } 22 | 23 | // Decrypt on mock returns static data 24 | func (q MockService) Decrypt(encKey string, envVal string) (result string, err error) { 25 | result = "Q_Qesb1Z2hA7H94iXu3_buJeQ7416" 26 | err = nil 27 | 28 | return result, err 29 | } 30 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/microscaling/microbadger 2 | 3 | go 1.14 4 | 5 | require ( 6 | code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 7 | github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 8 | github.com/aws/aws-sdk-go v1.29.14 9 | github.com/gorilla/mux v1.7.4 10 | github.com/gorilla/sessions v1.2.0 11 | github.com/jinzhu/gorm v1.9.12 12 | github.com/markbates/goth v1.61.2 13 | github.com/nats-io/nats-server/v2 v2.1.7 // indirect 14 | github.com/nats-io/nats.go v1.10.0 15 | github.com/onsi/ginkgo v1.12.0 // indirect 16 | github.com/onsi/gomega v1.9.0 // indirect 17 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 18 | github.com/rs/cors v1.7.0 19 | github.com/stretchr/testify v1.5.1 // indirect 20 | github.com/urfave/negroni v1.0.0 21 | github.com/wader/gormstore v0.0.0-20190904144442-d36772af4310 22 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect 23 | google.golang.org/protobuf v1.24.0 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /helm/microbadger/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "0.17.0" 3 | description: A Helm chart for MicroBadger API, Inspector, Size and Notifier components. 4 | name: microbadger 5 | version: 0.17.0 6 | -------------------------------------------------------------------------------- /helm/microbadger/templates/_microscaling-config: -------------------------------------------------------------------------------- 1 | {{ define "microscaling-config" }} 2 | { 3 | "user": "microscaling", 4 | "maxContainers": 10, 5 | "apps": [ 6 | { 7 | "name": {{ .Values.inspector.name | quote }}, 8 | "appType": "Kubernetes", 9 | "ruleType": "SimpleQueue", 10 | "metricType": "SQS", 11 | "priority": 1, 12 | "maxContainers": {{ .Values.inspector.maxReplicas }}, 13 | "minContainers": {{ .Values.inspector.minReplicas }}, 14 | "config": { 15 | "queueURL": "{{ .Values.sqs.baseURL }}{{ .Values.sqs.queues.inspect }}", 16 | "targetQueueLength": 2 17 | } 18 | }, 19 | { 20 | "name": {{ .Values.size.name | quote }}, 21 | "appType": "Kubernetes", 22 | "ruleType": "SimpleQueue", 23 | "metricType": "SQS", 24 | "priority": 2, 25 | "maxContainers": {{ .Values.size.maxReplicas }}, 26 | "minContainers": {{ .Values.size.minReplicas }}, 27 | "config": { 28 | "queueURL": "{{ .Values.sqs.baseURL }}{{ .Values.sqs.queues.size }}", 29 | "targetQueueLength": 2 30 | } 31 | } 32 | ] 33 | } 34 | {{ end }} 35 | -------------------------------------------------------------------------------- /helm/microbadger/templates/api-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.api.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.api.name }} 8 | spec: 9 | replicas: {{ .Values.api.replicas }} 10 | selector: 11 | matchLabels: 12 | app: {{ .Values.api.name }} 13 | template: 14 | metadata: 15 | labels: 16 | app: {{ .Values.api.name }} 17 | spec: 18 | imagePullSecrets: 19 | - name: {{ .Values.image.pullSecret }} 20 | containers: 21 | - name: {{ .Values.api.name }} 22 | image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 23 | args: [{{ .Values.api.args | quote }}] 24 | env: 25 | - name: AWS_ACCESS_KEY_ID 26 | valueFrom: 27 | secretKeyRef: 28 | name: {{ .Values.secret.name }} 29 | key: aws.accesskey 30 | - name: AWS_REGION 31 | valueFrom: 32 | configMapKeyRef: 33 | name: {{ .Values.configmap.name }} 34 | key: aws.region 35 | - name: AWS_SECRET_ACCESS_KEY 36 | valueFrom: 37 | secretKeyRef: 38 | name: {{ .Values.secret.name }} 39 | key: aws.secretkey 40 | - name: MB_DB_HOST 41 | valueFrom: 42 | configMapKeyRef: 43 | name: {{ .Values.configmap.name }} 44 | key: mb.db.host 45 | - name: MB_DB_NAME 46 | valueFrom: 47 | configMapKeyRef: 48 | name: {{ .Values.configmap.name }} 49 | key: mb.db.name 50 | - name: MB_DB_PASSWORD 51 | valueFrom: 52 | secretKeyRef: 53 | name: {{ .Values.secret.name }} 54 | key: database.password 55 | - name: MB_DB_USER 56 | valueFrom: 57 | configMapKeyRef: 58 | name: {{ .Values.configmap.name }} 59 | key: mb.db.user 60 | - name: MB_API_URL 61 | valueFrom: 62 | configMapKeyRef: 63 | name: {{ .Values.configmap.name }} 64 | key: mb.api.url 65 | - name: MB_SITE_URL 66 | valueFrom: 67 | configMapKeyRef: 68 | name: {{ .Values.configmap.name }} 69 | key: mb.site.url 70 | - name: MB_WEBHOOK_URL 71 | valueFrom: 72 | configMapKeyRef: 73 | name: {{ .Values.configmap.name }} 74 | key: mb.hooks.url 75 | - name: SLACK_WEBHOOK 76 | valueFrom: 77 | configMapKeyRef: 78 | name: {{ .Values.configmap.name }} 79 | key: slack.webhook 80 | - name: SQS_SEND_QUEUE_URL 81 | valueFrom: 82 | configMapKeyRef: 83 | name: {{ .Values.configmap.name }} 84 | key: sqs.inspect.queue 85 | - name: SQS_NOTIFY_QUEUE_URL 86 | valueFrom: 87 | configMapKeyRef: 88 | name: {{ .Values.configmap.name }} 89 | key: sqs.notify.queue 90 | - name: MB_SESSION_SECRET 91 | valueFrom: 92 | secretKeyRef: 93 | name: {{ .Values.secret.name }} 94 | key: session.secret 95 | - name: MB_DEBUG_CORS 96 | valueFrom: 97 | configMapKeyRef: 98 | name: {{ .Values.configmap.name }} 99 | key: mb.cors.debug 100 | - name: MB_CORS_ORIGIN 101 | valueFrom: 102 | configMapKeyRef: 103 | name: {{ .Values.configmap.name }} 104 | key: mb.cors.origin 105 | - name: MB_GITHUB_KEY 106 | valueFrom: 107 | secretKeyRef: 108 | name: {{ .Values.secret.name }} 109 | key: github.key 110 | - name: MB_GITHUB_SECRET 111 | valueFrom: 112 | secretKeyRef: 113 | name: {{ .Values.secret.name }} 114 | key: github.secret 115 | - name: KMS_ENCRYPTION_KEY_NAME 116 | valueFrom: 117 | configMapKeyRef: 118 | name: {{ .Values.configmap.name }} 119 | key: kms.encryption.key 120 | imagePullPolicy: IfNotPresent 121 | ports: 122 | - containerPort: {{ .Values.port }} 123 | protocol: TCP 124 | resources: 125 | {{ toYaml .Values.resources | indent 12 }} 126 | securityContext: 127 | privileged: false 128 | terminationMessagePath: /dev/termination-log 129 | dnsPolicy: ClusterFirst 130 | restartPolicy: Always 131 | securityContext: {} 132 | terminationGracePeriodSeconds: 30 133 | -------------------------------------------------------------------------------- /helm/microbadger/templates/api-ingress.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ingress.enabled }} 2 | apiVersion: extensions/v1beta1 3 | kind: Ingress 4 | metadata: 5 | name: {{ .Values.api.name }} 6 | namespace: {{ .Values.namespace }} 7 | labels: 8 | app: {{ .Values.api.name }} 9 | annotations: 10 | ingress.kubernetes.io/ssl-redirect: "true" 11 | kubernetes.io/tls-acme: "true" 12 | certmanager.k8s.io/issuer: {{ .Values.letsencrypt.issuer }} 13 | kubernetes.io/ingress.class: "nginx" 14 | spec: 15 | tls: 16 | - hosts: 17 | - {{ .Values.domains.api }} 18 | - {{ .Values.domains.hooks }} 19 | - {{ .Values.domains.images }} 20 | secretName: {{ .Values.letsencrypt.secret}} 21 | rules: 22 | - host: {{ .Values.domains.api }} 23 | http: 24 | paths: 25 | - path: / 26 | backend: 27 | serviceName: {{ .Values.api.name }} 28 | servicePort: 80 29 | - host: {{ .Values.domains.hooks }} 30 | http: 31 | paths: 32 | - path: / 33 | backend: 34 | serviceName: {{ .Values.api.name }} 35 | servicePort: 80 36 | - host: {{ .Values.domains.images }} 37 | http: 38 | paths: 39 | - path: / 40 | backend: 41 | serviceName: {{ .Values.api.name }} 42 | servicePort: 80 43 | {{ end }} 44 | -------------------------------------------------------------------------------- /helm/microbadger/templates/api-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ .Values.api.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.api.name }} 8 | spec: 9 | ports: 10 | - port: 80 11 | protocol: TCP 12 | targetPort: {{ .Values.port }} 13 | name: http 14 | selector: 15 | app: {{ .Values.api.name }} 16 | sessionAffinity: None 17 | type: ClusterIP 18 | -------------------------------------------------------------------------------- /helm/microbadger/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ .Values.configmap.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.name }} 8 | data: 9 | aws.region: {{ .Values.aws.region }} 10 | kms.encryption.key: {{ .Values.kms.key }} 11 | mb.api.url: https://{{ .Values.domains.api }} 12 | mb.cors.debug: 'false' 13 | mb.cors.origin: https://{{ .Values.domains.website }} 14 | mb.db.host: {{ .Values.database.host }} 15 | mb.db.name: {{ .Values.database.name }} 16 | mb.db.user: {{ .Values.database.user }} 17 | mb.hooks.url: https://{{ .Values.domains.hooks }} 18 | mb.site.url: https://{{ .Values.domains.website }} 19 | microscaling.config.data: {{ include "microscaling-config" . | quote }} 20 | slack.webhook: {{ .Values.slack.webhook }} 21 | sqs.inspect.queue: {{ .Values.sqs.baseURL }}{{ .Values.sqs.queues.inspect }} 22 | sqs.notify.queue: {{ .Values.sqs.baseURL }}{{ .Values.sqs.queues.notify }} 23 | sqs.size.queue: {{ .Values.sqs.baseURL }}{{ .Values.sqs.queues.size }} 24 | -------------------------------------------------------------------------------- /helm/microbadger/templates/inspector-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.inspector.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.inspector.name }} 8 | spec: 9 | replicas: {{ .Values.inspector.minReplicas }} 10 | selector: 11 | matchLabels: 12 | app: {{ .Values.inspector.name }} 13 | template: 14 | metadata: 15 | labels: 16 | app: {{ .Values.inspector.name }} 17 | spec: 18 | imagePullSecrets: 19 | - name: {{ .Values.image.pullSecret }} 20 | containers: 21 | - name: {{ .Values.inspector.name }} 22 | image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 23 | args: [{{ .Values.inspector.args | quote }}] 24 | env: 25 | - name: AWS_ACCESS_KEY_ID 26 | valueFrom: 27 | secretKeyRef: 28 | name: {{ .Values.secret.name }} 29 | key: aws.accesskey 30 | - name: AWS_REGION 31 | valueFrom: 32 | configMapKeyRef: 33 | name: {{ .Values.configmap.name }} 34 | key: aws.region 35 | - name: AWS_SECRET_ACCESS_KEY 36 | valueFrom: 37 | secretKeyRef: 38 | name: {{ .Values.secret.name }} 39 | key: aws.secretkey 40 | - name: MB_DB_HOST 41 | valueFrom: 42 | configMapKeyRef: 43 | name: {{ .Values.configmap.name }} 44 | key: mb.db.host 45 | - name: MB_DB_NAME 46 | valueFrom: 47 | configMapKeyRef: 48 | name: {{ .Values.configmap.name }} 49 | key: mb.db.name 50 | - name: MB_DB_PASSWORD 51 | valueFrom: 52 | secretKeyRef: 53 | name: {{ .Values.secret.name }} 54 | key: database.password 55 | - name: MB_DB_USER 56 | valueFrom: 57 | configMapKeyRef: 58 | name: {{ .Values.configmap.name }} 59 | key: mb.db.user 60 | - name: MB_SITE_URL 61 | valueFrom: 62 | configMapKeyRef: 63 | name: {{ .Values.configmap.name }} 64 | key: mb.site.url 65 | - name: MB_WEBHOOK_URL 66 | valueFrom: 67 | configMapKeyRef: 68 | name: {{ .Values.configmap.name }} 69 | key: mb.hooks.url 70 | - name: SLACK_WEBHOOK 71 | valueFrom: 72 | configMapKeyRef: 73 | name: {{ .Values.configmap.name }} 74 | key: slack.webhook 75 | - name: SQS_RECEIVE_QUEUE_URL 76 | valueFrom: 77 | configMapKeyRef: 78 | name: {{ .Values.configmap.name }} 79 | key: sqs.inspect.queue 80 | - name: SQS_SEND_QUEUE_URL 81 | valueFrom: 82 | configMapKeyRef: 83 | name: {{ .Values.configmap.name }} 84 | key: sqs.size.queue 85 | - name: SQS_NOTIFY_QUEUE_URL 86 | valueFrom: 87 | configMapKeyRef: 88 | name: {{ .Values.configmap.name }} 89 | key: sqs.notify.queue 90 | - name: KMS_ENCRYPTION_KEY_NAME 91 | valueFrom: 92 | configMapKeyRef: 93 | name: {{ .Values.configmap.name }} 94 | key: kms.encryption.key 95 | imagePullPolicy: IfNotPresent 96 | resources: 97 | {{ toYaml .Values.resources | indent 12 }} 98 | securityContext: 99 | privileged: false 100 | terminationMessagePath: /dev/termination-log 101 | dnsPolicy: ClusterFirst 102 | restartPolicy: Always 103 | securityContext: {} 104 | terminationGracePeriodSeconds: 30 105 | -------------------------------------------------------------------------------- /helm/microbadger/templates/microscaling-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.microscaling.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.microscaling.name }} 8 | spec: 9 | replicas: {{ .Values.microscaling.replicas }} 10 | selector: 11 | matchLabels: 12 | app: {{ .Values.microscaling.name }} 13 | template: 14 | metadata: 15 | labels: 16 | app: {{ .Values.microscaling.name }} 17 | spec: 18 | serviceAccountName: {{ .Values.microscaling.name }} 19 | containers: 20 | - name: {{ .Values.microscaling.name }} 21 | image: "{{ .Values.microscaling.image }}:{{ .Values.microscaling.tag }}" 22 | env: 23 | - name: AWS_ACCESS_KEY_ID 24 | valueFrom: 25 | secretKeyRef: 26 | name: {{ .Values.secret.name }} 27 | key: aws.accesskey 28 | - name: AWS_REGION 29 | valueFrom: 30 | configMapKeyRef: 31 | name: {{ .Values.configmap.name }} 32 | key: aws.region 33 | - name: AWS_SECRET_ACCESS_KEY 34 | valueFrom: 35 | secretKeyRef: 36 | name: {{ .Values.secret.name }} 37 | key: aws.secretkey 38 | - name: MSS_SCHEDULER 39 | value: "KUBERNETES" 40 | - name: MSS_DEMAND_ENGINE 41 | value: "LOCAL" 42 | - name: MSS_MONITOR 43 | value: "none" 44 | - name: MSS_SEND_METRICS_TO_API 45 | value: "false" 46 | - name: MSS_CONFIG 47 | value: "ENVVAR" 48 | - name: MSS_CONFIG_DATA 49 | valueFrom: 50 | configMapKeyRef: 51 | name: {{ .Values.configmap.name }} 52 | key: microscaling.config.data 53 | imagePullPolicy: Always 54 | resources: 55 | {{ toYaml .Values.resources | indent 12 }} 56 | securityContext: 57 | privileged: false 58 | terminationMessagePath: /dev/termination-log 59 | dnsPolicy: ClusterFirst 60 | restartPolicy: Always 61 | securityContext: {} 62 | terminationGracePeriodSeconds: 30 63 | -------------------------------------------------------------------------------- /helm/microbadger/templates/microscaling-rbac.yaml: -------------------------------------------------------------------------------- 1 | kind: Role 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | metadata: 4 | name: {{ .Values.microscaling.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.microscaling.name }} 8 | rules: 9 | - apiGroups: ["extensions", "apps"] 10 | resources: 11 | - deployments 12 | verbs: 13 | - get 14 | - watch 15 | - list 16 | - patch 17 | - update 18 | 19 | --- 20 | 21 | kind: RoleBinding 22 | apiVersion: rbac.authorization.k8s.io/v1beta1 23 | metadata: 24 | name: {{ .Values.microscaling.name }} 25 | namespace: {{ .Values.namespace }} 26 | labels: 27 | app: {{ .Values.microscaling.name }} 28 | subjects: 29 | - kind: ServiceAccount 30 | name: {{ .Values.microscaling.name }} 31 | namespace: {{ .Values.namespace }} 32 | roleRef: 33 | kind: Role 34 | name: {{ .Values.microscaling.name }} 35 | apiGroup: rbac.authorization.k8s.io 36 | -------------------------------------------------------------------------------- /helm/microbadger/templates/microscaling-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ .Values.microscaling.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.microscaling.name }} 8 | -------------------------------------------------------------------------------- /helm/microbadger/templates/notifier-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.notifier.enabled }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ .Values.notifier.name }} 6 | namespace: {{ .Values.namespace }} 7 | labels: 8 | app: {{ .Values.notifier.name }} 9 | spec: 10 | replicas: {{ .Values.notifier.replicas }} 11 | selector: 12 | matchLabels: 13 | app: {{ .Values.notifier.name }} 14 | template: 15 | metadata: 16 | labels: 17 | app: {{ .Values.notifier.name }} 18 | spec: 19 | imagePullSecrets: 20 | - name: {{ .Values.image.pullSecret }} 21 | containers: 22 | - name: {{ .Values.notifier.name }} 23 | image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 24 | command: [{{ .Values.notifier.command | quote }}] 25 | env: 26 | - name: AWS_ACCESS_KEY_ID 27 | valueFrom: 28 | secretKeyRef: 29 | name: {{ .Values.secret.name }} 30 | key: aws.accesskey 31 | - name: AWS_REGION 32 | valueFrom: 33 | configMapKeyRef: 34 | name: {{ .Values.configmap.name }} 35 | key: aws.region 36 | - name: AWS_SECRET_ACCESS_KEY 37 | valueFrom: 38 | secretKeyRef: 39 | name: {{ .Values.secret.name }} 40 | key: aws.secretkey 41 | - name: MB_DB_HOST 42 | valueFrom: 43 | configMapKeyRef: 44 | name: {{ .Values.configmap.name }} 45 | key: mb.db.host 46 | - name: MB_DB_NAME 47 | valueFrom: 48 | configMapKeyRef: 49 | name: {{ .Values.configmap.name }} 50 | key: mb.db.name 51 | - name: MB_DB_PASSWORD 52 | valueFrom: 53 | secretKeyRef: 54 | name: {{ .Values.secret.name }} 55 | key: database.password 56 | - name: MB_DB_USER 57 | valueFrom: 58 | configMapKeyRef: 59 | name: {{ .Values.configmap.name }} 60 | key: mb.db.user 61 | - name: MB_SITE_URL 62 | valueFrom: 63 | configMapKeyRef: 64 | name: {{ .Values.configmap.name }} 65 | key: mb.site.url 66 | - name: MB_WEBHOOK_URL 67 | valueFrom: 68 | configMapKeyRef: 69 | name: {{ .Values.configmap.name }} 70 | key: mb.hooks.url 71 | - name: SLACK_WEBHOOK 72 | valueFrom: 73 | configMapKeyRef: 74 | name: {{ .Values.configmap.name }} 75 | key: slack.webhook 76 | - name: SQS_NOTIFY_QUEUE_URL 77 | valueFrom: 78 | configMapKeyRef: 79 | name: {{ .Values.configmap.name }} 80 | key: sqs.notify.queue 81 | imagePullPolicy: IfNotPresent 82 | resources: 83 | {{ toYaml .Values.resources | indent 12 }} 84 | securityContext: 85 | privileged: false 86 | terminationMessagePath: /dev/termination-log 87 | dnsPolicy: ClusterFirst 88 | restartPolicy: Always 89 | securityContext: {} 90 | terminationGracePeriodSeconds: 30 91 | {{ end }} 92 | -------------------------------------------------------------------------------- /helm/microbadger/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ .Values.secret.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.name }} 8 | type: Opaque 9 | data: 10 | aws.accesskey: {{ .Values.secret.aws.accessKeyID | b64enc | quote }} 11 | aws.secretkey: {{ .Values.secret.aws.secretAccessKey | b64enc | quote }} 12 | database.password: {{ .Values.secret.database.password | b64enc | quote }} 13 | github.key: {{ .Values.secret.github.key | b64enc | quote }} 14 | github.secret: {{ .Values.secret.github.secret | b64enc | quote }} 15 | session.secret: {{ .Values.secret.session | b64enc | quote }} 16 | -------------------------------------------------------------------------------- /helm/microbadger/templates/size-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.size.name }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | app: {{ .Values.size.name }} 8 | spec: 9 | replicas: {{ .Values.size.minReplicas }} 10 | selector: 11 | matchLabels: 12 | app: {{ .Values.size.name }} 13 | template: 14 | metadata: 15 | labels: 16 | app: {{ .Values.size.name }} 17 | spec: 18 | imagePullSecrets: 19 | - name: {{ .Values.image.pullSecret }} 20 | containers: 21 | - name: {{ .Values.size.name }} 22 | image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 23 | args: [{{ .Values.size.args | quote }}] 24 | env: 25 | - name: AWS_ACCESS_KEY_ID 26 | valueFrom: 27 | secretKeyRef: 28 | name: {{ .Values.secret.name }} 29 | key: aws.accesskey 30 | - name: AWS_REGION 31 | valueFrom: 32 | configMapKeyRef: 33 | name: {{ .Values.configmap.name }} 34 | key: aws.region 35 | - name: AWS_SECRET_ACCESS_KEY 36 | valueFrom: 37 | secretKeyRef: 38 | name: {{ .Values.secret.name }} 39 | key: aws.secretkey 40 | - name: MB_DB_HOST 41 | valueFrom: 42 | configMapKeyRef: 43 | name: {{ .Values.configmap.name }} 44 | key: mb.db.host 45 | - name: MB_DB_NAME 46 | valueFrom: 47 | configMapKeyRef: 48 | name: {{ .Values.configmap.name }} 49 | key: mb.db.name 50 | - name: MB_DB_PASSWORD 51 | valueFrom: 52 | secretKeyRef: 53 | name: {{ .Values.secret.name }} 54 | key: database.password 55 | - name: MB_DB_USER 56 | valueFrom: 57 | configMapKeyRef: 58 | name: {{ .Values.configmap.name }} 59 | key: mb.db.user 60 | - name: MB_SITE_URL 61 | valueFrom: 62 | configMapKeyRef: 63 | name: {{ .Values.configmap.name }} 64 | key: mb.site.url 65 | - name: MB_WEBHOOK_URL 66 | valueFrom: 67 | configMapKeyRef: 68 | name: {{ .Values.configmap.name }} 69 | key: mb.hooks.url 70 | - name: SLACK_WEBHOOK 71 | valueFrom: 72 | configMapKeyRef: 73 | name: {{ .Values.configmap.name }} 74 | key: slack.webhook 75 | - name: SQS_RECEIVE_QUEUE_URL 76 | valueFrom: 77 | configMapKeyRef: 78 | name: {{ .Values.configmap.name }} 79 | key: sqs.size.queue 80 | - name: KMS_ENCRYPTION_KEY_NAME 81 | valueFrom: 82 | configMapKeyRef: 83 | name: {{ .Values.configmap.name }} 84 | key: kms.encryption.key 85 | imagePullPolicy: IfNotPresent 86 | resources: 87 | {{ toYaml .Values.resources | indent 12 }} 88 | securityContext: 89 | privileged: false 90 | terminationMessagePath: /dev/termination-log 91 | dnsPolicy: ClusterFirst 92 | restartPolicy: Always 93 | securityContext: {} 94 | terminationGracePeriodSeconds: 30 95 | -------------------------------------------------------------------------------- /helm/microbadger/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for microbadger-web. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | name: microbadger 6 | namespace: default 7 | port: 8080 8 | 9 | api: 10 | name: microbadger-api 11 | args: api 12 | replicas: 3 13 | 14 | aws: 15 | region: us-west-2 16 | 17 | configmap: 18 | name: microbadger-config 19 | 20 | database: 21 | host: microbadger-production.*.us-west-2.rds.amazonaws.com 22 | name: microbadger_production 23 | user: microbadger 24 | 25 | domains: 26 | api: api.microbadger.com 27 | hooks: hooks.microbadger.com 28 | images: images.microbadger.com 29 | website: microbadger.com 30 | 31 | image: 32 | pullSecret: quay-deploy-secret 33 | registry: quay.io 34 | repository: microscaling/microbadger 35 | tag: 0.15.17 36 | 37 | ingress: 38 | enabled: true 39 | 40 | inspector: 41 | name: inspector 42 | args: inspector 43 | maxReplicas: 8 44 | minReplicas: 2 45 | 46 | kms: 47 | key: alias/microbadger-production 48 | 49 | letsencrypt: 50 | issuer: letsencrypt-issuer 51 | secret: microbadger-api-letsencrypt 52 | 53 | microscaling: 54 | name: microscaling 55 | image: microscaling/microscaling 56 | replicas: 1 57 | tag: 0.9.2 58 | 59 | notifier: 60 | name: notifier 61 | command: /notifier 62 | enabled: true 63 | replicas: 1 64 | 65 | resources: 66 | limits: 67 | cpu: 100m 68 | memory: 150Mi 69 | requests: 70 | cpu: 100m 71 | memory: 150Mi 72 | 73 | secret: 74 | name: microbadger-secret 75 | aws: 76 | accessKeyID: aws-key 77 | secretAccessKey: aws-secret 78 | database: 79 | password: database-secret 80 | github: 81 | key: github-key 82 | secret: github-secret 83 | session: session-secret 84 | 85 | size: 86 | name: size 87 | args: size 88 | maxReplicas: 8 89 | minReplicas: 2 90 | 91 | slack: 92 | webhook: https://hooks.slack.com/* 93 | 94 | sqs: 95 | baseURL: https://sqs.us-west-2.amazonaws.com/*/ 96 | queues: 97 | inspect: microbadger-prod 98 | notify: microbadger-prod-notify 99 | size: microbadger-prod-size 100 | -------------------------------------------------------------------------------- /hub/hub.go: -------------------------------------------------------------------------------- 1 | package hub 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | "github.com/microscaling/microbadger/registry" 15 | "github.com/microscaling/microbadger/utils" 16 | "github.com/op/go-logging" 17 | ) 18 | 19 | const constHubURL = "hub.docker.com" 20 | const constImagesPerPage = 10 21 | 22 | var log = logging.MustGetLogger("mmhub") 23 | 24 | // Info is returned from the hub.docker.com/v2/repositories API 25 | type Info struct { 26 | Name string `json:"name"` 27 | Namespace string `json:"namespace"` 28 | Description string `json:"description"` 29 | FullDescription string `json:"full_description"` 30 | IsAutomated bool `json:"is_automated"` 31 | IsPrivate bool `json:"is_private"` 32 | LastUpdated *time.Time `json:"last_updated"` 33 | PullCount int `json:"pull_count"` 34 | StarCount int `json:"star_count"` 35 | } 36 | 37 | type infoResponse struct { 38 | Count int `json:"count"` 39 | Results []Info `json:"results"` 40 | } 41 | 42 | type loginRequest struct { 43 | Username string `json:"username"` 44 | Password string `json:"password"` 45 | } 46 | 47 | type loginResponse struct { 48 | Token string `json:"token"` 49 | } 50 | 51 | type namespacesResponse struct { 52 | Namespaces []string `json:"namespaces"` 53 | } 54 | 55 | type NamespaceList struct { 56 | Namespaces []string 57 | } 58 | 59 | type orgsResponse struct { 60 | Orgs []org `json:"results"` 61 | } 62 | 63 | type org struct { 64 | OrgName string `json:"orgname"` 65 | } 66 | 67 | type ImageList struct { 68 | CurrentPage int 69 | PageCount int 70 | ImageCount int 71 | Images []ImageInfo 72 | } 73 | 74 | type ImageInfo struct { 75 | ImageName string 76 | IsInspected bool 77 | IsPrivate bool 78 | } 79 | 80 | // InfoService connects to the Docker Hub over the internet 81 | type InfoService struct { 82 | client *http.Client 83 | baseURL string 84 | } 85 | 86 | // NewService is a real info service 87 | func NewService() InfoService { 88 | return InfoService{ 89 | client: &http.Client{ 90 | // TODO Make timeout configurable. 91 | Timeout: 10 * time.Second, 92 | }, 93 | baseURL: "https://" + constHubURL, 94 | } 95 | } 96 | 97 | // NewMockService is for testing 98 | func NewMockService(transport *http.Transport) InfoService { 99 | 100 | return InfoService{ 101 | client: &http.Client{ 102 | Transport: transport, 103 | }, 104 | baseURL: "http://fakehub", 105 | } 106 | } 107 | 108 | // Login logs in to get an auth token from Docker Hub 109 | func (hub *InfoService) Login(user string, password string) (token string, err error) { 110 | var lr loginResponse 111 | 112 | hubLoginURL := fmt.Sprintf("%s/v2/users/login/", hub.baseURL) 113 | log.Debugf("Logging into %s", hubLoginURL) 114 | 115 | l := loginRequest{ 116 | Username: user, 117 | Password: password, 118 | } 119 | 120 | b, err := json.Marshal(l) 121 | if err != nil { 122 | log.Errorf("Error marshalling login request - %v", err) 123 | return 124 | } 125 | 126 | buf := bytes.NewBuffer(b) 127 | req, _ := http.NewRequest("POST", hubLoginURL, buf) 128 | req.Header.Add("Content-Type", "application/json") 129 | 130 | resp, err := hub.client.Do(req) 131 | if err != nil { 132 | log.Errorf("Error making login request - %v", err) 133 | return 134 | } 135 | 136 | defer resp.Body.Close() 137 | if resp.StatusCode != http.StatusOK { 138 | log.Debugf("Failed to get login token %d: %s", resp.StatusCode, resp.Status) 139 | return "", errors.New("Incorrect credentials") 140 | } 141 | 142 | body, err := ioutil.ReadAll(resp.Body) 143 | if err != nil { 144 | log.Errorf("Error reading login response - %v", err) 145 | return 146 | } 147 | 148 | err = json.Unmarshal(body, &lr) 149 | if err != nil { 150 | log.Errorf("Error unmarshalling login response - %v", err) 151 | } 152 | 153 | token = lr.Token 154 | log.Debug("Got login token") 155 | 156 | return 157 | } 158 | 159 | // UserNamespaces gets the namespaces the user belongs to including any orgs they are a member of 160 | func (hub *InfoService) UserNamespaces(user string, password string) (namespaceList NamespaceList, err error) { 161 | dedupe := make(map[string]bool) 162 | 163 | loginToken, err := hub.Login(user, password) 164 | if err != nil { 165 | log.Errorf("Error logging in to docker hub %v", err) 166 | return 167 | } 168 | 169 | namespaces, err := hub.getNamespaces(loginToken) 170 | if err != nil { 171 | log.Errorf("Error getting namespaces %v", err) 172 | } 173 | 174 | for _, ns := range namespaces { 175 | dedupe[ns] = true 176 | } 177 | 178 | orgs, err := hub.getOrgs(loginToken) 179 | if err != nil { 180 | log.Errorf("Error getting orgs %v", err) 181 | } 182 | 183 | // Add any orgs that aren't in the namespaces list 184 | for _, org := range orgs { 185 | if _, ok := dedupe[org]; !ok { 186 | namespaces = append(namespaces, org) 187 | } 188 | } 189 | 190 | // Return a sorted slice 191 | namespaceList.Namespaces = namespaces 192 | sort.Strings(namespaceList.Namespaces) 193 | 194 | return 195 | } 196 | 197 | // UserNamespaceImages gets the list of images within a namespace 198 | func (hub *InfoService) UserNamespaceImages(user string, password string, namespace string, page int) (imageList ImageList, notfound bool, err error) { 199 | var ir infoResponse 200 | 201 | loginToken, err := hub.Login(user, password) 202 | if err != nil { 203 | log.Errorf("Error logging in to docker hub %v", err) 204 | return 205 | } 206 | 207 | hubURL := fmt.Sprintf("%s/v2/repositories/%s/?page=%d", hub.baseURL, namespace, page) 208 | log.Debugf("Getting namespace info %s", hubURL) 209 | 210 | req, _ := http.NewRequest("GET", hubURL, nil) 211 | req.Header.Add("Authorization", fmt.Sprintf("JWT %s", loginToken)) 212 | req.Header.Add("Content-Type", "application/json") 213 | 214 | resp, err := hub.client.Do(req) 215 | if err != nil { 216 | log.Errorf("Error getting hub info from %s: %v", hubURL, err) 217 | return 218 | } 219 | 220 | defer resp.Body.Close() 221 | if resp.StatusCode == http.StatusNotFound { 222 | log.Debugf("Not found") 223 | notfound = true 224 | return 225 | } 226 | 227 | if resp.StatusCode != http.StatusOK { 228 | log.Errorf("Failed to get namespace images %d: %s", resp.StatusCode, resp.Status) 229 | err = fmt.Errorf("Failed to get namespace images: %d %s", resp.StatusCode, resp.Status) 230 | return 231 | } 232 | 233 | body, err := ioutil.ReadAll(resp.Body) 234 | if err != nil { 235 | log.Errorf("Error reading namespace images response - %v", err) 236 | return 237 | } 238 | 239 | err = json.Unmarshal(body, &ir) 240 | if err != nil { 241 | log.Errorf("Error unmarshalling namespace images - %v", err) 242 | } 243 | 244 | images := make([]ImageInfo, len(ir.Results)) 245 | 246 | for i, info := range ir.Results { 247 | images[i] = ImageInfo{ 248 | ImageName: info.Namespace + "/" + info.Name, 249 | IsPrivate: info.IsPrivate, 250 | } 251 | } 252 | 253 | pageCount := ir.Count / constImagesPerPage 254 | if (ir.Count % constImagesPerPage) > 0 { 255 | pageCount += 1 256 | } 257 | 258 | imageList = ImageList{ 259 | CurrentPage: page, 260 | PageCount: pageCount, 261 | ImageCount: ir.Count, 262 | Images: images, 263 | } 264 | 265 | return 266 | } 267 | 268 | // Info gets information about this image 269 | func (hub *InfoService) Info(image registry.Image) (hubInfo Info, err error) { 270 | org, imageName, _ := utils.ParseDockerImage(image.Name) 271 | 272 | hubRepoURL := fmt.Sprintf("%s/v2/repositories/%s/%s/", hub.baseURL, org, imageName) 273 | 274 | log.Debugf("Getting hub info from %s", hubRepoURL) 275 | req, err := http.NewRequest("GET", hubRepoURL, nil) 276 | if err != nil { 277 | log.Errorf("Failed to build API GET request err %v", err) 278 | return 279 | } 280 | 281 | if image.User != "" && image.Password != "" { 282 | token, err := hub.Login(image.User, image.Password) 283 | if err != nil { 284 | log.Errorf("Failed to login to Docker Hub API %v", err) 285 | return hubInfo, err 286 | } 287 | 288 | req.Header.Add("Authorization", fmt.Sprintf("JWT %s", token)) 289 | } 290 | 291 | resp, err := hub.client.Do(req) 292 | if err != nil { 293 | log.Errorf("Error getting hub info from %s: %v", hubRepoURL, err) 294 | return 295 | } 296 | 297 | defer resp.Body.Close() 298 | if resp.StatusCode != http.StatusOK { 299 | log.Errorf("Error getting hub info %d: %s", resp.StatusCode, resp.Status) 300 | return 301 | } 302 | 303 | body, err := ioutil.ReadAll(resp.Body) 304 | // It's possible to get 200 OK but with json that represents nothing, rather than the hub info 305 | // We can check whether it's right by confirming that the name is correct 306 | if !strings.Contains(string(body), "name") && !strings.Contains(string(body), imageName) { 307 | err = fmt.Errorf("Failed to get hub info for %s/%s", org, imageName) 308 | return 309 | } 310 | 311 | err = json.Unmarshal(body, &hubInfo) 312 | if err != nil { 313 | log.Errorf("Error unmarshalling hub info for image %s/%s - %v", org, imageName, err) 314 | } 315 | 316 | log.Debugf("Got hub info %v", hubInfo) 317 | return 318 | } 319 | 320 | // getNamespaces returns namespaces where the user is an owner 321 | func (hub *InfoService) getNamespaces(loginToken string) (namespaces []string, err error) { 322 | var nr namespacesResponse 323 | 324 | hubURL := fmt.Sprintf("%s/v2/repositories/namespaces/", hub.baseURL) 325 | log.Debugf("Getting namespaces %s", hubURL) 326 | 327 | body, err := hub.getDockerHubData(hubURL, loginToken) 328 | if err != nil { 329 | log.Errorf("Error getting namespaces - %v", err) 330 | return 331 | } 332 | 333 | err = json.Unmarshal(body, &nr) 334 | if err != nil { 335 | log.Errorf("Error unmarshalling namespaces - %v", err) 336 | return 337 | } 338 | 339 | namespaces = nr.Namespaces 340 | log.Debugf("Got namespaces %v", namespaces) 341 | 342 | return 343 | } 344 | 345 | // getOrgs returns orgs where the user is a member 346 | func (hub *InfoService) getOrgs(loginToken string) (orgs []string, err error) { 347 | var or orgsResponse 348 | 349 | // Match large page size used by Docker Hub UI 350 | orgURL := fmt.Sprintf("%s/v2/user/orgs/?page_size=250", hub.baseURL) 351 | log.Debugf("Getting orgs %s", orgURL) 352 | 353 | orgBody, err := hub.getDockerHubData(orgURL, loginToken) 354 | if err != nil { 355 | log.Errorf("Error getting orgs - %v", err) 356 | return 357 | } 358 | 359 | err = json.Unmarshal(orgBody, &or) 360 | if err != nil { 361 | log.Errorf("Error unmarshalling orgs - %v", err) 362 | return 363 | } 364 | 365 | orgs = make([]string, len(or.Orgs)) 366 | 367 | for i, o := range or.Orgs { 368 | orgs[i] = o.OrgName 369 | } 370 | 371 | log.Debugf("Got orgs %v", orgs) 372 | 373 | return 374 | } 375 | 376 | func (hub *InfoService) getDockerHubData(hubURL string, loginToken string) (body []byte, err error) { 377 | req, _ := http.NewRequest("GET", hubURL, nil) 378 | req.Header.Add("Authorization", fmt.Sprintf("JWT %s", loginToken)) 379 | req.Header.Add("Content-Type", "application/json") 380 | 381 | resp, err := hub.client.Do(req) 382 | if err != nil { 383 | log.Errorf("Error getting hub data from %s: %v", hubURL, err) 384 | return 385 | } 386 | 387 | defer resp.Body.Close() 388 | if resp.StatusCode != http.StatusOK { 389 | log.Errorf("Failed to get %s %d: %s", hubURL, resp.StatusCode, resp.Status) 390 | return 391 | } 392 | 393 | body, err = ioutil.ReadAll(resp.Body) 394 | if err != nil { 395 | log.Errorf("Error reading hub response %s: %v", hubURL, err) 396 | } 397 | 398 | return 399 | } 400 | -------------------------------------------------------------------------------- /hub/hub_test.go: -------------------------------------------------------------------------------- 1 | package hub 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "net/url" 9 | "reflect" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/microscaling/microbadger/registry" 14 | ) 15 | 16 | func TestNewService(t *testing.T) { 17 | i := NewService() 18 | if i.baseURL != "https://hub.docker.com" { 19 | t.Errorf("Unexpected hub URL %s", i.baseURL) 20 | } 21 | } 22 | 23 | // Check that we can create a mock service and get some fake info from it 24 | func TestInfo(t *testing.T) { 25 | // Test server that always responds with 200 code and with a set payload 26 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 27 | w.WriteHeader(200) 28 | w.Header().Set("Content-Type", "application/json") 29 | fmt.Fprintln(w, `{"user": "lizrice", "name": "imagetest", "namespace": "lizrice", "status": 1, "description": "", "is_private": false, "is_automated": false, "can_edit": false, "star_count": 0, "pull_count": 17675, "last_updated": "2016-08-26T11:39:56.287301Z", "has_starred": false, "full_description": "blah-di-blah", "permissions": {"read": true, "write": false, "admin": false}}`) 30 | })) 31 | defer server.Close() 32 | 33 | // Make a transport that reroutes all traffic to the example server 34 | transport := &http.Transport{ 35 | Proxy: func(req *http.Request) (*url.URL, error) { 36 | return url.Parse(server.URL) 37 | }, 38 | } 39 | 40 | i := registry.Image{ 41 | Name: "org/image", 42 | } 43 | 44 | hs := NewMockService(transport) 45 | info, err := hs.Info(i) 46 | if err != nil { 47 | t.Errorf("Error getting fake hub info: %v", err) 48 | } 49 | t.Logf("Info %v", info) 50 | 51 | if info.PullCount != 17675 { 52 | t.Errorf("Unexpected pull count %d", info.PullCount) 53 | } 54 | 55 | if info.LastUpdated.Hour() != 11 { 56 | t.Errorf("Unexpected time %v", info.LastUpdated) 57 | } 58 | 59 | if info.FullDescription != "blah-di-blah" { 60 | t.Errorf("Unexpected full description %s", info.FullDescription) 61 | } 62 | } 63 | 64 | func TestLogin(t *testing.T) { 65 | transport, server := mockHub(t) 66 | defer server.Close() 67 | 68 | authToken := "abctokenabc" 69 | hs := NewMockService(transport) 70 | 71 | token, err := hs.Login("user", "password") 72 | if err != nil { 73 | t.Errorf("Unexpected error found logging in to Docker Hub %v", err) 74 | } 75 | 76 | if token != authToken { 77 | t.Errorf("Expected auth token to be %s but was %s", authToken, token) 78 | } 79 | 80 | token, err = hs.Login("user", "incorrect") 81 | if err == nil { 82 | t.Errorf("Expected an error logging in but found %v", err) 83 | } 84 | 85 | if token != "" { 86 | t.Errorf("Expected token to be blank but was %s", token) 87 | } 88 | } 89 | 90 | func TestUserNamespaces(t *testing.T) { 91 | transport, server := mockHub(t) 92 | defer server.Close() 93 | 94 | results := NamespaceList{ 95 | Namespaces: []string{"force12io", "microbadgertest", "microscaling"}, 96 | } 97 | 98 | hs := NewMockService(transport) 99 | namespaces, err := hs.UserNamespaces("user", "password") 100 | if err != nil { 101 | t.Errorf("Error getting fake user namespaces - %v", err) 102 | } 103 | 104 | if !reflect.DeepEqual(results, namespaces) { 105 | t.Errorf("Unexpected user namespaces was %v but expected %v", namespaces, results) 106 | } 107 | 108 | _, err = hs.UserNamespaces("user", "incorrect") 109 | if err == nil { 110 | t.Errorf("Expected an error getting namespaces but none was found") 111 | } 112 | } 113 | 114 | func TestUserNamespaceImages(t *testing.T) { 115 | // TODO!! 116 | // t.Errorf("Add tests for getting images in a namespace.") 117 | } 118 | 119 | func mockHub(t *testing.T) (transport *http.Transport, server *httptest.Server) { 120 | // Server that fakes responses from Docker Hub 121 | server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 122 | w.Header().Set("Content-Type", "application/json") 123 | // fmt.Printf("Request %s\n", r.URL.String()) 124 | switch r.URL.String() { 125 | case "http://fakehub/v2/users/login/": 126 | body, err := ioutil.ReadAll(r.Body) 127 | if err != nil { 128 | t.Fatalf("Couldn't read from test data") 129 | } 130 | 131 | if strings.Contains(string(body), "incorrect") { 132 | // fmt.Println("Password is, literally, incorrect") 133 | w.WriteHeader(401) 134 | fmt.Fprintln(w, `{"detail": "Incorrect authentication credentials."}`) 135 | } else { 136 | // fmt.Println("We say this password is OK") 137 | fmt.Fprintln(w, `{"token": "abctokenabc"}`) 138 | } 139 | case "http://fakehub/v2/repositories/namespaces/": 140 | fmt.Fprintf(w, `{"namespaces":["microbadgertest","microscaling"]}`) 141 | 142 | case "http://fakehub/v2/user/orgs/?page_size=250": // Match large page size used by Docker Hub UI 143 | fmt.Fprintf(w, `{"count": 1, "results": [{"orgname": "microscaling"},{"orgname": "force12io"}]}`) 144 | 145 | default: 146 | t.Errorf("Unexpected request to %s", r.URL.String()) 147 | } 148 | })) 149 | 150 | // Make a transport that reroutes all traffic to the test server 151 | transport = &http.Transport{ 152 | Proxy: func(req *http.Request) (*url.URL, error) { 153 | return url.Parse(server.URL) 154 | }, 155 | } 156 | 157 | return 158 | } 159 | -------------------------------------------------------------------------------- /inspector/labels.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "net/url" 7 | "path" 8 | "strings" 9 | 10 | "github.com/microscaling/microbadger/database" 11 | ) 12 | 13 | const ( 14 | constLicenseCode = "org.label-schema.license" 15 | constVersionControlType = "org.label-schema.vcs-type" 16 | constVersionControlURL = "org.label-schema.vcs-url" 17 | constVersionControlRef = "org.label-schema.vcs-ref" 18 | constGitHubSSH = "git@github.com:" 19 | constGitHubHTTPS = "https://github.com/" 20 | constLicenseFile = "inspector/licenses.json" 21 | ) 22 | 23 | var licenseCodeAltLabels = []string{ 24 | "license", 25 | } 26 | 27 | var vcsTypeAltLabels = []string{ 28 | "vcs-type", 29 | } 30 | 31 | var vcsRefAltLabels = []string{ 32 | "vcs-ref", 33 | } 34 | 35 | var vcsUrlAltLabels = []string{ 36 | "vcs-url", 37 | } 38 | 39 | var ( 40 | licenseCodes map[string]string 41 | ) 42 | 43 | func init() { 44 | // Load the license data 45 | licenses, err := getLicenses() 46 | if err != nil { 47 | log.Errorf("Error getting license list - %v", err) 48 | return 49 | } 50 | 51 | licenseCodes = make(map[string]string) 52 | 53 | for _, license := range licenses { 54 | licenseCodes[strings.ToLower(license.Code)] = license.URL 55 | } 56 | 57 | log.Debugf("License data initialized with %d licenses", len(licenseCodes)) 58 | } 59 | 60 | // ParseLabels inspects Docker labels for those matching the label-schema.org schema. 61 | // TODO Retire badgeCount as its no longer needed. 62 | func ParseLabels(iv *database.ImageVersion) (badgeCount int, license *database.License, vcs *database.VersionControl) { 63 | var labels map[string]string 64 | 65 | err := json.Unmarshal([]byte(iv.Labels), &labels) 66 | if err != nil { 67 | log.Errorf("Error unmarshalling labels %s: %v", iv.Labels, err) 68 | return 0, nil, nil 69 | } 70 | 71 | license = parseLicense(labels) 72 | if license != nil { 73 | badgeCount += 1 74 | } 75 | 76 | vcs = parseVersionControl(labels) 77 | if vcs != nil { 78 | badgeCount += 1 79 | } 80 | 81 | return badgeCount, license, vcs 82 | } 83 | 84 | func parseVersionControl(labels map[string]string) *database.VersionControl { 85 | 86 | vcs := &database.VersionControl{ 87 | Type: getLabel(labels, constVersionControlType, vcsTypeAltLabels), 88 | URL: getLabel(labels, constVersionControlURL, vcsUrlAltLabels), 89 | Commit: getLabel(labels, constVersionControlRef, vcsRefAltLabels), 90 | } 91 | 92 | if vcs.Type == "" || strings.ToLower(vcs.Type) == "git" { 93 | // Support for GitHub URLs 94 | if vcs.Commit != "" && strings.Contains(vcs.URL, "github.com") { 95 | // TODO Add support for BitBucket etc. 96 | return parseGitHubLabels(vcs) 97 | } 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func parseLicense(labels map[string]string) *database.License { 104 | code := getLabel(labels, constLicenseCode, licenseCodeAltLabels) 105 | if code != "" { 106 | license := &database.License{ 107 | Code: code, 108 | URL: licenseCodes[strings.ToLower(code)], 109 | } 110 | 111 | return license 112 | } 113 | 114 | return nil 115 | } 116 | 117 | func parseGitHubLabels(vcs *database.VersionControl) *database.VersionControl { 118 | // Set type to Git 119 | vcs.Type = "git" 120 | 121 | // Convert from SSH to HTTPS URL 122 | vcs.URL = strings.Replace(vcs.URL, constGitHubSSH, constGitHubHTTPS, 1) 123 | 124 | // Remove .git suffix if present 125 | vcs.URL = strings.TrimSuffix(vcs.URL, ".git") 126 | 127 | commitURL, err := url.Parse(vcs.URL) 128 | if err != nil { 129 | log.Errorf("Error parsing GitHub URL - %v", err) 130 | return nil 131 | } 132 | 133 | // Link to exact commit 134 | commitURL.Path = path.Join(commitURL.Path, "tree", vcs.Commit) 135 | vcs.URL = commitURL.String() 136 | 137 | return vcs 138 | } 139 | 140 | func getLabel(labels map[string]string, main string, alternatives []string) string { 141 | 142 | value, ok := labels[main] 143 | if !ok { 144 | // Check for alternative versions 145 | for _, alternative := range alternatives { 146 | for key, value := range labels { 147 | if strings.Contains(key, alternative) { 148 | log.Debugf("Found alternative label format %s", alternative) 149 | return value 150 | } 151 | } 152 | } 153 | } 154 | 155 | return value 156 | } 157 | 158 | func getLicenses() (licenses []*database.License, err error) { 159 | raw, err := ioutil.ReadFile(constLicenseFile) 160 | if err != nil { 161 | log.Errorf("Error reading licenses.json - %v", err) 162 | return nil, err 163 | } 164 | 165 | err = json.Unmarshal(raw, &licenses) 166 | if err != nil { 167 | log.Errorf("Error unmarshalling licenses.json - %v", err) 168 | return nil, err 169 | } 170 | 171 | return 172 | } 173 | -------------------------------------------------------------------------------- /inspector/labels_test.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "testing" 7 | 8 | "github.com/microscaling/microbadger/database" 9 | ) 10 | 11 | func TestGetHashFromLayers(t *testing.T) { 12 | layers := []database.ImageLayer{ 13 | {BlobSum: "10000", Command: "abcdefabcdefabcdefabcdef", DownloadSize: 345}, 14 | {BlobSum: "20000", Command: "aaaaaaaaaaaaaaaaaaaaaaaa", DownloadSize: 345}, 15 | {BlobSum: "30000", Command: "xx", DownloadSize: 345}, 16 | } 17 | 18 | checksum := sha256.Sum256([]byte("abcdefabcdefabcdefabcdef" + "aaaaaaaaaaaaaaaaaaaaaaaa" + "xx")) 19 | expectedResult := hex.EncodeToString(checksum[:]) 20 | 21 | result := GetHashFromLayers(layers) 22 | if result != expectedResult { 23 | t.Errorf("Unexpected hash result") 24 | } 25 | } 26 | 27 | func TestParseGithubLabels(t *testing.T) { 28 | var vcs *database.VersionControl 29 | var tests []string 30 | 31 | httpsString := "https://github.com/microscaling/microscaling.git" 32 | sshString := "git@github.com:microscaling/microscaling.git" 33 | result := "https://github.com/microscaling/microscaling/tree/12345" 34 | 35 | tests = []string{ 36 | httpsString, sshString, 37 | } 38 | 39 | for _, test := range tests { 40 | vcs = &database.VersionControl{} 41 | vcs.URL = test 42 | vcs.Commit = "12345" 43 | vcs = parseGitHubLabels(vcs) 44 | if vcs.Type != "git" { 45 | t.Fatalf("Wrong VCS type: %s", vcs.Type) 46 | } 47 | 48 | if vcs.URL != result { 49 | t.Fatalf("Wrong URL %s", vcs.URL) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /inspector/main.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "time" 8 | 9 | logging "github.com/op/go-logging" 10 | 11 | "github.com/microscaling/microbadger/database" 12 | "github.com/microscaling/microbadger/encryption" 13 | "github.com/microscaling/microbadger/hub" 14 | "github.com/microscaling/microbadger/queue" 15 | "github.com/microscaling/microbadger/registry" 16 | "github.com/microscaling/microbadger/utils" 17 | ) 18 | 19 | var ( 20 | webhookURL string 21 | log = logging.MustGetLogger("mminspect") 22 | ) 23 | 24 | func init() { 25 | webhookURL = os.Getenv("MB_WEBHOOK_URL") 26 | } 27 | 28 | // CheckImageExists checks if an image exists on DockerHub for this image. 29 | // Uses V2 of the Docker Registry API. If it's a private image this is only going to 30 | // work if you pass in the registry credentials as part of the image () 31 | func CheckImageExists(i registry.Image, rs *registry.Service) bool { 32 | log.Debugf("Checking image %s exists.", i.Name) 33 | 34 | t, err := registry.NewTokenAuth(i, rs) 35 | if err != nil { 36 | log.Debugf("Couldn't get auth for %s", i.Name) 37 | return false 38 | } 39 | 40 | // Getting the list of tags will fail if the image doesn't exist 41 | _, err = t.GetTags() 42 | if err != nil { 43 | err = fmt.Errorf("Error getting tags for %s: %v", i.Name, err) 44 | return false 45 | } 46 | 47 | return true 48 | } 49 | 50 | // Inspect creates a new database image record and populates it with data from the registry 51 | func Inspect(imageName string, db *database.PgDB, rs *registry.Service, hs *hub.InfoService, qs queue.Service, es encryption.Service) (err error) { 52 | log.Debugf("Inspecting %s", imageName) 53 | 54 | var hasChanged bool 55 | image := registry.Image{Name: imageName} 56 | 57 | img, err := db.GetOrCreateImage(imageName) 58 | if err != nil { 59 | log.Errorf("GetImage error for %s - %v", imageName, err) 60 | return err 61 | } 62 | 63 | // Check if there are saved credentials for this image. 64 | // If this errors try anyway as the image may be public 65 | image, _ = getRegistryCredentials(imageName, db, es) 66 | 67 | // Get the information from the hub first 68 | hubInfo, err := hs.Info(image) 69 | if err != nil { 70 | // We still want to carry on in this case as we may be able to get registry info anyway 71 | log.Errorf("Failed to get hub info for %s", imageName) 72 | } 73 | 74 | // Update Docker Hub metadata and stop if the image hasn't changed 75 | hasChanged, img = setHubInfo(img, hubInfo) 76 | if !hasChanged { 77 | return nil 78 | } 79 | 80 | versions, err := getVersionsFromRegistry(image, rs) 81 | if err != nil { 82 | log.Errorf("Failed to get metadata using registry: %v", err) 83 | 84 | if strings.Contains(err.Error(), "401 Unauthorized") { 85 | img.Status = "MISSING" 86 | } else { 87 | img.Status = "FAILED_INSPECTION" 88 | } 89 | img.BadgeCount = 0 90 | 91 | // Save image status so its no longer displayed if it has been deleted or made private. 92 | err := db.PutImageOnly(img) 93 | if err != nil { 94 | log.Errorf("Failed to save image %v", err) 95 | } 96 | 97 | return err 98 | } 99 | 100 | log.Debugf("%d versions found for %s", len(versions), img.Name) 101 | img.Versions = make([]database.ImageVersion, len(versions)) 102 | i := 0 103 | for _, v := range versions { 104 | img.Versions[i] = v 105 | i++ 106 | } 107 | 108 | img.Status = "SIZE" 109 | updateLatest(&img) 110 | img.BadgeCount++ // One badge for the link to Docker Hub 111 | 112 | if img.AuthToken == "" { 113 | token, err := utils.GenerateAuthToken() 114 | if err == nil { 115 | img.AuthToken = token 116 | } else { 117 | log.Errorf("Error generating auth token for %s: %v", img.Name, err) 118 | } 119 | } 120 | 121 | // Generate webhook URL including the auth token. 122 | img.WebhookURL = fmt.Sprintf("%s/images/%s/%s", webhookURL, img.Name, img.AuthToken) 123 | 124 | // Save image to the database 125 | nmc, err := db.PutImage(img) 126 | if err != nil { 127 | log.Errorf("Couldn't put image: %v", err) 128 | } 129 | 130 | // If anything has changed we do extra processing. 131 | if len(nmc.NewTags) > 0 || len(nmc.ChangedTags) > 0 || len(nmc.DeletedTags) > 0 { 132 | // We may have some users to notify. 133 | buildNotifications(db, qs, img.Name, nmc) 134 | } else { 135 | // Since we're not going to do a size inspection we can skip straight to INSPECTED state 136 | img.Status = "INSPECTED" 137 | err = db.PutImageOnly(img) 138 | if err != nil { 139 | log.Errorf("Couldn't put image (only): %v", err) 140 | } 141 | } 142 | 143 | log.Infof("Name: %s", img.Name) 144 | log.Infof("Status: %s", img.Status) 145 | log.Infof("Badges installed: %d", img.BadgesInstalled) 146 | log.Infof("Latest SHA: %s", img.Latest) 147 | 148 | return 149 | } 150 | 151 | // Checks if the image has stored credentials and returns the first users. 152 | // TODO Credentials can be revoked so on failure we should try the next user 153 | func getRegistryCredentials(image string, db *database.PgDB, es encryption.Service) (registry.Image, error) { 154 | img := registry.Image{Name: image} 155 | 156 | rcl, err := db.GetRegistryCredentialsForImage(image) 157 | if err != nil { 158 | log.Errorf("Error getting registry creds for image %s - %v", image, err) 159 | return img, err 160 | } 161 | 162 | if len(rcl) >= 1 { 163 | rc := rcl[0] 164 | 165 | password, err := es.Decrypt(rc.EncryptedKey, rc.EncryptedPassword) 166 | if err != nil { 167 | log.Errorf("Error decrypting password - %v", err) 168 | return img, err 169 | } 170 | 171 | img.User = rc.User 172 | img.Password = password 173 | } 174 | 175 | return img, err 176 | } 177 | 178 | // For public images calls the Docker Hub API to check if the image has changed. 179 | // Also sets the latest Docker Hub metadata. 180 | func setHubInfo(img database.Image, hubInfo hub.Info) (bool, database.Image) { 181 | var lastUpdated time.Time 182 | 183 | // Handle null last updated dates in the API response. 184 | if hubInfo.LastUpdated != nil { 185 | lastUpdated = *hubInfo.LastUpdated 186 | } 187 | 188 | // We can stop if the image hasn't changed since we last looked 189 | if (img.Status == "INSPECTED") && lastUpdated.Equal(img.LastUpdated) { 190 | log.Infof("Image %s is unchanged since we last looked at %#v", img.Name, lastUpdated) 191 | return false, img 192 | } 193 | 194 | if lastUpdated.Before(img.LastUpdated) { 195 | // We'll update the info anyway 196 | log.Errorf("Image %s LastUpdated is older than our own record!", img.Name) 197 | } 198 | 199 | img.LastUpdated = lastUpdated 200 | img.IsPrivate = hubInfo.IsPrivate 201 | img.IsAutomated = hubInfo.IsAutomated 202 | img.Description = hubInfo.Description 203 | img.PullCount = hubInfo.PullCount 204 | img.StarCount = hubInfo.StarCount 205 | img.BadgesInstalled = utils.BadgesInstalled(hubInfo.FullDescription) 206 | 207 | return true, img 208 | } 209 | -------------------------------------------------------------------------------- /inspector/main_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequire 2 | 3 | package inspector 4 | 5 | import ( 6 | "crypto/sha256" 7 | "encoding/hex" 8 | "testing" 9 | 10 | "github.com/microscaling/microbadger/database" 11 | ) 12 | 13 | // Setting up test database for this package 14 | // $ psql -c 'create database microbadger_inspector_test;' -U postgres 15 | 16 | func getDatabase(t *testing.T) database.PgDB { 17 | testdb, err := database.GetPostgres("localhost", "postgres", "microbadger_inspector_test", "", true) 18 | if err != nil { 19 | t.Fatalf("Failed to open test database: %v", err) 20 | } 21 | 22 | return testdb 23 | } 24 | 25 | func emptyDatabase(db database.PgDB) { 26 | db.Exec("DELETE FROM tags") 27 | db.Exec("DELETE FROM image_versions") 28 | db.Exec("DELETE FROM images") 29 | } 30 | 31 | func TestInspect(t *testing.T) { 32 | var err error 33 | 34 | db := getDatabase(t) 35 | emptyDatabase(db) 36 | 37 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 38 | w.WriteHeader(200) 39 | w.Header().Set("Content-Type", "application/json") 40 | t.Logf("Request %s", r.URL.String()) 41 | switch r.URL.String() { 42 | case "http://fakehub/v2/repositories/lizrice/imagetest/": 43 | fmt.Fprintln(w, `{"user": "lizrice", "name": "imagetest", "namespace": "lizrice", "status": 1, "description": "", "is_private": false, "is_automated": false, "can_edit": false, "star_count": 0, "pull_count": 17675, "last_updated": "2016-08-26T11:39:56.287301Z", "has_starred": false, "full_description": "blah-di-blah", "permissions": {"read": true, "write": false, "admin": false}}`) 44 | case "http://fakeauth/token?service=fakeservice&scope=repository:lizrice/imagetest:pull": 45 | fmt.Fprintln(w, `{"token": "abctokenabc"}`) 46 | case "http://fakereg/v2/lizrice/imagetest/tags/list": 47 | fmt.Fprintln(w, `{"name": "lizrice/imagetest", "tags": ["tag1", "tag2"]}`) 48 | case "http://fakereg/v2/lizrice/imagetest/manifests/tag1", "http://fakereg/v2/lizrice/imagetest/manifests/tag2": 49 | fmt.Fprintln(w, `{"schemaVersion": 1,"name": "lizrice/imagetest","tag": "tag1","architecture": "amd64", 50 | "fsLayers": [{"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"}, {"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"}, {"blobSum": "sha256:6c123565ed5e79b6c944d6da64bd785ad3ec03c6e853dcb733254aebb215ae55"}], 51 | "history": [{"v1Compatibility": "{\"architecture\":\"amd64\",\"author\":\"liz@lizrice.com\",\"config\":{\"Hostname\":\"ae2e58a6294e\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":null,\"Image\":\"sha256:af06c2834e821a5900985ae8f64f4c73de9d71a8b08b29019d346adb7f176e9c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":[],\"Labels\":{\"com.lizrice.test\":\"olé\",\"org.label-schema.vcs-ref\":\"2345678\",\"org.label-schema.vcs-url\":\"https://github.com/lizrice/imagetest\"}},\"container\":\"764bfb743437e31ebe8c909020bc9a0c738ed84cbecdc944876226990d71655c\",\"container_config\":{\"Hostname\":\"ae2e58a6294e\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) \",\"LABEL com.lizrice.test=olé org.label-schema.vcs-url=https://github.com/lizrice/imagetest org.label-schema.vcs-ref=2345678\"],\"Image\":\"sha256:af06c2834e821a5900985ae8f64f4c73de9d71a8b08b29019d346adb7f176e9c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":[],\"Labels\":{\"com.lizrice.test\":\"olé\",\"org.label-schema.vcs-ref\":\"2345678\",\"org.label-schema.vcs-url\":\"https://github.com/lizrice/imagetest\"}},\"created\":\"2016-08-26T11:39:30.603530099Z\",\"docker_version\":\"1.12.1-rc1\",\"id\":\"7d82ed3f20e8ad8f7515a30cbd6070555d59f81c2fdd9950a5799ef31149c48b\",\"os\":\"linux\",\"parent\":\"c8fd7d51bc1273614a21e4a7f7590331df7102527adfe0dcac7643548f9bf7cf\",\"throwaway\":true}"}, 52 | {"v1Compatibility": "{\"id\":\"c8fd7d51bc1273614a21e4a7f7590331df7102527adfe0dcac7643548f9bf7cf\",\"parent\":\"415049c7b80053ff6d962bef6d08058abe309c816319b97787e4a212a5d333d0\",\"created\":\"2016-06-30T13:36:14.910427799Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) MAINTAINER liz@lizrice.com\"]},\"author\":\"liz@lizrice.com\",\"throwaway\":true}"}]}`) 53 | default: 54 | t.Errorf("Unexpected request to %s", r.URL.String()) 55 | } 56 | })) 57 | defer server.Close() 58 | 59 | // Make a transport that reroutes all traffic to the example server 60 | transport := &http.Transport{ 61 | Proxy: func(req *http.Request) (*url.URL, error) { 62 | return url.Parse(server.URL) 63 | }, 64 | } 65 | 66 | hs := hub.NewMockService(transport) 67 | rs := registry.NewMockService(transport, "http://fakeauth", "http://fakereg", "fakeservice") 68 | qs := queue.NewMockService() 69 | es := encryption.NewMockService() 70 | err = Inspect("lizrice/imagetest", &db, &rs, &hs, qs, es) 71 | if err != nil { 72 | t.Fatalf("Error %v", err) 73 | } 74 | 75 | // TODO!! Test that this sets up the database as expected 76 | // TODO!! Test some different cases with different tags, image versions etc 77 | } 78 | -------------------------------------------------------------------------------- /inspector/notifications.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/microscaling/microbadger/database" 7 | "github.com/microscaling/microbadger/queue" 8 | ) 9 | 10 | // buildNotifications sends an SQS message to the notifier for each user who needs to be notified about this change 11 | func buildNotifications(db *database.PgDB, qs queue.Service, imageName string, nmc database.NotificationMessageChanges) { 12 | var nm database.NotificationMessage 13 | 14 | if len(nmc.NewTags) == 0 && len(nmc.ChangedTags) == 0 && len(nmc.DeletedTags) == 0 { 15 | return 16 | } 17 | 18 | log.Debugf("%s has changes that may need to generate notifications", imageName) 19 | notifications, err := db.GetNotificationsForImage(imageName) 20 | if err != nil { 21 | log.Errorf("Failed to generate notifications for %s", imageName) 22 | return 23 | } 24 | 25 | nmcAsJson, err := json.Marshal(nmc) 26 | if err != nil { 27 | log.Errorf("Failed to generate NMC message: %v", err) 28 | return 29 | } 30 | 31 | log.Infof("Generating %d notifications for image %s", len(notifications), imageName) 32 | 33 | // TODO!! We could consider having one SQS message per image, and have the notifier generate all the webhooks 34 | // We'll need to send a notification message for all the notifications for this image 35 | for _, n := range notifications { 36 | // Save an unsent notification message 37 | nm = database.NotificationMessage{ 38 | NotificationID: n.ID, 39 | ImageName: n.ImageName, 40 | WebhookURL: n.WebhookURL, 41 | Message: database.PostgresJSON{nmcAsJson}, 42 | } 43 | 44 | err := db.SaveNotificationMessage(&nm) 45 | if err != nil { 46 | log.Errorf("Failed to create notification message for %s, id %d: %v", imageName, n.ID, err) 47 | return 48 | } 49 | 50 | err = qs.SendNotification(nm.ID) 51 | if err != nil { 52 | log.Errorf("Failed to send notification message for %s, id %d: %v", imageName, n.ID, err) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /inspector/notifications_test.go: -------------------------------------------------------------------------------- 1 | // +build dbrequire 2 | 3 | package inspector 4 | 5 | import ( 6 | "encoding/json" 7 | "testing" 8 | 9 | "github.com/microscaling/microbadger/database" 10 | ) 11 | 12 | func addNotificationThings(db database.PgDB) { 13 | 14 | } 15 | 16 | func TestBuildNotifications(t *testing.T) { 17 | db := getDatabase(t) 18 | emptyDatabase(db) 19 | addNotificationThings(db) 20 | 21 | } 22 | 23 | func TestNotificationMessageChanges(t *testing.T) { 24 | nmc := database.NotificationMessageChanges{ 25 | NewTags: []database.Tag{{ 26 | Tag: "Tag1", SHA: "10000", ImageName: "image/name"}}, 27 | } 28 | 29 | msg, err := json.Marshal(nmc) 30 | if err != nil { 31 | t.Fatalf("Error with NMC %v", err) 32 | } 33 | 34 | log.Infof("%s", msg) 35 | } 36 | -------------------------------------------------------------------------------- /inspector/registry.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "strings" 9 | "time" 10 | 11 | "github.com/microscaling/microbadger/database" 12 | "github.com/microscaling/microbadger/registry" 13 | ) 14 | 15 | func getVersionsFromRegistry(i registry.Image, rs *registry.Service) (versions map[string]database.ImageVersion, err error) { 16 | versions = make(map[string]database.ImageVersion, 1) 17 | 18 | t, err := registry.NewTokenAuth(i, rs) 19 | if err != nil { 20 | err = fmt.Errorf("Error getting Token Auth Client for %s: %v", i.Name, err) 21 | return 22 | } 23 | 24 | // Get all the tags 25 | tags, err := t.GetTags() 26 | if err != nil { 27 | err = fmt.Errorf("Error getting tags for %s: %v", i.Name, err) 28 | return 29 | } 30 | 31 | if len(tags) == 0 { 32 | err = fmt.Errorf("No tags exist for %s", i.Name) 33 | return 34 | } 35 | 36 | // Get the version info for all the tags, looking out for any that match latest 37 | log.Debugf("%d tags for %s", len(tags), i.Name) 38 | for _, tagName := range tags { 39 | 40 | manifest, manifestBytes, err := t.GetManifest(tagName) 41 | if err != nil { 42 | // Sometimes the registry returns not found for a tag eevn though that tag was included 43 | // in the list of tags - presumably this is the registry in a bit of a bad state. 44 | // We don't want to fail the whole image in this situation. 45 | err = fmt.Errorf("Error getting manifest for tag %s: %v", tagName, err) 46 | if strings.Contains(err.Error(), "404") { 47 | continue 48 | } else { 49 | return versions, err 50 | } 51 | } 52 | 53 | version, err := getVersionInfo(manifest) 54 | if err != nil { 55 | log.Errorf("Error getting version info for %s tag %s: %v", i.Name, tagName, err) 56 | return versions, err 57 | } 58 | 59 | // We might already know about this version under a different tag name 60 | if v, ok := versions[version.SHA]; ok { 61 | version = v 62 | } 63 | 64 | tag := database.Tag{ 65 | Tag: tagName, 66 | ImageName: i.Name, 67 | SHA: version.SHA, 68 | } 69 | 70 | version.Tags = append(version.Tags, tag) 71 | version.Manifest = string(manifestBytes) 72 | versions[version.SHA] = version 73 | } 74 | 75 | return 76 | } 77 | 78 | func getVersionInfo(m registry.Manifest) (version database.ImageVersion, err error) { 79 | 80 | var v1c registry.V1Compatibility 81 | 82 | if len(m.History) == 0 { 83 | err = fmt.Errorf("No history for this image") 84 | return 85 | } 86 | 87 | // The first entry in the list is the one to look at 88 | h := m.History[0] 89 | err = json.Unmarshal([]byte(h.V1Compatibility), &v1c) 90 | if err != nil { 91 | err = fmt.Errorf("Error unmarshalling history: %v", err) 92 | return 93 | } 94 | 95 | created, err := time.Parse(time.RFC3339Nano, v1c.Created) 96 | if err != nil { 97 | log.Infof("Couldn't get created time for %s from string %s", m.Name, v1c.Created) 98 | } 99 | 100 | version = database.ImageVersion{ 101 | SHA: v1c.Id, 102 | ImageName: m.Name, 103 | Author: v1c.Author, 104 | Labels: string(v1c.Config.Labels), 105 | Created: created, 106 | LayerCount: len(m.FsLayers), 107 | } 108 | 109 | return 110 | } 111 | 112 | // formatHistory takes the raw command from the history in the manifest 113 | // and makes it human readable. 114 | func formatHistory(input []string) (cmd string) { 115 | 116 | // Get the raw command from the input array. 117 | cmd = strings.Join(input, " ") 118 | 119 | // Trim spaces just in case 120 | cmd = strings.TrimSpace(cmd) 121 | 122 | // Strip out the shell script prefix and strip whitespace again 123 | cmd = strings.TrimPrefix(cmd, "/bin/sh -c") 124 | cmd = strings.TrimSpace(cmd) 125 | 126 | // If the next word is #(nop) then the following word should be the Dockerfile directive 127 | if strings.HasPrefix(cmd, "#(nop)") { 128 | cmd = strings.TrimSpace(strings.TrimPrefix(cmd, "#(nop)")) 129 | 130 | // The exception is COPY where it's preceded by %s %s in %s 131 | cmd = strings.TrimPrefix(cmd, `%s %s in %s`) 132 | } else if cmd != "" { 133 | // If not, then it's a RUN 134 | cmd = "RUN " + strings.TrimSpace(cmd) 135 | } 136 | 137 | log.Debugf("Cmd is %s", cmd) 138 | return 139 | } 140 | 141 | func updateLatest(img *database.Image) { 142 | // See if there's a tag marked 'latest'; if not, find the most recent image 143 | var mostRecent time.Time 144 | 145 | // As we haven't stored into the database yet we need to go through all the versions 146 | for _, v := range img.Versions { 147 | for _, t := range v.Tags { 148 | if t.Tag == "latest" { 149 | img.Latest = t.SHA 150 | log.Debugf("Found version called latest for %s", img.Name) 151 | return 152 | } 153 | } 154 | 155 | if v.Created.After(mostRecent) { 156 | mostRecent = v.Created 157 | img.Latest = v.SHA 158 | } 159 | } 160 | 161 | log.Debugf("Latest version for %s is %s", img.Name, img.Latest) 162 | } 163 | 164 | // GetHashFromLayers hashes together all the layer commands 165 | func GetHashFromLayers(layers []database.ImageLayer) string { 166 | // log.Debugf("Making hash from %d layers", len(layers)) 167 | // This size is an approximation - we don't know how big the commands will really be, but it will do 168 | layerData := make([]byte, 0, len(layers)*sha256.Size) 169 | 170 | for _, layer := range layers { 171 | // Run all the cmd fields together to get our hashable string - or use the blobSum if there is no Command 172 | if layer.Command == "" { 173 | layerData = append(layerData, []byte(layer.BlobSum)...) 174 | } else { 175 | layerData = append(layerData, []byte(layer.Command)...) 176 | } 177 | } 178 | 179 | // log.Debugf("Layer data is %d bytes long", len(layerData)) 180 | checksum := sha256.Sum256(layerData) 181 | return hex.EncodeToString(checksum[:]) 182 | } 183 | -------------------------------------------------------------------------------- /inspector/size.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/microscaling/microbadger/database" 9 | "github.com/microscaling/microbadger/encryption" 10 | "github.com/microscaling/microbadger/registry" 11 | ) 12 | 13 | // InspectSize inspects the size of an image 14 | func InspectSize(imgName string, db *database.PgDB, rs *registry.Service, es encryption.Service) (err error) { 15 | log.Debugf("Inspecting size of %s", imgName) 16 | var m registry.Manifest 17 | 18 | img, err := db.GetImage(imgName) 19 | if err != nil || img.Status == "MISSING" { 20 | // We make the error nil so that we don't put this back on the queue for reinspection. 21 | // That means we can stop doing size inspection for a wrong'un simply by deleting the image from the DB. 22 | log.Infof("Image %s no longer available: %v", imgName, err) 23 | err = nil 24 | return 25 | } 26 | 27 | // We already have the manifests for each version 28 | versions, err := db.GetAllImageVersionsWithManifests(img) 29 | if err != nil { 30 | // Stay in SIZE state because we will want to reinspect it 31 | log.Errorf("Failed to get all versions for %s: %v", imgName, err) 32 | return err 33 | } 34 | 35 | // Check if there are saved credentials for this image. 36 | // If this errors try anyway as the image may be public 37 | image, _ := getRegistryCredentials(imgName, db, es) 38 | 39 | t, err := registry.NewTokenAuth(image, rs) 40 | if err != nil { 41 | log.Errorf("Error getting Token Auth Client for %s: %v", imgName, err) 42 | return err 43 | } 44 | 45 | log.Debugf("Image is %v and has %d versions", img, len(versions)) 46 | for _, iv := range versions { 47 | // No need to get the image size again if we already have it stored in a database 48 | if db.ImageVersionNeedsSizeOrLayers(&iv) { 49 | 50 | // Manifest is stored as a string in the database 51 | err = json.Unmarshal([]byte(iv.Manifest), &m) 52 | if err != nil { 53 | err = fmt.Errorf("Error unmarshalling manifest for %s: %v", img.Name, err) 54 | return err 55 | } 56 | 57 | // Get the total download size and the sizes of each individual layer. 58 | downloadSize, layerSizes, err := t.GetImageDownloadSize(m) 59 | if err != nil { 60 | log.Infof("Couldn't get download size for %s: %v", img.Name, err) 61 | if strings.Contains(err.Error(), "Rate limited") { 62 | // No point immediately trying to get other versions if we're rate limited 63 | break 64 | } 65 | // Other errors could be specific to this particular version, so we may still be able to 66 | // get information about other versions. 67 | continue 68 | } 69 | 70 | iv.DownloadSize = downloadSize 71 | 72 | // Check we have the layer sizes and that they match the history length. 73 | // Otherwise wait until the next time the size data is inspected. 74 | if layerSizes != nil && len(layerSizes) == len(m.History) { 75 | err = getLayerInfoFromManifest(&iv, m, layerSizes) 76 | if err != nil { 77 | err = fmt.Errorf("Error getting layer info from manifest for %s: %v", img.Name, err) 78 | return err 79 | } 80 | } 81 | 82 | // We have got all the information we could need from this manifest string, so we can 83 | // delete it and free up some space in the database 84 | iv.Manifest = "" 85 | 86 | log.Debugf("Updating image %s version %s with size %d", iv.ImageName, iv.SHA, iv.DownloadSize) 87 | db.PutImageVersion(iv) 88 | } 89 | } 90 | 91 | // Don't move this image to inspected if we got rate-limited. It should stay in SIZE state so we'll 92 | // try again another time. 93 | if err == nil { 94 | img.Status = "INSPECTED" 95 | err = db.PutImageOnly(img) 96 | } 97 | 98 | return err 99 | } 100 | 101 | func getLayerInfoFromManifest(v *database.ImageVersion, m registry.Manifest, layerSizes []int64) (err error) { 102 | var v1c registry.V1Compatibility 103 | layers := make([]database.ImageLayer, len(m.History)) 104 | 105 | // Process the history commands stored in the manifest. 106 | for i, h := range m.History { 107 | 108 | // Data is in the V1 compatibility section. 109 | err = json.Unmarshal([]byte(h.V1Compatibility), &v1c) 110 | if err != nil { 111 | err = fmt.Errorf("Error unmarshalling history: %v", err) 112 | return err 113 | } 114 | 115 | // Format the raw command for display. 116 | cmd := formatHistory(v1c.ContainerConfig.Cmd) 117 | 118 | // Add the command and the corresponding layer size. 119 | layers[i] = database.ImageLayer{ 120 | BlobSum: m.FsLayers[i].BlobSum, 121 | Command: cmd, 122 | DownloadSize: layerSizes[i], 123 | } 124 | } 125 | 126 | // Reverse the layers to make them human readable. 127 | for i, j := 0, len(layers)-1; i < j; i, j = i+1, j-1 { 128 | layers[i], layers[j] = layers[j], layers[i] 129 | } 130 | 131 | // Hash the layers together to identify this image 132 | v.Hash = GetHashFromLayers(layers) 133 | log.Debugf("Got hash %s", v.Hash) 134 | 135 | // Serialize the layers as JSON for storage. 136 | layersJSON, err := json.Marshal(layers) 137 | if err != nil { 138 | log.Infof("Couldn't convert layers for %s to string: %v", m.Name, err) 139 | } 140 | 141 | v.Layers = string(layersJSON) 142 | 143 | return err 144 | } 145 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/op/go-logging" 7 | 8 | "github.com/microscaling/microbadger/api" 9 | "github.com/microscaling/microbadger/database" 10 | "github.com/microscaling/microbadger/encryption" 11 | "github.com/microscaling/microbadger/hub" 12 | "github.com/microscaling/microbadger/inspector" 13 | "github.com/microscaling/microbadger/queue" 14 | "github.com/microscaling/microbadger/registry" 15 | "github.com/microscaling/microbadger/utils" 16 | ) 17 | 18 | var ( 19 | log = logging.MustGetLogger("microbadger") 20 | ) 21 | 22 | const constPollQueueTimeout = 250 // milliseconds - how often to check the queue for images. 23 | 24 | func init() { 25 | utils.InitLogging() 26 | } 27 | 28 | // For this prototype either inspects the image and saves the metadata or shows 29 | // the stored metadata. 30 | func main() { 31 | var err error 32 | 33 | var image string 34 | var db database.PgDB 35 | var qs queue.Service 36 | 37 | if os.Getenv("MB_QUEUE_TYPE") == "nats" { 38 | qs = queue.NewNatsService() 39 | } else { 40 | qs = queue.NewSqsService() 41 | } 42 | 43 | cmd := utils.GetArgOrLogError("cmd", 1) 44 | 45 | if cmd != "api" && cmd != "inspector" && cmd != "size" { 46 | image = utils.GetArgOrLogError("image", 2) 47 | } 48 | 49 | db, err = database.GetDB() 50 | if err != nil { 51 | log.Errorf("Failed to get DB: %v", err) 52 | return 53 | } 54 | 55 | switch cmd { 56 | case "api": 57 | log.Info("starting microbadger api") 58 | rs := registry.NewService() 59 | hs := hub.NewService() 60 | es := encryption.NewService() 61 | api.StartServer(db, qs, rs, hs, es) 62 | case "inspector": 63 | log.Info("starting inspector") 64 | hs := hub.NewService() 65 | rs := registry.NewService() 66 | es := encryption.NewService() 67 | startInspector(db, qs, hs, rs, es) 68 | case "size": 69 | log.Info("starting size inspector") 70 | rs := registry.NewService() 71 | es := encryption.NewService() 72 | startSizeInspector(db, qs, rs, es) 73 | case "feature": 74 | log.Infof("Feature image %s", image) 75 | err := db.FeatureImage(image, true) 76 | if err != nil { 77 | log.Error("Failed to feature %s: %v", image, err) 78 | } 79 | case "unfeature": 80 | log.Infof("Unfeature image %s", image) 81 | err := db.FeatureImage(image, false) 82 | if err != nil { 83 | log.Error("Failed to unfeature %s: %v", image, err) 84 | } 85 | default: 86 | log.Errorf("Unrecognised command: %v", cmd) 87 | } 88 | } 89 | 90 | func startInspector(db database.PgDB, qs queue.Service, hs hub.InfoService, rs registry.Service, es encryption.Service) { 91 | for { 92 | img := qs.ReceiveImage() 93 | if img != nil && img.ImageName != "" { 94 | log.Infof("Received Image: %v", img.ImageName) 95 | err := inspector.Inspect(img.ImageName, &db, &rs, &hs, qs, es) 96 | 97 | // If we failed to inspect this item, it might well be because Docker Hub has behaved badly. 98 | // By not deleting it, it will get resent again at some point in the future. 99 | if err == nil { 100 | qs.DeleteImage(img) 101 | 102 | // Getting the size information can take a while so we do this asynchronously. 103 | // We have different environment variables for deciding which queue to send & receive on. 104 | qs.SendImage(img.ImageName, "Sent for size inspection") 105 | } else { 106 | log.Errorf("Failed to inspect %s", img.ImageName) 107 | } 108 | } 109 | } 110 | } 111 | 112 | func startSizeInspector(db database.PgDB, qs queue.Service, rs registry.Service, es encryption.Service) { 113 | for { 114 | img := qs.ReceiveImage() 115 | if img != nil { 116 | log.Debugf("Received Image for size processing: %v", img.ImageName) 117 | err := inspector.InspectSize(img.ImageName, &db, &rs, es) 118 | if err == nil { 119 | qs.DeleteImage(img) 120 | } else { 121 | log.Errorf("size inspection of %s: %v", img.ImageName, err) 122 | } 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/microscaling/microbadger/database" 7 | ) 8 | 9 | func getTestDB(t *testing.T) database.PgDB { 10 | host := "microbadger-dev.c5iapezpnud7.us-east-1.rds.amazonaws.com" 11 | user := "microbadger" 12 | password := "cupremotebox" 13 | dbname := "microbadger" 14 | db, err := database.GetPostgres(host, user, dbname, password, false) 15 | if err != nil { 16 | t.Fatalf("Failed to open database: %v", err) 17 | } 18 | 19 | return db 20 | } 21 | -------------------------------------------------------------------------------- /notifier/Makefile: -------------------------------------------------------------------------------- 1 | # Binary can be overidden with an env var. 2 | NOTIFY_BINARY ?= notifier 3 | 4 | SOURCES := $(shell find ../. -name '*.go') 5 | 6 | $(NOTIFY_BINARY): $(SOURCES) 7 | # Compile for Linux 8 | GOOS=linux go build -o $(NOTIFY_BINARY) 9 | -------------------------------------------------------------------------------- /notifier/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net/http" 7 | "os" 8 | "time" 9 | 10 | "github.com/op/go-logging" 11 | 12 | "github.com/microscaling/microbadger/database" 13 | "github.com/microscaling/microbadger/queue" 14 | "github.com/microscaling/microbadger/utils" 15 | ) 16 | 17 | var ( 18 | log = logging.MustGetLogger("mbnotify") 19 | ) 20 | 21 | const constPollQueueTimeout = 250 // milliseconds - how often to check the queue for messages. 22 | const constNotificationRetries = 5 23 | 24 | func init() { 25 | utils.InitLogging() 26 | } 27 | 28 | func main() { 29 | var err error 30 | 31 | var db database.PgDB 32 | var qs queue.Service 33 | 34 | db, err = database.GetDB() 35 | if err != nil { 36 | log.Errorf("Failed to get DB: %v", err) 37 | return 38 | } 39 | 40 | if os.Getenv("MB_QUEUE_TYPE") == "nats" { 41 | qs = queue.NewNatsService() 42 | } else { 43 | qs = queue.NewSqsService() 44 | } 45 | 46 | log.Info("starting notifier") 47 | startNotifier(db, qs) 48 | } 49 | 50 | // Polls an SQS queue for notifications that need to be sent. 51 | func startNotifier(db database.PgDB, qs queue.Service) { 52 | pollQueueTimeout := time.NewTicker(constPollQueueTimeout * time.Millisecond) 53 | for range pollQueueTimeout.C { 54 | msg := qs.ReceiveNotification() 55 | if msg != nil { 56 | notifyMsgID := msg.NotificationID 57 | 58 | log.Infof("Sending notification for: %d", notifyMsgID) 59 | success, attempts, err := sendNotification(db, notifyMsgID) 60 | if err != nil { 61 | log.Errorf("Error sending notification for %d: %v", notifyMsgID, err) 62 | } 63 | 64 | // Don't retry more than a certain number of times 65 | if success || (attempts >= constNotificationRetries) { 66 | log.Infof("Notification %d stopping after %d attempts", notifyMsgID, attempts) 67 | qs.DeleteNotification(msg) 68 | } 69 | } 70 | } 71 | } 72 | 73 | // Sends a notification that an image has changed. 74 | func sendNotification(db database.PgDB, notifyMsgID uint) (success bool, attempts int, err error) { 75 | nm, err := db.GetNotificationMessage(notifyMsgID) 76 | if err != nil { 77 | return false, 0, err 78 | } 79 | 80 | // Call the webhook to send the notification. 81 | statusCode, resp, err := postMessage(nm.WebhookURL, nm.Message.RawMessage) 82 | if err != nil { 83 | log.Errorf("Error sending notification %v", err) 84 | } 85 | 86 | nm.Attempts++ 87 | nm.StatusCode = statusCode 88 | nm.Response = string(resp) 89 | nm.SentAt = time.Now() 90 | 91 | // If the webhook returns a response in the 200s its counted as a success. 92 | if statusCode >= 200 && statusCode <= 299 { 93 | success = true 94 | } else { 95 | log.Infof("Notification response %d for ID %d", statusCode, notifyMsgID) 96 | } 97 | 98 | err = db.SaveNotificationMessage(&nm) 99 | return success, nm.Attempts, err 100 | } 101 | 102 | // Post a message to a webhook. 103 | func postMessage(url string, request []byte) (status int, response []byte, err error) { 104 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(request)) 105 | req.Header.Set("Content-Type", "application/json") 106 | 107 | client := &http.Client{} 108 | resp, err := client.Do(req) 109 | if err != nil { 110 | return status, response, err 111 | } 112 | defer resp.Body.Close() 113 | 114 | response, err = ioutil.ReadAll(resp.Body) 115 | 116 | return resp.StatusCode, response, err 117 | } 118 | -------------------------------------------------------------------------------- /postgres/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:10.6 2 | 3 | COPY init-user-db.sh /docker-entrypoint-initdb.d -------------------------------------------------------------------------------- /postgres/init-user-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL 5 | CREATE USER microbadger; 6 | ALTER USER microbadger WITH PASSWORD 'microbadger'; 7 | CREATE DATABASE microbadger; 8 | GRANT ALL PRIVILEGES ON DATABASE microbadger TO microbadger; 9 | EOSQL -------------------------------------------------------------------------------- /queue/mock.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | // MockService for tests 4 | type MockService struct{} 5 | 6 | // make sure it satisfies the interface 7 | var _ Service = (*MockService)(nil) 8 | 9 | func NewMockService() MockService { 10 | return MockService{} 11 | } 12 | 13 | // SendImage on mock queue always succeeds 14 | func (q MockService) SendImage(imageName string, state string) (err error) { 15 | log.Infof("Sending %s image %s to queue", state, imageName) 16 | return nil 17 | } 18 | 19 | // ReceiveImage on mock queue always succeeds 20 | func (q MockService) ReceiveImage() *ImageQueueMessage { 21 | log.Infof("Received image with no name from mock queue.") 22 | return &ImageQueueMessage{} 23 | } 24 | 25 | // DeleteImage on mock queue always succeeds 26 | func (q MockService) DeleteImage(img *ImageQueueMessage) error { 27 | log.Infof("Deleted image %s from queue.", img.ImageName) 28 | return nil 29 | } 30 | 31 | // SendNotification on mock queue always succeeds 32 | func (q MockService) SendNotification(id uint) (err error) { 33 | log.Infof("Sending notification message %d to queue", id) 34 | return nil 35 | } 36 | 37 | // ReceiveNotification on mock queue always succeeds 38 | func (q MockService) ReceiveNotification() *NotificationQueueMessage { 39 | log.Infof("Received notification from mock queue.") 40 | return &NotificationQueueMessage{} 41 | } 42 | 43 | // DeleteNotification on mock queue always succeeds 44 | func (q MockService) DeleteNotification(notify *NotificationQueueMessage) error { 45 | log.Info("Deleted notification from queue.") 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /queue/nats.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "os" 7 | "time" 8 | 9 | nats "github.com/nats-io/nats.go" 10 | ) 11 | 12 | // NatsService for sending and receiving messages to Nats. 13 | type NatsService struct { 14 | nc *nats.Conn 15 | // Image send & receive queues are different for inspector & size processes, 16 | // so that's why we need both send & receive 17 | imageSendQueueName string 18 | imageReceiveQueueName string 19 | notificationQueueName string 20 | } 21 | 22 | // NewNatsService opens a new session with Nats. 23 | func NewNatsService() NatsService { 24 | baseURL := os.Getenv("NATS_BASE_URL") 25 | 26 | nc, err := nats.Connect(baseURL) 27 | if err != nil { 28 | log.Errorf("Unable to connect to queue at %s: %v", nats.DefaultURL, err) 29 | } 30 | 31 | receiveQueueURL := os.Getenv("NATS_RECEIVE_QUEUE_NAME") 32 | 33 | return NatsService{ 34 | nc: nc, 35 | imageSendQueueName: os.Getenv("NATS_SEND_QUEUE_NAME"), 36 | imageReceiveQueueName: receiveQueueURL, 37 | notificationQueueName: os.Getenv("MB_NOTIFY_QUEUE_NAME"), 38 | } 39 | } 40 | 41 | // SendImage to the Nats queue for processing by the Inspector. 42 | func (q NatsService) SendImage(imageName string, state string) (err error) { 43 | log.Debugf("Sending image %s to queue", imageName) 44 | 45 | msg := ImageQueueMessage{ 46 | ImageName: imageName, 47 | } 48 | 49 | bytes, err := json.Marshal(msg) 50 | if err != nil { 51 | log.Errorf("Error: %v", err) 52 | return err 53 | } 54 | 55 | err = q.natsSend(q.imageSendQueueName, bytes) 56 | if err != nil { 57 | log.Errorf("Failed to send image %s: %v", imageName, err) 58 | return 59 | } 60 | 61 | log.Infof("%s image %s to queue", state, imageName) 62 | return err 63 | } 64 | 65 | // ReceiveImage from the Nats queue for processing by the inspector. 66 | func (q NatsService) ReceiveImage() *ImageQueueMessage { 67 | log.Debugf("Receiving on queue %s", q.imageReceiveQueueName) 68 | 69 | msg, err := q.natsReceive(q.imageReceiveQueueName) 70 | if err != nil { 71 | log.Errorf("Error receiving image from queue: %v", err) 72 | return nil 73 | } 74 | 75 | if msg != nil { 76 | var img ImageQueueMessage 77 | 78 | err = json.Unmarshal(msg, &img) 79 | if err != nil { 80 | log.Errorf("Error unmarshaling from %v, error is %v", img, err) 81 | return nil 82 | } 83 | 84 | log.Infof("Received image %s from queue.", img.ImageName) 85 | 86 | return &ImageQueueMessage{ 87 | ImageName: img.ImageName, 88 | } 89 | } 90 | 91 | return nil 92 | } 93 | 94 | // DeleteImage from the queue once it has successfully been inspected. 95 | func (q NatsService) DeleteImage(img *ImageQueueMessage) error { 96 | log.Infof("Deleting image %s not implemented for NATS.", img.ImageName) 97 | return nil 98 | } 99 | 100 | // SendNotification to the SQS queue for processing by the Notifier. 101 | func (q NatsService) SendNotification(notificationID uint) (err error) { 102 | log.Debugf("Sending notification %s to queue", notificationID) 103 | 104 | msg := NotificationQueueMessage{ 105 | NotificationID: notificationID, 106 | } 107 | 108 | bytes, err := json.Marshal(msg) 109 | if err != nil { 110 | log.Errorf("Error: %v", err) 111 | return err 112 | } 113 | 114 | q.natsSend(q.notificationQueueName, bytes) 115 | 116 | return err 117 | } 118 | 119 | // ReceiveNotification from the SQS queue for sending by the notifier 120 | func (q NatsService) ReceiveNotification() *NotificationQueueMessage { 121 | log.Debug("Checking queue for notification messages.") 122 | 123 | msg, err := q.natsReceive(q.notificationQueueName) 124 | if err != nil { 125 | log.Errorf("Error receiving notification from queue: %v", err) 126 | return nil 127 | } 128 | 129 | if msg != nil { 130 | var notification NotificationQueueMessage 131 | 132 | err = json.Unmarshal(msg, ¬ification) 133 | if err != nil { 134 | log.Errorf("Error unmarshaling from %v, error is %v", notification, err) 135 | return nil 136 | } 137 | 138 | log.Infof("Received message for notification %d from queue.", notification.NotificationID) 139 | 140 | return ¬ification 141 | } 142 | 143 | return nil 144 | } 145 | 146 | // DeleteNotification from the queue once it has been sent. 147 | func (q NatsService) DeleteNotification(notification *NotificationQueueMessage) error { 148 | log.Infof("Deleting notification %d not implemented for NATS.", notification.NotificationID) 149 | return nil 150 | } 151 | 152 | func (q NatsService) natsSend(queueName string, message []byte) (err error) { 153 | log.Debugf("Sending on queue %s", queueName) 154 | 155 | err = q.nc.Publish(queueName, message) 156 | if err != nil { 157 | log.Errorf("Nats send Error: %v", err) 158 | } 159 | 160 | return err 161 | } 162 | 163 | func (q NatsService) natsReceive(queueName string) (message []byte, err error) { 164 | sub, err := q.nc.SubscribeSync(queueName) 165 | if err != nil { 166 | log.Errorf("NATS subscribe error: %v", err) 167 | return 168 | } 169 | 170 | msg, err := sub.NextMsg(30 * time.Second) 171 | if errors.Is(err, nats.ErrTimeout) { 172 | // Waiting for a message timed out. We return nil and will retry. 173 | return nil, nil 174 | } else if err != nil { 175 | log.Errorf("NATS next message error: %v", err) 176 | return 177 | } 178 | 179 | return msg.Data, nil 180 | } 181 | -------------------------------------------------------------------------------- /queue/spec.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | // TODO!! Find a better way of structuring this. Split of responsiblities between here and sender/receivers doesn't seem right 4 | // and I don't like the asymmetry of messages we pass in and out of queues, this all seems a bit long-winded. 5 | // Also, tests. 6 | 7 | // ImageQueueMessage is sent to the queue to trigger an inspection. 8 | type ImageQueueMessage struct { 9 | ImageName string `json:"ImageName"` 10 | ReceiptHandle *string 11 | } 12 | 13 | // NotificationQueueMessage is sent to the queue to trigger a notification. 14 | type NotificationQueueMessage struct { 15 | NotificationID uint `json:"NotificationID"` 16 | ReceiptHandle *string 17 | } 18 | 19 | // Service interface so we can mock it out for tests 20 | // TODO!! There must be a cleaner way that doesn't involve a different set of methods for each type of message 21 | type Service interface { 22 | SendImage(imageName string, state string) (err error) 23 | ReceiveImage() *ImageQueueMessage 24 | DeleteImage(img *ImageQueueMessage) error 25 | SendNotification(notificationID uint) error 26 | ReceiveNotification() *NotificationQueueMessage 27 | DeleteNotification(notify *NotificationQueueMessage) error 28 | } 29 | -------------------------------------------------------------------------------- /queue/sqs.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "strconv" 7 | 8 | "github.com/op/go-logging" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/sqs" 13 | ) 14 | 15 | const ( 16 | constImagePagePath = "/images/" 17 | constSentStateMessage = "Sent" 18 | ) 19 | 20 | var ( 21 | log = logging.MustGetLogger("mmqueue") 22 | ) 23 | 24 | // SqsService for sending and receiving messages on SQS 25 | type SqsService struct { 26 | svc *sqs.SQS 27 | // Image send & receive queues are different for inspector & size processes, 28 | // so that's why we need both send & receive 29 | imageSendQueueURL string 30 | imageReceiveQueueURL string 31 | notificationQueueURL string 32 | } 33 | 34 | // make sure it satisfies the interface 35 | var _ Service = (*SqsService)(nil) 36 | 37 | // NewSqsService opens a new session with SQS 38 | func NewSqsService() SqsService { 39 | svc := sqs.New(session.New()) 40 | receiveQueueURL := os.Getenv("SQS_RECEIVE_QUEUE_URL") 41 | 42 | // Enable long polling of up to 5 seconds on the queue we're receiving on 43 | _, err := svc.SetQueueAttributes(&sqs.SetQueueAttributesInput{ 44 | QueueUrl: aws.String(receiveQueueURL), 45 | Attributes: aws.StringMap(map[string]string{ 46 | "ReceiveMessageWaitTimeSeconds": strconv.Itoa(5), 47 | }), 48 | }) 49 | if err != nil { 50 | log.Errorf("Unable to update queue at %s: %v", receiveQueueURL, err) 51 | } 52 | 53 | return SqsService{ 54 | svc: svc, 55 | imageSendQueueURL: os.Getenv("SQS_SEND_QUEUE_URL"), 56 | imageReceiveQueueURL: receiveQueueURL, 57 | notificationQueueURL: os.Getenv("SQS_NOTIFY_QUEUE_URL"), 58 | } 59 | } 60 | 61 | // SendImage to the SQS queue for processing by the Inspector. 62 | func (q SqsService) SendImage(imageName string, state string) (err error) { 63 | log.Debugf("Sending image %s to queue", imageName) 64 | 65 | msg := ImageQueueMessage{ 66 | ImageName: imageName, 67 | } 68 | 69 | bytes, err := json.Marshal(msg) 70 | if err != nil { 71 | log.Errorf("Error: %v", err) 72 | return err 73 | } 74 | 75 | err = q.sqsSend(q.imageSendQueueURL, string(bytes)) 76 | if err != nil { 77 | log.Errorf("Failed to send image %s: %v", imageName, err) 78 | return 79 | } 80 | 81 | log.Infof("%s image %s to queue", state, imageName) 82 | return err 83 | } 84 | 85 | // ReceiveImage from the SQS queue for processing by the inspector. 86 | func (q SqsService) ReceiveImage() *ImageQueueMessage { 87 | log.Debugf("Receiving on queue %s", q.imageReceiveQueueURL) 88 | 89 | msg, err := q.sqsReceive(q.imageReceiveQueueURL) 90 | if err != nil { 91 | log.Errorf("Error receiving image from queue: %v", err) 92 | return nil 93 | } 94 | 95 | if msg != nil { 96 | var img ImageQueueMessage 97 | 98 | err = json.Unmarshal([]byte(*msg.Body), &img) 99 | if err != nil { 100 | log.Errorf("Error unmarshaling from %v, error is %v", img, err) 101 | return nil 102 | } 103 | 104 | log.Infof("Received image %s from queue.", img.ImageName) 105 | 106 | img.ReceiptHandle = msg.ReceiptHandle 107 | 108 | return &img 109 | } 110 | 111 | return nil 112 | } 113 | 114 | // DeleteImage from the queue once it has successfully been inspected. 115 | func (q SqsService) DeleteImage(img *ImageQueueMessage) error { 116 | log.Debugf("Deleting image %s from queue.", img.ImageName) 117 | 118 | err := q.sqsDelete(q.imageReceiveQueueURL, img.ReceiptHandle) 119 | if err == nil { 120 | log.Infof("Deleted image %s from queue.", img.ImageName) 121 | } 122 | 123 | return err 124 | } 125 | 126 | // SendNotification to the SQS queue for processing by the Notifier. 127 | func (q SqsService) SendNotification(notificationID uint) (err error) { 128 | log.Debugf("Sending notification %s to queue", notificationID) 129 | 130 | msg := NotificationQueueMessage{ 131 | NotificationID: notificationID, 132 | } 133 | 134 | bytes, err := json.Marshal(msg) 135 | if err != nil { 136 | log.Errorf("Error: %v", err) 137 | return err 138 | } 139 | 140 | q.sqsSend(q.notificationQueueURL, string(bytes)) 141 | 142 | return err 143 | } 144 | 145 | // ReceiveNotification from the SQS queue for sending by the notifier 146 | func (q SqsService) ReceiveNotification() *NotificationQueueMessage { 147 | log.Debug("Checking queue for notification messages.") 148 | 149 | msg, err := q.sqsReceive(q.notificationQueueURL) 150 | if err != nil { 151 | log.Errorf("Error receiving notification from queue: %v", err) 152 | return nil 153 | } 154 | 155 | if msg != nil { 156 | var notification NotificationQueueMessage 157 | 158 | err = json.Unmarshal([]byte(*msg.Body), ¬ification) 159 | if err != nil { 160 | log.Errorf("Error unmarshaling from %v, error is %v", notification, err) 161 | return nil 162 | } 163 | 164 | log.Infof("Received message for notification %d from queue.", notification.NotificationID) 165 | 166 | notification.ReceiptHandle = msg.ReceiptHandle 167 | 168 | return ¬ification 169 | } 170 | 171 | return nil 172 | } 173 | 174 | // DeleteNotification from the queue once it has been sent. 175 | func (q SqsService) DeleteNotification(notification *NotificationQueueMessage) error { 176 | log.Debugf("Deleting message %s from queue.", notification.NotificationID) 177 | 178 | err := q.sqsDelete(q.notificationQueueURL, notification.ReceiptHandle) 179 | if err == nil { 180 | log.Infof("Deleted message %s from queue.", notification.NotificationID) 181 | } 182 | 183 | return err 184 | } 185 | 186 | func (q SqsService) sqsSend(queueURL string, message string) (err error) { 187 | log.Debugf("Sending on queue %s", queueURL) 188 | 189 | params := &sqs.SendMessageInput{ 190 | MessageBody: aws.String(message), 191 | QueueUrl: aws.String(queueURL), 192 | } 193 | 194 | _, err = q.svc.SendMessage(params) 195 | if err != nil { 196 | log.Errorf("SQS send Error: %v", err) 197 | } 198 | 199 | return err 200 | } 201 | 202 | func (q SqsService) sqsReceive(queueURL string) (msg *sqs.Message, err error) { 203 | params := &sqs.ReceiveMessageInput{ 204 | QueueUrl: aws.String(queueURL), 205 | MaxNumberOfMessages: aws.Int64(1), 206 | WaitTimeSeconds: aws.Int64(5), 207 | } 208 | 209 | resp, err := q.svc.ReceiveMessage(params) 210 | if err != nil { 211 | log.Errorf("SQS receive error: %v", err) 212 | return 213 | } 214 | 215 | if len(resp.Messages) > 0 { 216 | msg = resp.Messages[0] 217 | } 218 | 219 | return 220 | } 221 | 222 | func (q SqsService) sqsDelete(queueURL string, receiptHandle *string) error { 223 | params := &sqs.DeleteMessageInput{ 224 | QueueUrl: aws.String(queueURL), 225 | ReceiptHandle: aws.String(*receiptHandle), 226 | } 227 | 228 | _, err := q.svc.DeleteMessage(params) 229 | if err != nil { 230 | log.Errorf("Error deleting SQS message: %v", err) 231 | } 232 | return err 233 | } 234 | -------------------------------------------------------------------------------- /registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | logging "github.com/op/go-logging" 14 | 15 | "github.com/microscaling/microbadger/utils" 16 | ) 17 | 18 | const registryURL = "https://registry.hub.docker.com" 19 | const authURL = "https://auth.docker.io" 20 | const serviceURL = "registry.docker.io" 21 | 22 | var log = logging.MustGetLogger("mminspect") 23 | 24 | // Service connects to the Docker Hub over the internet 25 | type Service struct { 26 | client *http.Client 27 | authURL string 28 | serviceURL string 29 | registryURL string 30 | } 31 | 32 | // NewService is a real info service 33 | func NewService() Service { 34 | return Service{ 35 | client: &http.Client{ 36 | // TODO Make timeout configurable. 37 | Timeout: 10 * time.Second, 38 | }, 39 | authURL: authURL, 40 | registryURL: registryURL, 41 | serviceURL: serviceURL, 42 | } 43 | } 44 | 45 | // NewMockService is for testing 46 | func NewMockService(transport *http.Transport, aurl string, rurl string, surl string) Service { 47 | 48 | return Service{ 49 | client: &http.Client{ 50 | Transport: transport, 51 | }, 52 | authURL: aurl, 53 | registryURL: rurl, 54 | serviceURL: surl, 55 | } 56 | } 57 | 58 | type Image struct { 59 | Name string 60 | User string 61 | Password string 62 | } 63 | 64 | type registryTagsList struct { 65 | Name string `json:"name"` 66 | Tags []string `json:"tags"` 67 | } 68 | 69 | type registryConfig struct { 70 | Labels json.RawMessage 71 | } 72 | 73 | type containerConfig struct { 74 | Cmd []string `json:"Cmd,omitempty"` 75 | } 76 | 77 | type V1Compatibility struct { 78 | Id string `json:"id"` 79 | Throwaway bool `json:"throwaway"` 80 | Config registryConfig `json:"config,omitempty"` 81 | ContainerConfig containerConfig `json:"container_config,omitempty"` 82 | Created string `json:"created,omitempty"` 83 | Author string `json:"author,omitempty"` 84 | } 85 | 86 | type registryHistory struct { 87 | V1Compatibility string `json:"v1Compatibility"` 88 | } 89 | 90 | type registryFsLayers struct { 91 | BlobSum string `json:"blobSum"` 92 | } 93 | 94 | type Manifest struct { 95 | SchemaVersion int `json:"schemaVersion"` 96 | Name string `json:"name"` 97 | History []registryHistory `json:"history"` 98 | FsLayers []registryFsLayers `json:"fsLayers"` 99 | } 100 | 101 | // DockerAuth is returned by the Docker Auth API. 102 | type dockerAuth struct { 103 | Token string `json:"token"` 104 | } 105 | 106 | // TokenAuthClient is associated with a particular org / image 107 | type TokenAuthClient struct { 108 | org string 109 | image string 110 | user string 111 | password string 112 | token string 113 | tokenURL string 114 | 115 | service *Service 116 | 117 | rl sync.RWMutex 118 | rateLimited bool 119 | rateLimitDelay int 120 | } 121 | 122 | func NewTokenAuth(i Image, rs *Service) (t *TokenAuthClient, err error) { 123 | org, image, _ := utils.ParseDockerImage(i.Name) 124 | 125 | t = &TokenAuthClient{ 126 | org: org, 127 | image: image, 128 | user: i.User, 129 | password: i.Password, 130 | rateLimitDelay: 10, 131 | } 132 | 133 | t.tokenURL = fmt.Sprintf("%s/token?service=%s&scope=repository:%s/%s:pull", rs.authURL, rs.serviceURL, org, image) 134 | t.service = rs 135 | 136 | err = t.getToken() 137 | if err != nil { 138 | log.Errorf("Failed to get auth token for %s/%s", org, image) 139 | } 140 | 141 | return t, err 142 | } 143 | 144 | func (t *TokenAuthClient) getToken() (err error) { 145 | var auth dockerAuth 146 | 147 | log.Debugf("Getting Auth Token URL for %s", t.tokenURL) 148 | 149 | req, err := http.NewRequest("GET", t.tokenURL, nil) 150 | if err != nil { 151 | log.Errorf("Failed to build API GET request err %v", err) 152 | return 153 | } 154 | 155 | if t.user != "" && t.password != "" { 156 | log.Debugf("Setting authorization for user %s", t.user) 157 | req.SetBasicAuth(t.user, t.password) 158 | } 159 | 160 | resp, err := t.service.client.Do(req) 161 | if err != nil { 162 | log.Errorf("Error getting auth token from %s: %v", t.tokenURL, err) 163 | return 164 | } 165 | 166 | defer resp.Body.Close() 167 | 168 | if resp.StatusCode != http.StatusOK { 169 | log.Errorf("Error getting auth token %d: %s", resp.StatusCode, resp.Status) 170 | return 171 | } 172 | 173 | body, err := ioutil.ReadAll(resp.Body) 174 | 175 | err = json.Unmarshal(body, &auth) 176 | if err != nil { 177 | log.Errorf("Error getting auth token for image %s/%s - %v", t.org, t.image, err) 178 | return 179 | } 180 | 181 | log.Debug("Got new auth token") 182 | t.token = auth.Token 183 | 184 | return 185 | } 186 | 187 | // ReqWithAuth sets the Authorization header on the request to use the provided token, 188 | func (t *TokenAuthClient) reqWithAuth(reqType string, reqURL string) (resp *http.Response, err error) { 189 | req, err := http.NewRequest(reqType, reqURL, nil) 190 | if err != nil { 191 | log.Errorf("Failed to build API GET request err %v", err) 192 | return 193 | } 194 | 195 | req.Header.Set("Content-Type", "application/json") 196 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.token)) 197 | 198 | resp, err = t.service.client.Do(req) 199 | if err != nil { 200 | log.Errorf("Error sending request: %v", err) 201 | return 202 | } 203 | 204 | if resp.StatusCode == http.StatusUnauthorized { 205 | log.Debug("Unauthorized on first attempt") 206 | 207 | // Perhaps this token has expired - try getting a new one and retrying the request 208 | err = t.getToken() 209 | if err != nil { 210 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.token)) 211 | resp, err = t.service.client.Do(req) 212 | if err != nil { 213 | log.Errorf("Error sending request the second time: %v", err) 214 | return 215 | } 216 | } 217 | } 218 | 219 | if resp.StatusCode != http.StatusOK { 220 | log.Debugf("Req failed: %d %s", resp.StatusCode, resp.Status) 221 | err = fmt.Errorf("Req failed: %d %s", resp.StatusCode, resp.Status) 222 | } 223 | 224 | return 225 | } 226 | 227 | func (t *TokenAuthClient) GetTags() (tags []string, err error) { 228 | var tagList registryTagsList 229 | 230 | tagsURL := fmt.Sprintf("%s/v2/%s/%s/tags/list", t.service.registryURL, t.org, t.image) 231 | log.Debugf("Getting tags at URL %s", tagsURL) 232 | 233 | resp, err := t.reqWithAuth("GET", tagsURL) 234 | if err != nil { 235 | log.Errorf("Failed to get tags: %v", err) 236 | return 237 | } 238 | 239 | defer resp.Body.Close() 240 | 241 | body, err := ioutil.ReadAll(resp.Body) 242 | err = json.Unmarshal(body, &tagList) 243 | if err != nil { 244 | log.Errorf("Error unmarshalling tags: %v", err) 245 | return 246 | } 247 | 248 | tags = tagList.Tags 249 | 250 | return 251 | } 252 | 253 | func (t *TokenAuthClient) GetManifest(tag string) (manifest Manifest, body []byte, err error) { 254 | 255 | manifestURL := fmt.Sprintf("%s/v2/%s/%s/manifests/%s", t.service.registryURL, t.org, t.image, tag) 256 | log.Debugf("Getting manifest at URL %s", manifestURL) 257 | 258 | resp, err := t.reqWithAuth("GET", manifestURL) 259 | if err != nil { 260 | log.Errorf("Failed to get manifest: %v", err) 261 | return 262 | } 263 | 264 | defer resp.Body.Close() 265 | 266 | body, err = ioutil.ReadAll(resp.Body) 267 | err = json.Unmarshal(body, &manifest) 268 | if err != nil { 269 | log.Errorf("Error unmarshalling manifest: %v", err) 270 | return 271 | } 272 | 273 | return 274 | } 275 | 276 | func (t *TokenAuthClient) getBlobDownloadSize(blobSum string) (size int64, err error) { 277 | blobURL := fmt.Sprintf("%s/v2/%s/%s/blobs/%s", t.service.registryURL, t.org, t.image, blobSum) 278 | log.Debugf("Getting blob at URL %s", blobURL) 279 | 280 | // Make a HEAD request because only the response headers are needed. 281 | resp, err := t.reqWithAuth("HEAD", blobURL) 282 | if err != nil { 283 | log.Errorf("Failed to get blob: %v", err) 284 | return 285 | } 286 | 287 | defer resp.Body.Close() 288 | 289 | size, err = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) 290 | if err != nil { 291 | log.Errorf("Failed to convert content length to int: %v", err) 292 | return 293 | } 294 | 295 | return 296 | } 297 | 298 | // GetImageDownloadSize gets the download size of the image. The total size is returned as well as an array 299 | // with the size of each layer. Only layers that affect the filesystem generate a new blob. If the blob already 300 | // exists for the image the layer size will be 0. The blobs are gzip compressed so the size on disk will be larger. 301 | func (t *TokenAuthClient) GetImageDownloadSize(m Manifest) (size int64, layerSizes []int64, err error) { 302 | 303 | if t.getRateLimited() { 304 | return 0, nil, fmt.Errorf("Rate limited") 305 | } 306 | 307 | blobs := make(map[string]int64) 308 | layerSizes = make([]int64, len(m.FsLayers)) 309 | 310 | // Get the download size for each layer. 311 | for i, l := range m.FsLayers { 312 | 313 | // Blob already exists so this layer is empty. 314 | if _, ok := blobs[l.BlobSum]; ok { 315 | layerSizes[i] = 0 316 | } else { 317 | // Blob is new so get the download size from the Registry API. 318 | blobSize, err := t.getBlobDownloadSize(l.BlobSum) 319 | if err != nil { 320 | log.Infof("Error getting blob size for image %s/%s blob %s: %v", t.org, t.image, l.BlobSum, err) 321 | 322 | if strings.Contains(err.Error(), "HAP429") { 323 | log.Info("Rate limited") 324 | t.setRateLimited() 325 | } 326 | 327 | return 0, nil, err 328 | } 329 | 330 | // Add blob to the map of known blobs. 331 | blobs[l.BlobSum] = blobSize 332 | 333 | // Set the download size for the layer and add to the total size. 334 | layerSizes[i] = blobSize 335 | size += blobSize 336 | } 337 | } 338 | 339 | return 340 | } 341 | 342 | func (t *TokenAuthClient) getRateLimited() bool { 343 | t.rl.RLock() 344 | defer t.rl.RUnlock() 345 | 346 | return t.rateLimited 347 | } 348 | 349 | func (t *TokenAuthClient) setRateLimited() { 350 | t.rl.Lock() 351 | defer t.rl.Unlock() 352 | 353 | log.Debug("Setting rate limiter") 354 | if t.rateLimited { 355 | log.Fatal("Shouldn't call setRateLimited when already rate limited!!") 356 | } 357 | 358 | t.rateLimited = true 359 | time.AfterFunc(time.Duration(t.rateLimitDelay)*time.Second, func() { 360 | log.Debug("Unsetting rate limiter") 361 | t.rl.Lock() 362 | t.rateLimited = false 363 | t.rl.Unlock() 364 | }) 365 | } 366 | -------------------------------------------------------------------------------- /registry/registry_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "net/url" 9 | "testing" 10 | ) 11 | 12 | func TestDecode(t *testing.T) { 13 | var v1c V1Compatibility 14 | 15 | // thing := `{"id":"a34e7649147cf4f5e2c0d3fb6a4bc51ac56eb1919e886bd71b5046bf2988ea10","parent":"867bcad75309b381a6401d7bcc41127e0c0b70ec22d54f82bd9e9d095189b42a","created":"2016-04-05T09:38:14.14489366Z","container":"19935b629d2aa27829b4cf3b1debce32c46e5c22053b290d5f7e464210277d58","container_config":{"Hostname":"523c4185767e","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","BUILD_PACKAGES=bash curl-dev"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT \u0026{[\"/run.sh\"]}"],"Image":"867bcad75309b381a6401d7bcc41127e0c0b70ec22d54f82bd9e9d095189b42a","Volumes":null,"WorkingDir":"","Entrypoint":["/run.sh"],"OnBuild":[],"Labels":{}},"docker_version":"1.9.1","author":"Ross Fairbanks \"ross@microscaling.com\"","config":{"Hostname":"523c4185767e","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","BUILD_PACKAGES=bash curl-dev"],"Cmd":null,"Image":"867bcad75309b381a6401d7bcc41127e0c0b70ec22d54f82bd9e9d095189b42a","Volumes":null,"WorkingDir":"","Entrypoint":["/run.sh"],"OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux"}` 16 | // thing := `{"id":"a34e7649147cf4f5e2c0d3fb6a4bc51ac56eb1919e886bd71b5046bf2988ea10","parent":"867bcad75309b381a6401d7bcc41127e0c0b70ec22d54f82bd9e9d095189b42a","created":"2016-04-05T09:38:14.14489366Z","container_config":{},"docker_version":"1.9.1","architecture":"amd64","os":"linux"}` 17 | // ,"config":{"Hostname":"523c4185767e","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","BUILD_PACKAGES=bash curl-dev"],"Cmd":null,"Image":"867bcad75309b381a6401d7bcc41127e0c0b70ec22d54f82bd9e9d095189b42a","Volumes":null,"WorkingDir":"","Entrypoint":["/run.sh"],"OnBuild":[],"Labels":{}} 18 | thing := `{"id":"a34e7649147cf4f5e2c0d3fb6a4bc51ac56eb1919e886bd71b5046bf2988ea10","parent":"867bcad75309b381a6401d7bcc41127e0c0b70ec22d54f82bd9e9d095189b42a","created":"2016-04-05T09:38:14.14489366Z", "author":"Ross Fairbanks \"ross@microscaling.com\"","config":{"Hostname":"523c4185767e", "Labels":{}}}` 19 | err := json.Unmarshal([]byte(thing), &v1c) 20 | if err != nil { 21 | t.Errorf("Failed: %v", err) 22 | } 23 | } 24 | 25 | func TestNewService(t *testing.T) { 26 | NewService() 27 | } 28 | 29 | // Check that we can create a mock service and get some fake info from it 30 | func TestRegistry(t *testing.T) { 31 | // Test server that always responds with 200 code and with a set payload 32 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 33 | w.WriteHeader(200) 34 | w.Header().Set("Content-Type", "application/json") 35 | switch r.URL.String() { 36 | case "http://fakehub/v2/repositories/org/image/": 37 | fmt.Fprintln(w, `{"user": "org", "name": "image", "namespace": "org", "status": 1, "description": "", "is_private": false, "is_automated": false, "can_edit": false, "star_count": 0, "pull_count": 17675, "last_updated": "2016-08-26T11:39:56.287301Z", "has_starred": false, "full_description": "blah-di-blah", "permissions": {"read": true, "write": false, "admin": false}}`) 38 | case "http://fakeauth/token?service=fake.service&scope=repository:org/image:pull": 39 | fmt.Fprintln(w, `{"token": "abctokenabc"}`) 40 | case "http://fakereg/v2/org/image/tags/list": 41 | fmt.Fprintln(w, `{"name": "org/image", "tags": ["tag1", "tag2"]}`) 42 | default: 43 | t.Errorf("Unexpected request to %s", r.URL.String()) 44 | } 45 | })) 46 | defer server.Close() 47 | 48 | // Make a transport that reroutes all traffic to the example server 49 | transport := &http.Transport{ 50 | Proxy: func(req *http.Request) (*url.URL, error) { 51 | return url.Parse(server.URL) 52 | }, 53 | } 54 | 55 | rs := NewMockService(transport, "http://fakeauth", "http://fakereg", "fake.service") 56 | i := Image{Name: "org/image"} 57 | 58 | tac, err := NewTokenAuth(i, &rs) 59 | if err != nil { 60 | t.Errorf("Unexpectedly failed to get token: %v", err) 61 | } 62 | 63 | if tac.token != "abctokenabc" { 64 | t.Errorf("Unexpcted token %s", tac.token) 65 | } 66 | 67 | tags, err := tac.GetTags() 68 | if err != nil { 69 | t.Errorf("Unexpectedly failed to get tags: %v", err) 70 | } 71 | 72 | if len(tags) != 2 { 73 | t.Errorf("Unexpected number of tags %d", len(tags)) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /utils/analytics.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | // "fmt" 5 | // "io/ioutil" 6 | "net/http" 7 | "net/url" 8 | ) 9 | 10 | const gaUrl = "https://www.google-analytics.com" 11 | const fixedCid = "555" 12 | 13 | // See https://developers.google.com/analytics/devguides/collection/protocol/v1/devguide 14 | 15 | type GoogleAnalytics struct { 16 | property string 17 | host string 18 | } 19 | 20 | func NewGoogleAnalytics(property string, host string) GoogleAnalytics { 21 | return GoogleAnalytics{ 22 | property: property, 23 | host: host, 24 | } 25 | } 26 | 27 | func post(postUrl string, params map[string]string) (err error) { 28 | vals := make(url.Values, len(params)+1) 29 | vals.Add("v", "1") // Measurement protocol version 30 | for key, val := range params { 31 | vals.Add(key, val) 32 | } 33 | 34 | _, err = http.PostForm(gaUrl+postUrl, vals) 35 | 36 | // This commented out code lets you view Google Analytics debug response 37 | // see https://developers.google.com/analytics/devguides/collection/protocol/v1/validating-hits 38 | 39 | // fmt.Printf("Sending analytics event") 40 | // resp, err := http.PostForm(gaUrl+postUrl, vals) 41 | 42 | // if err != nil { 43 | // fmt.Printf("Error: %v", err) 44 | // } else { 45 | // htmlData, err := ioutil.ReadAll(resp.Body) 46 | // if err != nil { 47 | // fmt.Printf("Error: %v", err) 48 | // } else { 49 | // fmt.Printf(string(htmlData)) 50 | // resp.Body.Close() 51 | // } 52 | // } 53 | 54 | return err 55 | } 56 | 57 | // BadgeImageView records a page view for an image. 58 | // imageURl is the URL without the host e.g. /badges/image/microscaling/microscaling.svg 59 | func (a *GoogleAnalytics) ImageView(dataSource string, campaignSource string, campaignMedium string, campaignName string, r *http.Request) error { 60 | 61 | // GA seems to show these as source / medium / campaign 62 | params := map[string]string{ 63 | "tid": a.property, 64 | "ds": dataSource, 65 | "cid": fixedCid, 66 | "t": "pageview", 67 | "dh": a.host, 68 | "dp": r.URL.Path, 69 | "cs": campaignSource, 70 | "cm": campaignMedium, 71 | "cn": campaignName, 72 | "dr": r.Referer(), 73 | } 74 | 75 | // Use the /debug/collect version if we want to see Google Analytics debug response 76 | // return post("/debug/collect", params) 77 | return post("/collect", params) 78 | } 79 | -------------------------------------------------------------------------------- /utils/token.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/base64" 6 | ) 7 | 8 | const ( 9 | constAuthTokenLengthBytes = 20 10 | ) 11 | 12 | func generateRandomBytes(n int) ([]byte, error) { 13 | b := make([]byte, n) 14 | _, err := rand.Read(b) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | return b, nil 20 | } 21 | 22 | func generateRandomString(s int) (string, error) { 23 | b, err := generateRandomBytes(s) 24 | return base64.URLEncoding.EncodeToString(b), err 25 | } 26 | 27 | // GenerateAuthToken generates a string token for use in URLs. 28 | func GenerateAuthToken() (string, error) { 29 | return generateRandomString(constAuthTokenLengthBytes) 30 | } 31 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "regexp" 7 | "strings" 8 | 9 | logging "github.com/op/go-logging" 10 | ) 11 | 12 | var ( 13 | log = logging.MustGetLogger("microbadger") 14 | loggingInitialized bool 15 | ) 16 | 17 | // ParseDockerImage splits input into Docker org, image and tag portions. If no org is 18 | // provided the default library org is returned. If no tag is provided the latest tag is 19 | // returned. 20 | // TODO Latest tag logic is probably incorrect but changing it needs careful testing. 21 | func ParseDockerImage(imageInput string) (org string, image string, tag string) { 22 | // Separate image and tag for the Registry API. 23 | if strings.Contains(imageInput, ":") { 24 | image = strings.Split(imageInput, ":")[0] 25 | tag = strings.Split(imageInput, ":")[1] 26 | } else { 27 | image = imageInput 28 | tag = "latest" 29 | } 30 | 31 | // Separate organization and image name. 32 | if strings.Contains(image, "/") { 33 | org = strings.Split(image, "/")[0] 34 | image = strings.Split(image, "/")[1] 35 | } else { 36 | org = "library" 37 | } 38 | 39 | return org, image, tag 40 | } 41 | 42 | var badgeRe = regexp.MustCompile(`https?:\/\/images\.microbadger\.com\/badges(\/[a-z\-]*)(\/[a-z0-9\-\._]*)?\/[a-z0-9\-\._]*(:[a-zA-Z0-9\-\._]+)?\.svg`) 43 | 44 | // BadgesInstalled counts the number of MicroBadger badges we can find in a string (thus far, this is the Full Description) 45 | func BadgesInstalled(s string) int { 46 | matches := badgeRe.FindAllString(s, -1) 47 | return len(matches) 48 | } 49 | 50 | // GetEnvOrDefault returns an env var or the default value if it doesn't exist. 51 | func GetEnvOrDefault(name string, defaultValue string) string { 52 | v := os.Getenv(name) 53 | if v == "" { 54 | v = defaultValue 55 | } 56 | 57 | return v 58 | } 59 | 60 | // GetArgOrLogError gets a command line argument or raises an error. 61 | func GetArgOrLogError(name string, i int) string { 62 | v := "" 63 | if len(os.Args) >= i+1 { 64 | v = os.Args[i] 65 | } else { 66 | log.Errorf("No command line arg for %v", name) 67 | } 68 | 69 | return v 70 | } 71 | 72 | // InitLogging configures the logging settings. 73 | func InitLogging() { 74 | if loggingInitialized { 75 | log.Infof("Logging already initialized") 76 | return 77 | } 78 | 79 | // The DH_LOG_DEBUG environment variable controls what logging is output 80 | // By default the log level is INFO for all components 81 | // Adding a component name to DH_LOG_DEBUG makes its logging level DEBUG 82 | // In addition, if "detail" is included in the environment variable details of the process ID and file name / line number are included in the logs 83 | // DH_LOG_DEBUG="all" - turn on DEBUG for all components 84 | // DH_LOG_DEBUG="dhinspect,detail" - turn on DEBUG for the api package, and use the detailed logging format 85 | basicLogFormat := logging.MustStringFormatter(`%{color}%{level:.4s} %{time:15:04:05.000}: %{color:reset} %{message}`) 86 | detailLogFormat := logging.MustStringFormatter(`%{color}%{level:.4s} %{time:15:04:05.000} %{pid} %{shortfile}: %{color:reset} %{message}`) 87 | 88 | logComponents := GetEnvOrDefault("MB_LOG_DEBUG", "none") 89 | if strings.Contains(logComponents, "detail") { 90 | logging.SetFormatter(detailLogFormat) 91 | } else { 92 | logging.SetFormatter(basicLogFormat) 93 | } 94 | 95 | fmt.Println("Init Logging") 96 | logBackend := logging.NewLogBackend(os.Stderr, "", 0) 97 | logging.SetBackend(logBackend) 98 | 99 | var components = []string{"microbadger", "mmapi", "mmauth", "mminspect", "mmdata", "mmhub", "mmnotify", "mmqueue", "mmslack", "mmenc"} 100 | 101 | for _, component := range components { 102 | if strings.Contains(logComponents, component) || strings.Contains(logComponents, "all") { 103 | logging.SetLevel(logging.DEBUG, component) 104 | } else { 105 | logging.SetLevel(logging.INFO, component) 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBadgesInstalled(t *testing.T) { 8 | type test struct { 9 | s string 10 | n int 11 | } 12 | 13 | tests := []test{ 14 | {s: "hello", n: 0}, 15 | {s: "https://images.microbadger.com/badges/image/alpine.svg", n: 1}, 16 | {s: "https://images.microbadger.com/badges/version/alpine.svg", n: 1}, 17 | {s: "https://images.microbadger.com/badges/commit/alpine.svg", n: 1}, 18 | {s: "https://images.microbadger.com/badges/license/alpine.svg", n: 1}, 19 | {s: "https://images.microbadger.com/badges/version/bateau/alpine_baseimage.svg", n: 1}, 20 | {s: "https://images.microbadger.com/badges/commit/linuxadmin/cuda-7.5.svg", n: 1}, 21 | {s: "https://images.microbadger.com/badges/image/jetstack/kube-lego:tag.svg", n: 1}, 22 | {s: `blah blah [](http://microbadger.com/images/jetstack/kube-lego "Get your own image badge on microbadger.com") blah bladibalh`, n: 2},
23 | {s: "https://images.microbadger.com/badges/commit/j3tst4ck/kub3-l3g0:tag.svg", n: 1},
24 | {s: "https://images.microbadger.com/badges/license/j3tst4ck/kub3-l3g0:tag.svg", n: 1},
25 | // a link, but not a badge
26 | {s: "https://images.microbadger.com/image/alpine.svg", n: 0},
27 | }
28 |
29 | for id, tt := range tests {
30 | if n := BadgesInstalled(tt.s); n != tt.n {
31 | t.Errorf("#%d Unexpected count %d", id, n)
32 | }
33 | }
34 | }
35 |
--------------------------------------------------------------------------------