├── .gitignore ├── .travis.yml ├── Dockerfile ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── Makefile ├── README.md ├── asg_lifecycle_hook.go ├── controller.go ├── controller_test.go ├── docs ├── configmap.yaml └── deployment.yaml ├── label_selector.go ├── label_selector_test.go ├── main.go ├── observer.go ├── pkg └── aws │ └── session.go ├── pod_selector.go └── pod_selector_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | vendor/ 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.x 5 | 6 | before_install: 7 | - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 8 | - go get github.com/mattn/goveralls 9 | 10 | install: 11 | - dep ensure -vendor-only 12 | 13 | script: 14 | - make build.docker 15 | - goveralls -service=travis-ci 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.opensource.zalan.do/stups/alpine:latest 2 | MAINTAINER Team Teapot @ Zalando SE 3 | 4 | # add binary 5 | ADD build/linux/kube-node-ready-controller / 6 | 7 | ENTRYPOINT ["/kube-node-ready-controller"] 8 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | branch = "master" 6 | name = "github.com/alecthomas/template" 7 | packages = [ 8 | ".", 9 | "parse" 10 | ] 11 | revision = "a0175ee3bccc567396460bf5acd36800cb10c49c" 12 | 13 | [[projects]] 14 | branch = "master" 15 | name = "github.com/alecthomas/units" 16 | packages = ["."] 17 | revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a" 18 | 19 | [[projects]] 20 | name = "github.com/aws/aws-sdk-go" 21 | packages = [ 22 | "aws", 23 | "aws/awserr", 24 | "aws/awsutil", 25 | "aws/client", 26 | "aws/client/metadata", 27 | "aws/corehandlers", 28 | "aws/credentials", 29 | "aws/credentials/ec2rolecreds", 30 | "aws/credentials/endpointcreds", 31 | "aws/credentials/stscreds", 32 | "aws/defaults", 33 | "aws/ec2metadata", 34 | "aws/endpoints", 35 | "aws/request", 36 | "aws/session", 37 | "aws/signer/v4", 38 | "internal/sdkio", 39 | "internal/sdkrand", 40 | "internal/shareddefaults", 41 | "private/protocol", 42 | "private/protocol/ec2query", 43 | "private/protocol/query", 44 | "private/protocol/query/queryutil", 45 | "private/protocol/rest", 46 | "private/protocol/xml/xmlutil", 47 | "service/autoscaling", 48 | "service/autoscaling/autoscalingiface", 49 | "service/ec2", 50 | "service/ec2/ec2iface", 51 | "service/sts" 52 | ] 53 | revision = "31bd69f7db00cbf3d85d129e16d42304cb6e455f" 54 | version = "v1.13.44" 55 | 56 | [[projects]] 57 | branch = "master" 58 | name = "github.com/beorn7/perks" 59 | packages = ["quantile"] 60 | revision = "3a771d992973f24aa725d07868b467d1ddfceafb" 61 | 62 | [[projects]] 63 | name = "github.com/cenkalti/backoff" 64 | packages = ["."] 65 | revision = "2ea60e5f094469f9e65adb9cd103795b73ae743e" 66 | version = "v2.0.0" 67 | 68 | [[projects]] 69 | name = "github.com/ghodss/yaml" 70 | packages = ["."] 71 | revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" 72 | version = "v1.0.0" 73 | 74 | [[projects]] 75 | name = "github.com/go-ini/ini" 76 | packages = ["."] 77 | revision = "6529cf7c58879c08d927016dde4477f18a0634cb" 78 | version = "v1.36.0" 79 | 80 | [[projects]] 81 | name = "github.com/gogo/protobuf" 82 | packages = [ 83 | "proto", 84 | "sortkeys" 85 | ] 86 | revision = "1adfc126b41513cc696b209667c8656ea7aac67c" 87 | version = "v1.0.0" 88 | 89 | [[projects]] 90 | branch = "master" 91 | name = "github.com/golang/glog" 92 | packages = ["."] 93 | revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" 94 | 95 | [[projects]] 96 | name = "github.com/golang/protobuf" 97 | packages = [ 98 | "proto", 99 | "ptypes", 100 | "ptypes/any", 101 | "ptypes/duration", 102 | "ptypes/timestamp" 103 | ] 104 | revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" 105 | version = "v1.1.0" 106 | 107 | [[projects]] 108 | branch = "master" 109 | name = "github.com/google/gofuzz" 110 | packages = ["."] 111 | revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" 112 | 113 | [[projects]] 114 | name = "github.com/googleapis/gnostic" 115 | packages = [ 116 | "OpenAPIv2", 117 | "compiler", 118 | "extensions" 119 | ] 120 | revision = "ee43cbb60db7bd22502942cccbc39059117352ab" 121 | version = "v0.1.0" 122 | 123 | [[projects]] 124 | name = "github.com/jmespath/go-jmespath" 125 | packages = ["."] 126 | revision = "0b12d6b5" 127 | 128 | [[projects]] 129 | name = "github.com/json-iterator/go" 130 | packages = ["."] 131 | revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4" 132 | version = "1.1.3" 133 | 134 | [[projects]] 135 | name = "github.com/matttproud/golang_protobuf_extensions" 136 | packages = ["pbutil"] 137 | revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" 138 | version = "v1.0.0" 139 | 140 | [[projects]] 141 | name = "github.com/modern-go/concurrent" 142 | packages = ["."] 143 | revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" 144 | version = "1.0.3" 145 | 146 | [[projects]] 147 | name = "github.com/modern-go/reflect2" 148 | packages = ["."] 149 | revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f" 150 | version = "1.0.0" 151 | 152 | [[projects]] 153 | name = "github.com/prometheus/client_golang" 154 | packages = [ 155 | "prometheus", 156 | "prometheus/promhttp" 157 | ] 158 | revision = "c5b7fccd204277076155f10851dad72b76a49317" 159 | version = "v0.8.0" 160 | 161 | [[projects]] 162 | branch = "master" 163 | name = "github.com/prometheus/client_model" 164 | packages = ["go"] 165 | revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" 166 | 167 | [[projects]] 168 | branch = "master" 169 | name = "github.com/prometheus/common" 170 | packages = [ 171 | "expfmt", 172 | "internal/bitbucket.org/ww/goautoneg", 173 | "model" 174 | ] 175 | revision = "d811d2e9bf898806ecfb6ef6296774b13ffc314c" 176 | 177 | [[projects]] 178 | branch = "master" 179 | name = "github.com/prometheus/procfs" 180 | packages = [ 181 | ".", 182 | "internal/util", 183 | "nfs", 184 | "xfs" 185 | ] 186 | revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e" 187 | 188 | [[projects]] 189 | name = "github.com/sirupsen/logrus" 190 | packages = ["."] 191 | revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" 192 | version = "v1.0.5" 193 | 194 | [[projects]] 195 | name = "github.com/spf13/pflag" 196 | packages = ["."] 197 | revision = "583c0c0531f06d5278b7d917446061adc344b5cd" 198 | version = "v1.0.1" 199 | 200 | [[projects]] 201 | branch = "master" 202 | name = "golang.org/x/crypto" 203 | packages = ["ssh/terminal"] 204 | revision = "21052ae46654ecf18dfdba0f7c12701a1e2b3164" 205 | 206 | [[projects]] 207 | branch = "master" 208 | name = "golang.org/x/net" 209 | packages = [ 210 | "context", 211 | "http/httpguts", 212 | "http2", 213 | "http2/hpack", 214 | "idna" 215 | ] 216 | revision = "f73e4c9ed3b7ebdd5f699a16a880c2b1994e50dd" 217 | 218 | [[projects]] 219 | branch = "master" 220 | name = "golang.org/x/sys" 221 | packages = [ 222 | "unix", 223 | "windows" 224 | ] 225 | revision = "7db1c3b1a98089d0071c84f646ff5c96aad43682" 226 | 227 | [[projects]] 228 | name = "golang.org/x/text" 229 | packages = [ 230 | "collate", 231 | "collate/build", 232 | "internal/colltab", 233 | "internal/gen", 234 | "internal/tag", 235 | "internal/triegen", 236 | "internal/ucd", 237 | "language", 238 | "secure/bidirule", 239 | "transform", 240 | "unicode/bidi", 241 | "unicode/cldr", 242 | "unicode/norm", 243 | "unicode/rangetable" 244 | ] 245 | revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" 246 | version = "v0.3.0" 247 | 248 | [[projects]] 249 | branch = "master" 250 | name = "golang.org/x/time" 251 | packages = ["rate"] 252 | revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" 253 | 254 | [[projects]] 255 | name = "gopkg.in/alecthomas/kingpin.v2" 256 | packages = ["."] 257 | revision = "947dcec5ba9c011838740e680966fd7087a71d0d" 258 | version = "v2.2.6" 259 | 260 | [[projects]] 261 | name = "gopkg.in/inf.v0" 262 | packages = ["."] 263 | revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" 264 | version = "v0.9.1" 265 | 266 | [[projects]] 267 | name = "gopkg.in/yaml.v2" 268 | packages = ["."] 269 | revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" 270 | version = "v2.2.1" 271 | 272 | [[projects]] 273 | branch = "master" 274 | name = "k8s.io/api" 275 | packages = [ 276 | "admissionregistration/v1alpha1", 277 | "admissionregistration/v1beta1", 278 | "apps/v1", 279 | "apps/v1beta1", 280 | "apps/v1beta2", 281 | "authentication/v1", 282 | "authentication/v1beta1", 283 | "authorization/v1", 284 | "authorization/v1beta1", 285 | "autoscaling/v1", 286 | "autoscaling/v2beta1", 287 | "batch/v1", 288 | "batch/v1beta1", 289 | "batch/v2alpha1", 290 | "certificates/v1beta1", 291 | "core/v1", 292 | "events/v1beta1", 293 | "extensions/v1beta1", 294 | "networking/v1", 295 | "policy/v1beta1", 296 | "rbac/v1", 297 | "rbac/v1alpha1", 298 | "rbac/v1beta1", 299 | "scheduling/v1alpha1", 300 | "settings/v1alpha1", 301 | "storage/v1", 302 | "storage/v1alpha1", 303 | "storage/v1beta1" 304 | ] 305 | revision = "8ab7c6a2fbd1724a52514855629cec31a8f86929" 306 | 307 | [[projects]] 308 | name = "k8s.io/apimachinery" 309 | packages = [ 310 | "pkg/api/errors", 311 | "pkg/api/meta", 312 | "pkg/api/resource", 313 | "pkg/apis/meta/v1", 314 | "pkg/apis/meta/v1/unstructured", 315 | "pkg/apis/meta/v1beta1", 316 | "pkg/conversion", 317 | "pkg/conversion/queryparams", 318 | "pkg/fields", 319 | "pkg/labels", 320 | "pkg/runtime", 321 | "pkg/runtime/schema", 322 | "pkg/runtime/serializer", 323 | "pkg/runtime/serializer/json", 324 | "pkg/runtime/serializer/protobuf", 325 | "pkg/runtime/serializer/recognizer", 326 | "pkg/runtime/serializer/streaming", 327 | "pkg/runtime/serializer/versioning", 328 | "pkg/selection", 329 | "pkg/types", 330 | "pkg/util/clock", 331 | "pkg/util/errors", 332 | "pkg/util/framer", 333 | "pkg/util/intstr", 334 | "pkg/util/json", 335 | "pkg/util/net", 336 | "pkg/util/runtime", 337 | "pkg/util/sets", 338 | "pkg/util/validation", 339 | "pkg/util/validation/field", 340 | "pkg/util/wait", 341 | "pkg/util/yaml", 342 | "pkg/version", 343 | "pkg/watch", 344 | "third_party/forked/golang/reflect" 345 | ] 346 | revision = "302974c03f7e50f16561ba237db776ab93594ef6" 347 | 348 | [[projects]] 349 | name = "k8s.io/client-go" 350 | packages = [ 351 | "discovery", 352 | "discovery/fake", 353 | "kubernetes", 354 | "kubernetes/fake", 355 | "kubernetes/scheme", 356 | "kubernetes/typed/admissionregistration/v1alpha1", 357 | "kubernetes/typed/admissionregistration/v1alpha1/fake", 358 | "kubernetes/typed/admissionregistration/v1beta1", 359 | "kubernetes/typed/admissionregistration/v1beta1/fake", 360 | "kubernetes/typed/apps/v1", 361 | "kubernetes/typed/apps/v1/fake", 362 | "kubernetes/typed/apps/v1beta1", 363 | "kubernetes/typed/apps/v1beta1/fake", 364 | "kubernetes/typed/apps/v1beta2", 365 | "kubernetes/typed/apps/v1beta2/fake", 366 | "kubernetes/typed/authentication/v1", 367 | "kubernetes/typed/authentication/v1/fake", 368 | "kubernetes/typed/authentication/v1beta1", 369 | "kubernetes/typed/authentication/v1beta1/fake", 370 | "kubernetes/typed/authorization/v1", 371 | "kubernetes/typed/authorization/v1/fake", 372 | "kubernetes/typed/authorization/v1beta1", 373 | "kubernetes/typed/authorization/v1beta1/fake", 374 | "kubernetes/typed/autoscaling/v1", 375 | "kubernetes/typed/autoscaling/v1/fake", 376 | "kubernetes/typed/autoscaling/v2beta1", 377 | "kubernetes/typed/autoscaling/v2beta1/fake", 378 | "kubernetes/typed/batch/v1", 379 | "kubernetes/typed/batch/v1/fake", 380 | "kubernetes/typed/batch/v1beta1", 381 | "kubernetes/typed/batch/v1beta1/fake", 382 | "kubernetes/typed/batch/v2alpha1", 383 | "kubernetes/typed/batch/v2alpha1/fake", 384 | "kubernetes/typed/certificates/v1beta1", 385 | "kubernetes/typed/certificates/v1beta1/fake", 386 | "kubernetes/typed/core/v1", 387 | "kubernetes/typed/core/v1/fake", 388 | "kubernetes/typed/events/v1beta1", 389 | "kubernetes/typed/events/v1beta1/fake", 390 | "kubernetes/typed/extensions/v1beta1", 391 | "kubernetes/typed/extensions/v1beta1/fake", 392 | "kubernetes/typed/networking/v1", 393 | "kubernetes/typed/networking/v1/fake", 394 | "kubernetes/typed/policy/v1beta1", 395 | "kubernetes/typed/policy/v1beta1/fake", 396 | "kubernetes/typed/rbac/v1", 397 | "kubernetes/typed/rbac/v1/fake", 398 | "kubernetes/typed/rbac/v1alpha1", 399 | "kubernetes/typed/rbac/v1alpha1/fake", 400 | "kubernetes/typed/rbac/v1beta1", 401 | "kubernetes/typed/rbac/v1beta1/fake", 402 | "kubernetes/typed/scheduling/v1alpha1", 403 | "kubernetes/typed/scheduling/v1alpha1/fake", 404 | "kubernetes/typed/settings/v1alpha1", 405 | "kubernetes/typed/settings/v1alpha1/fake", 406 | "kubernetes/typed/storage/v1", 407 | "kubernetes/typed/storage/v1/fake", 408 | "kubernetes/typed/storage/v1alpha1", 409 | "kubernetes/typed/storage/v1alpha1/fake", 410 | "kubernetes/typed/storage/v1beta1", 411 | "kubernetes/typed/storage/v1beta1/fake", 412 | "pkg/apis/clientauthentication", 413 | "pkg/apis/clientauthentication/v1alpha1", 414 | "pkg/version", 415 | "plugin/pkg/client/auth/exec", 416 | "rest", 417 | "rest/watch", 418 | "testing", 419 | "tools/clientcmd/api", 420 | "tools/metrics", 421 | "tools/reference", 422 | "transport", 423 | "util/cert", 424 | "util/flowcontrol", 425 | "util/integer" 426 | ] 427 | revision = "23781f4d6632d88e869066eaebb743857aa1ef9b" 428 | version = "v7.0.0" 429 | 430 | [solve-meta] 431 | analyzer-name = "dep" 432 | analyzer-version = 1 433 | inputs-digest = "26832df13f066c0a33f654c7c2b5456fe8a9a20dc520ec130f6c7c5b1a6d5b79" 434 | solver-name = "gps-cdcl" 435 | solver-version = 1 436 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | name = "github.com/aws/aws-sdk-go" 30 | version = "1.13.44" 31 | 32 | [[constraint]] 33 | name = "github.com/sirupsen/logrus" 34 | version = "1.0.5" 35 | 36 | [[constraint]] 37 | name = "gopkg.in/alecthomas/kingpin.v2" 38 | version = "2.2.6" 39 | 40 | [[constraint]] 41 | name = "gopkg.in/yaml.v2" 42 | version = "2.2.1" 43 | 44 | [[constraint]] 45 | branch = "master" 46 | name = "k8s.io/api" 47 | 48 | [[constraint]] 49 | revision = "302974c03f7e50f16561ba237db776ab93594ef6" 50 | name = "k8s.io/apimachinery" 51 | 52 | [[constraint]] 53 | name = "k8s.io/client-go" 54 | version = "7.0.0" 55 | 56 | [prune] 57 | go-tests = true 58 | unused-packages = true 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean test check build.local build.linux build.osx build.docker build.push 2 | 3 | BINARY ?= kube-node-ready-controller 4 | VERSION ?= $(shell git describe --tags --always --dirty) 5 | IMAGE ?= mikkeloscar/$(BINARY) 6 | TAG ?= $(VERSION) 7 | SOURCES = $(shell find . -name '*.go') 8 | DOCKERFILE ?= Dockerfile 9 | GOPKGS = $(shell go list ./...) 10 | BUILD_FLAGS ?= -v 11 | LDFLAGS ?= -X main.version=$(VERSION) -w -s 12 | 13 | default: build.local 14 | 15 | clean: 16 | rm -rf build 17 | 18 | test: 19 | go test -v $(GOPKGS) 20 | 21 | check: 22 | golint $(GOPKGS) 23 | go vet -v $(GOPKGS) 24 | 25 | build.local: build/$(BINARY) 26 | build.linux: build/linux/$(BINARY) 27 | build.osx: build/osx/$(BINARY) 28 | 29 | build/$(BINARY): $(SOURCES) 30 | CGO_ENABLED=0 go build -o build/$(BINARY) $(BUILD_FLAGS) -ldflags "$(LDFLAGS)" . 31 | 32 | build/linux/$(BINARY): $(SOURCES) 33 | GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(BUILD_FLAGS) -o build/linux/$(BINARY) -ldflags "$(LDFLAGS)" . 34 | 35 | build/osx/$(BINARY): $(SOURCES) 36 | GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build $(BUILD_FLAGS) -o build/osx/$(BINARY) -ldflags "$(LDFLAGS)" . 37 | 38 | build.docker: build.linux 39 | docker build --rm -t "$(IMAGE):$(TAG)" -f $(DOCKERFILE) . 40 | 41 | build.push: build.docker 42 | docker push "$(IMAGE):$(TAG)" 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kube Node Ready Controller 2 | [![Build Status](https://travis-ci.org/mikkeloscar/kube-node-ready-controller.svg?branch=master)](https://travis-ci.org/mikkeloscar/kube-node-ready-controller) 3 | [![Coverage Status](https://coveralls.io/repos/github/mikkeloscar/kube-node-ready-controller/badge.svg)](https://coveralls.io/github/mikkeloscar/kube-node-ready-controller) 4 | 5 | It is common to run a number of system pods (usually as DaemonSets) on each 6 | node in a Kubernetes cluster in order to provide basic functionality. For 7 | instance, you might want to run [kube2iam][kube2iam] to control AWS IAM access 8 | for the services in your cluster, or you might run a 9 | [logging-agent][logging-agent] to automatically ship logs to a central 10 | location. Whatever your use case might be, you would expect these components to 11 | run on all nodes, ideally before "normal" services are scheduled to the nodes. 12 | 13 | By default in Kubernetes a node is considered `Ready`/`NotReady` based 14 | on the node health independent of what system pods might be scheduled on the 15 | node. 16 | 17 | `kube-node-ready-controller` adds a layer on top to indicate whether a 18 | node is ready for workloads based on a list of system pods which must be 19 | running on the node before it is considered ready. 20 | 21 | ## How it works 22 | 23 | The controller is configured with a list of pod selectors (namespace + labels) 24 | and for each node it will check if the pods are scheduled and has status ready. 25 | If all expected pods are ready it will make sure the node doesn't have the 26 | [taint][taints-tolerations] `node.alpha.kubernetes.io/notReady-workload`. If 27 | some expected pods aren't ready, it will make sure to set the taint on the 28 | node. 29 | 30 | ## Setup 31 | 32 | The `kube-node-ready-controller` can be run as a deployment in the cluster. See 33 | [deployment.yaml](/docs/deployment.yaml). 34 | 35 | To deploy it to your cluster modify the `--pod-selector` args to match your 36 | system pods. The format for the selector is 37 | `:=,=`. Alternatively 38 | you can set the flag `--pod-selector-configmap` and use a configMap to 39 | configure the selectors ([full example](/docs/configmap.yaml)): 40 | 41 | ```yaml 42 | selectors: 43 | - namespace: kube-system 44 | labels: 45 | foo: bar 46 | ``` 47 | 48 | With this approach you can change the selectors at runtime, just by updating 49 | the config map. 50 | 51 | Once configured, deploy it by running: 52 | 53 | ```bash 54 | $ kubectl apply -f docs/deployment.yaml 55 | ``` 56 | 57 | Note that we set the following toleration on the pod: 58 | 59 | ```yaml 60 | tolerations: 61 | - key: node.alpha.kubernetes.io/notReady-workload 62 | operator: Exists 63 | effect: NoSchedule 64 | ``` 65 | 66 | This is done to ensure that it can be scheduled even on nodes that are not 67 | ready. 68 | 69 | You must add the same toleration to all the system pods that should be 70 | scheduled before the node is considered ready. If you fail to add the 71 | toleration, the pod won't get scheduled and the node will thus never become 72 | ready. 73 | 74 | Lastly you must configure the nodes to have the `notReady-workload` taint when 75 | they register with the cluster. This can be done by setting the flag 76 | `--register-with-taints=node.alpha.kubernetes.io/notReady-workload=:NoSchedule` 77 | on the `kubelet`. 78 | 79 | You can also add the taint manually with `kubectl` to test it: 80 | 81 | ```bash 82 | $ kubectl taint nodes "node.alpha.kubernetes.io/notReady-workload=:NoSchedule" 83 | ``` 84 | 85 | ## Hooks 86 | 87 | As an extra feature `kube-node-ready-controller` has optional support for 88 | triggering hooks when a node is marked as ready. 89 | 90 | ### AWS Autoscaling Lifecycle Hook 91 | 92 | Trigger AWS Autoscaling Group lifecycle hook when node becomes ready. This can 93 | be used to signal the Autoscaling Group that the node is in service. 94 | 95 | Enable the hook with the flag `--asg-lifecycle-hook=`. This assumes 96 | you have a hook with the defined name on the Autoscaling groups of all the 97 | nodes managed by the controller. 98 | 99 | ## TODO 100 | 101 | * [x] Make it possible to configure pod selectors via a config map. 102 | 103 | * [ ] Instead of long polling the node list, add a Watch feature. 104 | 105 | 106 | [kube2iam]: https://github.com/jtblin/kube2iam 107 | [logging-agent]: https://github.com/zalando-incubator/kubernetes-log-watcher 108 | [taints-tolerations]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature 109 | -------------------------------------------------------------------------------- /asg_lifecycle_hook.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/aws/aws-sdk-go/service/autoscaling" 10 | "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" 11 | ) 12 | 13 | const ( 14 | lifecycleActionContinue = "CONTINUE" 15 | ) 16 | 17 | // Hook is an interface describing a hook which can be triggered given an 18 | // instance id. 19 | type Hook interface { 20 | Name() string 21 | Trigger(providerID string) error 22 | } 23 | 24 | // ASGLifecycleHook defines an ASG lifecycle hook to be triggered on node 25 | // Ready. 26 | type ASGLifecycleHook struct { 27 | hookName string 28 | svc autoscalingiface.AutoScalingAPI 29 | } 30 | 31 | // NewASGLifecycleHook creates a new asg lifecycle hook. 32 | func NewASGLifecycleHook(sess *session.Session, hookName string) *ASGLifecycleHook { 33 | return &ASGLifecycleHook{ 34 | hookName: hookName, 35 | svc: autoscaling.New(sess), 36 | } 37 | } 38 | 39 | // Name returns the hook name. 40 | func (a *ASGLifecycleHook) Name() string { 41 | return a.hookName 42 | } 43 | 44 | // Trigger triggers a the ASG lifecycle hook for a given instance. 45 | func (a *ASGLifecycleHook) Trigger(providerID string) error { 46 | instanceID, err := instanceIDFromProviderID(providerID) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | // get ASG from instance-id 52 | instances := &autoscaling.DescribeAutoScalingInstancesInput{ 53 | InstanceIds: []*string{ 54 | aws.String(instanceID), 55 | }, 56 | } 57 | 58 | result, err := a.svc.DescribeAutoScalingInstances(instances) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | if len(result.AutoScalingInstances) != 1 { 64 | return fmt.Errorf("expected 1 instance returned, got %d", len(result.AutoScalingInstances)) 65 | } 66 | 67 | input := &autoscaling.CompleteLifecycleActionInput{ 68 | AutoScalingGroupName: result.AutoScalingInstances[0].AutoScalingGroupName, 69 | InstanceId: aws.String(instanceID), 70 | LifecycleActionResult: aws.String(lifecycleActionContinue), 71 | LifecycleHookName: aws.String(a.hookName), 72 | } 73 | 74 | _, err = a.svc.CompleteLifecycleAction(input) 75 | return err 76 | } 77 | 78 | // instanceIDFromProviderID extracts the EC2 instanceID from a Kubernetes 79 | // ProviderID. 80 | func instanceIDFromProviderID(providerID string) (string, error) { 81 | full := strings.TrimPrefix(providerID, "aws:///") 82 | split := strings.Split(full, "/") 83 | if len(split) != 2 { 84 | return "", fmt.Errorf("unexpected providerID format: %s", providerID) 85 | } 86 | 87 | return split[1], nil 88 | } 89 | -------------------------------------------------------------------------------- /controller.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "strings" 7 | "time" 8 | 9 | "github.com/cenkalti/backoff" 10 | log "github.com/sirupsen/logrus" 11 | "k8s.io/api/core/v1" 12 | "k8s.io/apimachinery/pkg/api/errors" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/apimachinery/pkg/labels" 15 | "k8s.io/client-go/kubernetes" 16 | ) 17 | 18 | const ( 19 | // ConfigMapSelectorsKey defines the key name of the config map where 20 | // the pod selector definition is defined. 21 | ConfigMapSelectorsKey = "pod_selectors" 22 | serviceAccountNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" 23 | maxConflictRetries = 50 24 | ) 25 | 26 | // NodeController updates the readiness taint of nodes based on expected 27 | // resources defined by selectors. 28 | type NodeController struct { 29 | kubernetes.Interface 30 | selectors []*PodSelector 31 | nodeSelectorLabels labels.Set 32 | interval time.Duration 33 | configMap string 34 | namespace string 35 | nodeReadyHooks []Hook 36 | nodeStartUpObserver NodeStartUpObserver 37 | taintNodeNotReadyName string 38 | } 39 | 40 | // NewNodeController initializes a new NodeController. 41 | func NewNodeController(client kubernetes.Interface, selectors []*PodSelector, nodeSelectorLabels map[string]string, taintNodeNotReadyName string, interval time.Duration, configMap string, hooks []Hook, nodeStartUpObserver NodeStartUpObserver) (*NodeController, error) { 42 | controller := &NodeController{ 43 | Interface: client, 44 | selectors: selectors, 45 | nodeSelectorLabels: labels.Set(nodeSelectorLabels), 46 | interval: interval, 47 | configMap: configMap, 48 | nodeReadyHooks: hooks, 49 | nodeStartUpObserver: nodeStartUpObserver, 50 | taintNodeNotReadyName: taintNodeNotReadyName, 51 | } 52 | 53 | if controller.configMap != "" { 54 | // get Current Namespace 55 | data, err := ioutil.ReadFile(serviceAccountNamespace) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | if ns := strings.TrimSpace(string(data)); len(ns) > 0 { 61 | controller.namespace = ns 62 | } 63 | } 64 | 65 | return controller, nil 66 | } 67 | 68 | func (n *NodeController) runOnce() error { 69 | // update selectors based on config map. 70 | if n.configMap != "" { 71 | err := n.getConfig() 72 | if err != nil { 73 | return err 74 | } 75 | } 76 | 77 | opts := metav1.ListOptions{ 78 | LabelSelector: n.nodeSelectorLabels.String(), 79 | } 80 | 81 | nodes, err := n.CoreV1().Nodes().List(opts) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | log.Infof("Checking %d nodes for readiness", len(nodes.Items)) 87 | 88 | for _, node := range nodes.Items { 89 | err = n.handleNode(&node) 90 | if err != nil { 91 | log.Error(err) 92 | continue 93 | } 94 | } 95 | 96 | return nil 97 | } 98 | 99 | // Run runs the controller loop until it receives a stop signal over the stop 100 | // channel. 101 | func (n *NodeController) Run(stopChan <-chan struct{}) { 102 | for { 103 | err := n.runOnce() 104 | if err != nil { 105 | log.Error(err) 106 | } 107 | 108 | select { 109 | case <-time.After(n.interval): 110 | case <-stopChan: 111 | log.Info("Terminating main controller loop.") 112 | return 113 | } 114 | } 115 | } 116 | 117 | // handleNode checks if a node is ready and updates the notReady taint 118 | // accordingly. 119 | func (n *NodeController) handleNode(node *v1.Node) error { 120 | ready, err := n.nodeReady(node) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | err = n.setNodeReady(node, ready) 126 | if err != nil { 127 | return err 128 | } 129 | 130 | return nil 131 | } 132 | 133 | // nodeReady checks if the required pods are scheduled on the node and has 134 | // status ready. 135 | func (n *NodeController) nodeReady(node *v1.Node) (bool, error) { 136 | opts := metav1.ListOptions{ 137 | FieldSelector: fmt.Sprintf("spec.nodeName=%s", node.ObjectMeta.Name), 138 | } 139 | 140 | pods, err := n.CoreV1().Pods(v1.NamespaceAll).List(opts) 141 | if err != nil { 142 | return false, err 143 | } 144 | 145 | readyResources := make([]*PodSelector, 0, len(n.selectors)) 146 | for _, identifier := range n.selectors { 147 | for _, pod := range pods.Items { 148 | if pod.ObjectMeta.Namespace == identifier.Namespace && 149 | containLabels(pod.ObjectMeta.Labels, identifier.Labels) { 150 | if podReady(&pod) { 151 | readyResources = append(readyResources, identifier) 152 | } else { 153 | // TODO: find all not ready pods 154 | log.WithFields(log.Fields{ 155 | "pod": pod.Name, 156 | "namespace": pod.Namespace, 157 | "node": node.Name, 158 | }).Warn("Pod not ready.") 159 | } 160 | break 161 | } 162 | } 163 | } 164 | 165 | if len(readyResources) != len(n.selectors) { 166 | return false, nil 167 | } 168 | 169 | return true, nil 170 | } 171 | 172 | // setNodeReady sets node taint macthing ready value. E.g. sets NotReady taint 173 | // if ready is false, and removes the taint (if exists) when ready is true. 174 | func (n *NodeController) setNodeReady(node *v1.Node, ready bool) error { 175 | setNodeReadiness := func() error { 176 | updatedNode, err := n.CoreV1().Nodes().Get(node.Name, metav1.GetOptions{}) 177 | if err != nil { 178 | return backoff.Permanent(err) 179 | } 180 | 181 | // if ready, remove notReady taint if exists on the node 182 | if ready { 183 | var newTaints []v1.Taint 184 | for _, taint := range updatedNode.Spec.Taints { 185 | if taint.Key != n.taintNodeNotReadyName { 186 | newTaints = append(newTaints, taint) 187 | } 188 | } 189 | 190 | if len(newTaints) == len(updatedNode.Spec.Taints) { 191 | return nil 192 | } 193 | updatedNode.Spec.Taints = newTaints 194 | } else { 195 | if hasTaint(updatedNode, n.taintNodeNotReadyName) { 196 | return nil 197 | } 198 | 199 | taint := v1.Taint{ 200 | Key: n.taintNodeNotReadyName, 201 | Effect: v1.TaintEffectNoSchedule, 202 | } 203 | updatedNode.Spec.Taints = append(updatedNode.Spec.Taints, taint) 204 | } 205 | 206 | _, err = n.CoreV1().Nodes().Update(updatedNode) 207 | if err != nil { 208 | // automatically retry if there was a conflicting update. 209 | if errors.IsConflict(err) { 210 | return err 211 | } 212 | return backoff.Permanent(err) 213 | } 214 | 215 | if ready { 216 | log.WithFields(log.Fields{ 217 | "action": "removed", 218 | "taint": n.taintNodeNotReadyName, 219 | "node": updatedNode.ObjectMeta.Name, 220 | }).Info("") 221 | 222 | if n.nodeStartUpObserver != nil { 223 | // observe node startup duration 224 | n.nodeStartUpObserver.ObserveNode(*updatedNode) 225 | } 226 | 227 | // trigger hooks on node ready. 228 | for _, hook := range n.nodeReadyHooks { 229 | err := hook.Trigger(updatedNode.Spec.ProviderID) 230 | if err != nil { 231 | log.Errorf("Failed to trigger hook '%s': %v", hook.Name(), err) 232 | } 233 | } 234 | } else { 235 | log.WithFields(log.Fields{ 236 | "action": "added", 237 | "taint": n.taintNodeNotReadyName, 238 | "node": updatedNode.ObjectMeta.Name, 239 | }).Info("") 240 | } 241 | 242 | return nil 243 | } 244 | 245 | backoffCfg := backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), maxConflictRetries) 246 | return backoff.Retry(setNodeReadiness, backoffCfg) 247 | } 248 | 249 | // getConfig gets a selector config from a config map. 250 | func (n *NodeController) getConfig() error { 251 | configMap, err := n.CoreV1().ConfigMaps(n.namespace).Get(n.configMap, metav1.GetOptions{}) 252 | if err != nil { 253 | return err 254 | } 255 | 256 | data, ok := configMap.Data[ConfigMapSelectorsKey] 257 | if !ok { 258 | return fmt.Errorf("expected key '%s' not present in config map", ConfigMapSelectorsKey) 259 | } 260 | 261 | selectors, err := ReadSelectors(data) 262 | if err != nil { 263 | return err 264 | } 265 | 266 | n.selectors = selectors 267 | return nil 268 | } 269 | 270 | // hasTaint returns true if the node has the taint. 271 | func hasTaint(node *v1.Node, taintName string) bool { 272 | for _, taint := range node.Spec.Taints { 273 | if taint.Key == taintName { 274 | return true 275 | } 276 | } 277 | return false 278 | } 279 | 280 | // containLabels reports whether expectedLabels are in labels. 281 | func containLabels(labels, expectedLabels map[string]string) bool { 282 | for key, val := range expectedLabels { 283 | if v, ok := labels[key]; !ok || v != val { 284 | return false 285 | } 286 | } 287 | return true 288 | } 289 | 290 | // podReady returns true if all containers in the pod are ready. 291 | func podReady(pod *v1.Pod) bool { 292 | for _, containerStatus := range pod.Status.ContainerStatuses { 293 | if !containerStatus.Ready { 294 | return false 295 | } 296 | } 297 | return true 298 | } 299 | -------------------------------------------------------------------------------- /controller_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "k8s.io/api/core/v1" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/client-go/kubernetes" 9 | "k8s.io/client-go/kubernetes/fake" 10 | ) 11 | 12 | const ( 13 | namespace = "default" 14 | taintNodeNotReadyName = "notReady" 15 | ) 16 | 17 | func setupMockKubernetes(t *testing.T, node *v1.Node, config *v1.ConfigMap) kubernetes.Interface { 18 | client := fake.NewSimpleClientset() 19 | 20 | if node != nil { 21 | _, err := client.CoreV1().Nodes().Create(node) 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | } 26 | 27 | if config != nil { 28 | _, err := client.CoreV1().ConfigMaps(namespace).Create(config) 29 | if err != nil { 30 | t.Error(err) 31 | } 32 | } 33 | 34 | pod := &v1.Pod{ 35 | ObjectMeta: metav1.ObjectMeta{ 36 | Namespace: "default", 37 | Name: "foo", 38 | Labels: map[string]string{"foo": "bar"}, 39 | }, 40 | } 41 | 42 | _, err := client.CoreV1().Pods(pod.Namespace).Create(pod) 43 | if err != nil { 44 | t.Error(err) 45 | } 46 | return client 47 | } 48 | 49 | func TestRunOnce(t *testing.T) { 50 | for _, tc := range []struct { 51 | msg string 52 | node *v1.Node 53 | config *v1.ConfigMap 54 | success bool 55 | }{ 56 | { 57 | msg: "runOnce should succeed.", 58 | node: &v1.Node{ 59 | ObjectMeta: metav1.ObjectMeta{ 60 | Name: "foo", 61 | }, 62 | Spec: v1.NodeSpec{ 63 | Taints: []v1.Taint{ 64 | { 65 | Key: taintNodeNotReadyName, 66 | }, 67 | { 68 | Key: "foo", 69 | }, 70 | }, 71 | }, 72 | }, 73 | config: &v1.ConfigMap{ 74 | ObjectMeta: metav1.ObjectMeta{ 75 | Name: "config", 76 | Namespace: namespace, 77 | }, 78 | Data: map[string]string{ConfigMapSelectorsKey: `selectors: 79 | - namespace: kube-system 80 | labels: 81 | foo: bar`}, 82 | }, 83 | success: true, 84 | }, 85 | } { 86 | t.Run(tc.msg, func(t *testing.T) { 87 | controller := &NodeController{ 88 | Interface: setupMockKubernetes(t, tc.node, tc.config), 89 | configMap: "", 90 | namespace: namespace, 91 | } 92 | if tc.config != nil { 93 | controller.configMap = tc.config.Name 94 | } 95 | 96 | err := controller.runOnce() 97 | if err != nil && tc.success { 98 | t.Errorf("should not fail: %s", err) 99 | } 100 | }) 101 | } 102 | } 103 | 104 | func TestRun(t *testing.T) { 105 | stopCh := make(chan struct{}, 1) 106 | node := &v1.Node{ 107 | ObjectMeta: metav1.ObjectMeta{ 108 | Name: "foo", 109 | }, 110 | Spec: v1.NodeSpec{ 111 | Taints: []v1.Taint{ 112 | { 113 | Key: taintNodeNotReadyName, 114 | }, 115 | { 116 | Key: "foo", 117 | }, 118 | }, 119 | }, 120 | } 121 | 122 | config := &v1.ConfigMap{ 123 | ObjectMeta: metav1.ObjectMeta{ 124 | Name: "config", 125 | Namespace: namespace, 126 | }, 127 | Data: map[string]string{ConfigMapSelectorsKey: `selectors: 128 | - namespace: kube-system 129 | labels: 130 | foo: bar`}, 131 | } 132 | 133 | controller := &NodeController{ 134 | Interface: setupMockKubernetes(t, node, config), 135 | configMap: config.Name, 136 | namespace: namespace, 137 | } 138 | 139 | go controller.Run(stopCh) 140 | stopCh <- struct{}{} 141 | } 142 | 143 | func TestNodeReady(t *testing.T) { 144 | for _, tc := range []struct { 145 | msg string 146 | selectors []*PodSelector 147 | ready bool 148 | }{ 149 | { 150 | msg: "node should be ready when pod is found", 151 | selectors: []*PodSelector{ 152 | { 153 | Namespace: "default", 154 | Labels: map[string]string{"foo": "bar"}, 155 | }, 156 | }, 157 | ready: true, 158 | }, 159 | { 160 | msg: "node should not be ready when pod is not found", 161 | selectors: []*PodSelector{ 162 | { 163 | Namespace: "default", 164 | Labels: map[string]string{"foo": "baz"}, 165 | }, 166 | }, 167 | ready: false, 168 | }, 169 | } { 170 | t.Run(tc.msg, func(t *testing.T) { 171 | controller := &NodeController{ 172 | Interface: setupMockKubernetes(t, nil, nil), 173 | selectors: tc.selectors, 174 | } 175 | ready, _ := controller.nodeReady(&v1.Node{}) 176 | 177 | if ready != tc.ready { 178 | t.Errorf("expected ready %t, got %t", tc.ready, ready) 179 | } 180 | }) 181 | } 182 | } 183 | 184 | func TestSetNodeReady(t *testing.T) { 185 | for _, tc := range []struct { 186 | msg string 187 | node *v1.Node 188 | ready bool 189 | }{ 190 | { 191 | msg: "taint should be removed when node is ready", 192 | node: &v1.Node{ 193 | ObjectMeta: metav1.ObjectMeta{ 194 | Name: "foo", 195 | }, 196 | Spec: v1.NodeSpec{ 197 | Taints: []v1.Taint{ 198 | { 199 | Key: taintNodeNotReadyName, 200 | }, 201 | { 202 | Key: "foo", 203 | }, 204 | }, 205 | }, 206 | }, 207 | ready: true, 208 | }, 209 | { 210 | msg: "taint should be added when node is not ready", 211 | node: &v1.Node{ 212 | ObjectMeta: metav1.ObjectMeta{ 213 | Name: "foo", 214 | }, 215 | Spec: v1.NodeSpec{ 216 | Taints: []v1.Taint{ 217 | { 218 | Key: "foo", 219 | }, 220 | }, 221 | }, 222 | }, 223 | ready: false, 224 | }, 225 | } { 226 | t.Run(tc.msg, func(t *testing.T) { 227 | controller := &NodeController{ 228 | Interface: setupMockKubernetes(t, tc.node, nil), 229 | taintNodeNotReadyName: taintNodeNotReadyName, 230 | } 231 | _ = controller.setNodeReady(tc.node, tc.ready) 232 | 233 | n, err := controller.CoreV1().Nodes().Get(tc.node.Name, metav1.GetOptions{}) 234 | if err != nil { 235 | t.Errorf("should not fail: %s", err) 236 | } 237 | 238 | if tc.ready && hasTaint(n, taintNodeNotReadyName) { 239 | t.Errorf("node should not have taint when ready") 240 | } 241 | 242 | if !tc.ready && !hasTaint(n, taintNodeNotReadyName) { 243 | t.Errorf("node should have taint when not ready") 244 | } 245 | }) 246 | } 247 | } 248 | 249 | func TestGetConfig(t *testing.T) { 250 | for _, tc := range []struct { 251 | msg string 252 | config *v1.ConfigMap 253 | success bool 254 | }{ 255 | { 256 | msg: "valid config map should overwrite selectors", 257 | config: &v1.ConfigMap{ 258 | ObjectMeta: metav1.ObjectMeta{ 259 | Name: "config", 260 | Namespace: namespace, 261 | }, 262 | Data: map[string]string{ConfigMapSelectorsKey: `selectors: 263 | - namespace: kube-system 264 | labels: 265 | foo: bar`}, 266 | }, 267 | success: true, 268 | }, 269 | { 270 | msg: "config map with invalid key should fail", 271 | config: &v1.ConfigMap{ 272 | ObjectMeta: metav1.ObjectMeta{ 273 | Name: "config", 274 | Namespace: namespace, 275 | }, 276 | Data: map[string]string{"invalid": `selectors: 277 | - namespace: kube-system 278 | labels: 279 | foo: bar`}, 280 | }, 281 | success: false, 282 | }, 283 | { 284 | msg: "config map with invalid content should fail", 285 | config: &v1.ConfigMap{ 286 | ObjectMeta: metav1.ObjectMeta{ 287 | Name: "config", 288 | Namespace: namespace, 289 | }, 290 | Data: map[string]string{ConfigMapSelectorsKey: `selectors`}, 291 | }, 292 | success: false, 293 | }, 294 | { 295 | msg: "no configMap exists should fail", 296 | config: nil, 297 | success: false, 298 | }, 299 | } { 300 | t.Run(tc.msg, func(t *testing.T) { 301 | controller := &NodeController{ 302 | Interface: setupMockKubernetes(t, nil, tc.config), 303 | configMap: "config", 304 | namespace: namespace, 305 | } 306 | 307 | err := controller.getConfig() 308 | if err != nil && tc.success { 309 | t.Errorf("should not fail: %s", err) 310 | } 311 | 312 | if err == nil && !tc.success { 313 | t.Error("expected failure") 314 | } 315 | 316 | // n, err := controller.CoreV1().Nodes().Get(tc.node.Name, metav1.GetOptions{}) 317 | // if err != nil { 318 | // t.Errorf("should not fail: %s", err) 319 | // } 320 | 321 | // if tc.ready && hasTaint(n) { 322 | // t.Errorf("node should not have taint when ready") 323 | // } 324 | 325 | // if !tc.ready && !hasTaint(n) { 326 | // t.Errorf("node should have taint when not ready") 327 | // } 328 | }) 329 | } 330 | } 331 | 332 | func TestContainLabels(t *testing.T) { 333 | labels := map[string]string{ 334 | "foo": "bar", 335 | } 336 | 337 | expected := map[string]string{ 338 | "foo": "bar", 339 | } 340 | 341 | if !containLabels(labels, expected) { 342 | t.Errorf("expected %s to be contained in %s", expected, labels) 343 | } 344 | 345 | notExpected := map[string]string{ 346 | "foo": "baz", 347 | } 348 | 349 | if containLabels(labels, notExpected) { 350 | t.Errorf("did not expect %s to be contained in %s", notExpected, labels) 351 | } 352 | } 353 | 354 | func TestPodReady(t *testing.T) { 355 | pod := &v1.Pod{ 356 | Status: v1.PodStatus{ 357 | ContainerStatuses: []v1.ContainerStatus{ 358 | { 359 | Ready: true, 360 | }, 361 | }, 362 | }, 363 | } 364 | 365 | if !podReady(pod) { 366 | t.Error("expected pod to be ready") 367 | } 368 | 369 | pod.Status.ContainerStatuses[0].Ready = false 370 | 371 | if podReady(pod) { 372 | t.Error("expected pod to not be ready") 373 | } 374 | } 375 | -------------------------------------------------------------------------------- /docs/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: node-ready-selectors 5 | namespace: kube-system 6 | data: 7 | pod_selectors: 8 | selectors: 9 | - namespace: kube-system 10 | labels: 11 | foo: bar 12 | -------------------------------------------------------------------------------- /docs/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: kube-node-ready-controller 5 | namespace: kube-system 6 | labels: 7 | application: kube-node-ready-controller 8 | version: latest 9 | spec: 10 | strategy: 11 | type: Recreate 12 | selector: 13 | matchLabels: 14 | application: kube-node-ready-controller 15 | template: 16 | metadata: 17 | labels: 18 | application: kube-node-ready-controller 19 | version: latest 20 | spec: 21 | tolerations: 22 | - key: node.alpha.kubernetes.io/notReady-workload 23 | operator: Exists 24 | effect: NoSchedule 25 | containers: 26 | - name: kube-node-ready-controller 27 | image: mikkeloscar/kube-node-ready-controller:latest 28 | args: 29 | # format :=,+ 30 | - "--pod-selector=kube-system:application=skipper-ingress" 31 | - "--pod-selector=kube-system:application=kube2iam" 32 | - "--pod-selector=kube-system:application=kube-proxy" 33 | - "--pod-selector=kube-system:application=logging-agent" 34 | - "--pod-selector=kube-system:application=prometheus-node-exporter" 35 | resources: 36 | limits: 37 | cpu: 20m 38 | memory: 50Mi 39 | requests: 40 | cpu: 20m 41 | memory: 50Mi 42 | -------------------------------------------------------------------------------- /label_selector.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // Labels is a map of labels. 9 | type Labels map[string]string 10 | 11 | func (l Labels) String() string { 12 | labels := make([]string, 0, len(l)) 13 | for k, v := range l { 14 | labels = append(labels, fmt.Sprintf("%s=%s", k, v)) 15 | } 16 | 17 | return strings.Join(labels, ",") 18 | } 19 | 20 | // Set parses a pod selector string and adds it to the list. 21 | func (l Labels) Set(value string) error { 22 | if l == nil { 23 | l = Labels(map[string]string{}) 24 | } 25 | 26 | labelsStrs := strings.Split(value, ",") 27 | for _, labelStr := range labelsStrs { 28 | kv := strings.Split(labelStr, "=") 29 | if len(kv) < 1 || len(kv) > 2 { 30 | return fmt.Errorf("invalid pod selector format") 31 | } 32 | 33 | val := "" 34 | if len(kv) == 2 { 35 | val = kv[1] 36 | } 37 | 38 | l[kv[0]] = val 39 | } 40 | 41 | return nil 42 | } 43 | 44 | // IsCumulative always return true because it's allowed to call Set multiple 45 | // times. 46 | func (l Labels) IsCumulative() bool { 47 | return true 48 | } 49 | -------------------------------------------------------------------------------- /label_selector_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestLabelsString(t *testing.T) { 6 | labels := Labels(map[string]string{ 7 | "master": "true", 8 | }) 9 | expected := "master=true" 10 | 11 | if labels.String() != expected { 12 | t.Errorf("expected %s, got %s", expected, labels.String()) 13 | } 14 | 15 | } 16 | 17 | func TestSetLabelsValue(t *testing.T) { 18 | for _, tc := range []struct { 19 | msg string 20 | value string 21 | valid bool 22 | }{ 23 | { 24 | msg: "test valid labels", 25 | value: "master=true,worker=false", 26 | valid: true, 27 | }, 28 | { 29 | msg: "test invalid labels", 30 | value: "master=true=false", 31 | valid: false, 32 | }, 33 | } { 34 | t.Run(tc.msg, func(t *testing.T) { 35 | var labels Labels 36 | err := labels.Set(tc.value) 37 | if err != nil && tc.valid { 38 | t.Errorf("should not fail: %s", err) 39 | } 40 | 41 | if err == nil && !tc.valid { 42 | t.Error("expected failure") 43 | } 44 | }) 45 | } 46 | } 47 | 48 | func TestLabelsIsCumulative(t *testing.T) { 49 | var labels Labels 50 | if !labels.IsCumulative() { 51 | t.Error("expected IsCumulative = true") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "net/url" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | pkgAWS "github.com/mikkeloscar/kube-node-ready-controller/pkg/aws" 15 | "github.com/prometheus/client_golang/prometheus/promhttp" 16 | log "github.com/sirupsen/logrus" 17 | "gopkg.in/alecthomas/kingpin.v2" 18 | "k8s.io/client-go/kubernetes" 19 | "k8s.io/client-go/rest" 20 | ) 21 | 22 | const ( 23 | defaultInterval = "15s" 24 | defaultMetricsAddress = ":7979" 25 | defaultTaintNodeNotReadyName = "node.alpha.kubernetes.io/notReady-workload" 26 | ) 27 | 28 | var ( 29 | config struct { 30 | Interval time.Duration 31 | MetricsAddress string 32 | PodSelectors PodSelectors 33 | NodeSelectors Labels 34 | ConfigMap string 35 | ASGLifecycleHook string 36 | EnableNodeStartUpMetrics bool 37 | TaintNodeNotReadyName string 38 | APIServer *url.URL 39 | } 40 | ) 41 | 42 | func init() { 43 | kingpin.Flag("interval", "Interval between checks."). 44 | Default(defaultInterval).DurationVar(&config.Interval) 45 | kingpin.Flag("apiserver", "API server url.").URLVar(&config.APIServer) 46 | kingpin.Flag("metrics-address", "defines where to serve metrics"). 47 | Default(defaultMetricsAddress).StringVar(&config.MetricsAddress) 48 | kingpin.Flag("pod-selector", "Pod selector specified by :=,+."). 49 | SetValue(&config.PodSelectors) 50 | kingpin.Flag("node-selector", "Node selector labels =,+."). 51 | SetValue(&config.NodeSelectors) 52 | kingpin.Flag("pod-selector-configmap", "Name of configMap with pod selector definition. Must be in the same namespace."). 53 | StringVar(&config.ConfigMap) 54 | kingpin.Flag("asg-lifecycle-hook", "Name of ASG lifecycle hook to trigger on node Ready."). 55 | StringVar(&config.ASGLifecycleHook) 56 | kingpin.Flag("enable-node-startup-metrics", "Enable node startup duration metrics."). 57 | BoolVar(&config.EnableNodeStartUpMetrics) 58 | kingpin.Flag("not-ready-taint-name", "Name of the taint set for not ready nodes."). 59 | StringVar(&config.TaintNodeNotReadyName) 60 | } 61 | 62 | func main() { 63 | kingpin.Parse() 64 | 65 | var awsSession *session.Session 66 | var err error 67 | if config.ASGLifecycleHook != "" || config.EnableNodeStartUpMetrics { 68 | awsSession, err = pkgAWS.Session(aws.NewConfig()) 69 | if err != nil { 70 | log.Fatalf("Failed to setup aws Session: %v", err) 71 | } 72 | } 73 | 74 | var hooks []Hook 75 | if config.ASGLifecycleHook != "" { 76 | hooks = append(hooks, NewASGLifecycleHook(awsSession, config.ASGLifecycleHook)) 77 | } 78 | 79 | var startupObserver NodeStartUpObserver 80 | if config.EnableNodeStartUpMetrics { 81 | startupObserver, err = NewASGNodeStartUpObserver(awsSession) 82 | if err != nil { 83 | log.Fatalf("Failed to setup observer: %v", err) 84 | } 85 | } 86 | 87 | var kubeConfig *rest.Config 88 | if config.APIServer != nil { 89 | kubeConfig = &rest.Config{ 90 | Host: config.APIServer.String(), 91 | } 92 | } else { 93 | kubeConfig, err = rest.InClusterConfig() 94 | if err != nil { 95 | log.Fatal(err) 96 | } 97 | } 98 | 99 | // set timeouts for kube client 100 | tr := &http.Transport{ 101 | Dial: (&net.Dialer{ 102 | Timeout: 30 * time.Second, 103 | KeepAlive: 30 * time.Second, 104 | }).Dial, 105 | TLSHandshakeTimeout: 10 * time.Second, 106 | ResponseHeaderTimeout: 10 * time.Second, 107 | ExpectContinueTimeout: 1 * time.Second, 108 | } 109 | 110 | stopChan := make(chan struct{}) 111 | // We need this to reliably fade on DNS change, which is right 112 | // now not fixed with IdleConnTimeout in the http.Transport. 113 | // https://github.com/golang/go/issues/23427 114 | go func() { 115 | for { 116 | select { 117 | case <-time.After(30 * time.Second): 118 | tr.CloseIdleConnections() 119 | case <-stopChan: 120 | return 121 | } 122 | } 123 | }() 124 | 125 | kubeConfig.Transport = tr 126 | client, err := kubernetes.NewForConfig(kubeConfig) 127 | if err != nil { 128 | log.Fatal(err) 129 | } 130 | 131 | controller, err := NewNodeController( 132 | client, 133 | config.PodSelectors, 134 | config.NodeSelectors, 135 | config.TaintNodeNotReadyName, 136 | config.Interval, 137 | config.ConfigMap, 138 | hooks, 139 | startupObserver, 140 | ) 141 | if err != nil { 142 | log.Fatal(err) 143 | } 144 | 145 | go handleSigterm(stopChan) 146 | 147 | go serveMetrics(config.MetricsAddress) 148 | 149 | controller.Run(stopChan) 150 | } 151 | 152 | func handleSigterm(stopChan chan struct{}) { 153 | signals := make(chan os.Signal, 1) 154 | signal.Notify(signals, syscall.SIGTERM) 155 | <-signals 156 | log.Info("Received Term signal. Terminating...") 157 | close(stopChan) 158 | } 159 | 160 | func serveMetrics(address string) { 161 | http.Handle("/metrics", promhttp.Handler()) 162 | log.Fatal(http.ListenAndServe(address, nil)) 163 | } 164 | -------------------------------------------------------------------------------- /observer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/ec2" 11 | "github.com/aws/aws-sdk-go/service/ec2/ec2iface" 12 | "github.com/prometheus/client_golang/prometheus" 13 | log "github.com/sirupsen/logrus" 14 | "k8s.io/api/core/v1" 15 | ) 16 | 17 | // NodeStartUpObserver describes an observer which can observe the startup 18 | // time duration of a node. 19 | type NodeStartUpObserver interface { 20 | ObserveNode(node v1.Node) 21 | } 22 | 23 | // ASGNodeStartUpObserver is a node startup duration oberserver which determines 24 | // the statup time duration based on ec2 instance launch time. 25 | type ASGNodeStartUpObserver struct { 26 | ec2Client ec2iface.EC2API 27 | nodesObserved sync.Map 28 | startUpDurationSeconds prometheus.Summary 29 | } 30 | 31 | // NewASGNodeStartUpObserver registers a prometheus summary vec and returns a 32 | // ASGNodeStartUpObserver. 33 | func NewASGNodeStartUpObserver(sess *session.Session) (*ASGNodeStartUpObserver, error) { 34 | startUpDurationSeconds := prometheus.NewSummary( 35 | prometheus.SummaryOpts{ 36 | Name: "startup_duration_seconds", 37 | Help: "The node startup latencies in seconds.", 38 | Subsystem: "node", 39 | Objectives: prometheus.DefObjectives, 40 | }, 41 | ) 42 | 43 | err := prometheus.Register(startUpDurationSeconds) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | return &ASGNodeStartUpObserver{ 49 | ec2Client: ec2.New(sess), 50 | startUpDurationSeconds: startUpDurationSeconds, 51 | }, nil 52 | } 53 | 54 | // ObserveNode observes the node startup time duration based on the launch 55 | // time of the underlying ec2 instance. 56 | // The observation is executed in a go routine to not block the caller. 57 | func (o *ASGNodeStartUpObserver) ObserveNode(node v1.Node) { 58 | go func() { 59 | now := time.Now().UTC() 60 | 61 | if _, ok := o.nodesObserved.Load(node.Name); ok { 62 | log.Infof("Ignoring node %s already observed", node.Name) 63 | return 64 | } 65 | 66 | launchTime, err := o.nodeLaunchTime(node.Spec.ProviderID) 67 | if err != nil { 68 | log.Errorf("Failed to get node launch time: %v", err) 69 | return 70 | } 71 | 72 | o.startUpDurationSeconds.Observe(now.Sub(launchTime).Seconds()) 73 | 74 | // record that node was observed 75 | o.nodesObserved.Store(node.Name, nil) 76 | }() 77 | } 78 | 79 | // nodeLaunchTime get the startup time of the underlying ec2 instance. 80 | func (o *ASGNodeStartUpObserver) nodeLaunchTime(providerID string) (time.Time, error) { 81 | instanceID, err := instanceIDFromProviderID(providerID) 82 | if err != nil { 83 | return time.Time{}, fmt.Errorf("Failed to get instanceID for node: %v", err) 84 | } 85 | 86 | params := &ec2.DescribeInstancesInput{ 87 | InstanceIds: []*string{aws.String(instanceID)}, 88 | } 89 | 90 | resp, err := o.ec2Client.DescribeInstances(params) 91 | if err != nil { 92 | return time.Time{}, fmt.Errorf("Failed to describe instance: %v", err) 93 | } 94 | 95 | if len(resp.Reservations) != 1 { 96 | return time.Time{}, fmt.Errorf("Expected one reservation, got %d", len(resp.Reservations)) 97 | } 98 | 99 | if len(resp.Reservations[0].Instances) != 1 { 100 | return time.Time{}, fmt.Errorf("Expected one instance, got %d", len(resp.Reservations[0].Instances)) 101 | } 102 | 103 | return aws.TimeValue(resp.Reservations[0].Instances[0].LaunchTime), nil 104 | } 105 | -------------------------------------------------------------------------------- /pkg/aws/session.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 6 | "github.com/aws/aws-sdk-go/aws/session" 7 | ) 8 | 9 | // Session sets up an AWS session with the region automatically detected from 10 | // the environment or the ec2 metadata service if running on ec2. 11 | func Session(config *aws.Config) (*session.Session, error) { 12 | sess, err := session.NewSessionWithOptions(session.Options{ 13 | Config: *config, 14 | SharedConfigState: session.SharedConfigEnable, 15 | }) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | if aws.StringValue(sess.Config.Region) == "" { 21 | // try to get region from metadata service 22 | metadata := ec2metadata.New(sess) 23 | region, err := metadata.Region() 24 | if err != nil { 25 | return nil, err 26 | } 27 | sess.Config.Region = aws.String(region) 28 | } 29 | 30 | return sess, nil 31 | } 32 | -------------------------------------------------------------------------------- /pod_selector.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | yaml "gopkg.in/yaml.v2" 8 | ) 9 | 10 | // PodSelector consist of namespace and labels that can identify a Pod. 11 | type PodSelector struct { 12 | Namespace string 13 | Labels map[string]string 14 | } 15 | 16 | // PodSelectors is a list of PodSelector definitions. 17 | type PodSelectors []*PodSelector 18 | 19 | func (p PodSelectors) String() string { 20 | strs := make([]string, len(p)) 21 | for i, t := range p { 22 | labels := make([]string, 0, len(t.Labels)) 23 | for k, v := range t.Labels { 24 | labels = append(labels, fmt.Sprintf("%s=%s", k, v)) 25 | } 26 | strs[i] = fmt.Sprintf("%s:%s", t.Namespace, strings.Join(labels, ",")) 27 | } 28 | 29 | return strings.Join(strs, " - ") 30 | } 31 | 32 | // Set parses a pod selector string and adds it to the list. 33 | func (p *PodSelectors) Set(value string) error { 34 | divide := strings.Split(value, ":") 35 | if len(divide) != 2 { 36 | return fmt.Errorf("invalid pod selector format") 37 | } 38 | 39 | namespace := divide[0] 40 | 41 | labelsStrs := strings.Split(divide[1], ",") 42 | labels := make(map[string]string, len(labelsStrs)) 43 | for _, labelStr := range labelsStrs { 44 | kv := strings.Split(labelStr, "=") 45 | if len(kv) != 2 { 46 | return fmt.Errorf("invalid pod selector format") 47 | } 48 | labels[kv[0]] = kv[1] 49 | } 50 | 51 | *p = append(*p, &PodSelector{Namespace: namespace, Labels: labels}) 52 | 53 | return nil 54 | } 55 | 56 | // IsCumulative always return true because it's allowed to call Set multiple 57 | // times. 58 | func (p PodSelectors) IsCumulative() bool { 59 | return true 60 | } 61 | 62 | type selectors struct { 63 | Selectors []*PodSelector `yaml:"selectors"` 64 | } 65 | 66 | // ReadSelectors reads selectors defined as a yaml in the following format: 67 | // 68 | // selectors: 69 | // - namespace: kube-system 70 | // labels: 71 | // foo: bar 72 | func ReadSelectors(data string) ([]*PodSelector, error) { 73 | var s selectors 74 | err := yaml.Unmarshal([]byte(data), &s) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return s.Selectors, nil 79 | } 80 | -------------------------------------------------------------------------------- /pod_selector_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestPodSelectorString(t *testing.T) { 6 | PodSelectors := PodSelectors([]*PodSelector{ 7 | { 8 | Namespace: "kube-system", 9 | Labels: map[string]string{"key": "value"}, 10 | }, 11 | }) 12 | expected := "kube-system:key=value" 13 | 14 | if PodSelectors.String() != expected { 15 | t.Errorf("expected %s, got %s", expected, PodSelectors.String()) 16 | } 17 | 18 | } 19 | 20 | func TestSetPodSelectorValue(t *testing.T) { 21 | for _, tc := range []struct { 22 | msg string 23 | value string 24 | valid bool 25 | }{ 26 | { 27 | msg: "test valid selector", 28 | value: "kube-system:application=skipper-ingress", 29 | valid: true, 30 | }, 31 | { 32 | msg: "test invalid selector with missing labels", 33 | value: "kube-system", 34 | valid: false, 35 | }, 36 | { 37 | msg: "test invalid selector with invalid label definition", 38 | value: "kube-system:key-value", 39 | valid: false, 40 | }, 41 | } { 42 | t.Run(tc.msg, func(t *testing.T) { 43 | podSelectors := PodSelectors([]*PodSelector{}) 44 | err := podSelectors.Set(tc.value) 45 | if err != nil && tc.valid { 46 | t.Errorf("should not fail: %s", err) 47 | } 48 | 49 | if err == nil && !tc.valid { 50 | t.Error("expected failure") 51 | } 52 | }) 53 | } 54 | } 55 | 56 | func TestPodSelectorIsCumulative(t *testing.T) { 57 | podSelectors := PodSelectors([]*PodSelector{}) 58 | if !podSelectors.IsCumulative() { 59 | t.Error("expected IsCumulative = true") 60 | } 61 | } 62 | 63 | func TestReadSelectors(t *testing.T) { 64 | const data = `selectors: 65 | - namespace: kube-system 66 | labels: 67 | foo: bar` 68 | 69 | selectors, err := ReadSelectors(data) 70 | if err != nil { 71 | t.Errorf("should not fail: %s", err) 72 | } 73 | 74 | if len(selectors) != 1 { 75 | t.Errorf("expected %d selectors, got %d", 1, len(selectors)) 76 | } 77 | 78 | if selectors[0].Namespace != "kube-system" { 79 | t.Errorf("expected namespace '%s', got '%s'", "kube-system", selectors[0].Namespace) 80 | } 81 | 82 | if len(selectors[0].Labels) != 1 { 83 | t.Errorf("expected %d selectors, got %d", 1, len(selectors[0].Labels)) 84 | } 85 | 86 | const invalidData = `selectors: 87 | ` 88 | _, err = ReadSelectors(invalidData) 89 | if err == nil { 90 | t.Errorf("expected error") 91 | } 92 | } 93 | --------------------------------------------------------------------------------