├── LICENSE ├── helm └── openstack-ingress-controller │ ├── Chart.yaml │ ├── README.md │ ├── templates │ ├── NOTES.txt │ ├── configmap.yaml │ ├── deployment.yaml │ └── rbac.yaml │ └── values.yaml ├── installation └── ansible │ ├── external-cloud-provider │ ├── README.md │ ├── reset_k8s.yaml │ ├── roles │ │ ├── docker │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── k8s_cli │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ ├── install_kube_prompt_tasks.yml │ │ │ │ └── main.yml │ │ ├── kube_cloud_controller_manager │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ ├── files │ │ │ │ ├── cloud-controller-manager-role-bindings.yaml │ │ │ │ ├── cloud-controller-manager-roles.yaml │ │ │ │ └── initializer_config_manifest.yaml │ │ │ ├── tasks │ │ │ │ └── main.yml │ │ │ └── templates │ │ │ │ └── openstack-cloud-controller-manager-ds.yaml.j2 │ │ ├── kube_master │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ ├── tasks │ │ │ │ └── main.yml │ │ │ └── templates │ │ │ │ ├── cloud-config.j2 │ │ │ │ └── kubeadm.conf.j2 │ │ └── kube_node │ │ │ ├── defaults │ │ │ └── main.yml │ │ │ ├── tasks │ │ │ └── main.yml │ │ │ └── templates │ │ │ └── cloud-config.j2 │ └── site.yaml │ ├── version_1 │ ├── README.md │ ├── deploy_k8s.yml │ ├── install_docker.yml │ ├── install_k8s_cli.yml │ ├── install_kube_prompt.yml │ └── reset_k8s.yml │ ├── version_2 │ ├── README.md │ ├── reset_k8s.yml │ ├── roles │ │ ├── docker │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ ├── meta │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── k8s_cli │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ ├── install_kube_prompt_tasks.yml │ │ │ │ └── main.yml │ │ ├── kube_master │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ └── main.yml │ │ └── kube_node │ │ │ └── tasks │ │ │ └── main.yml │ └── site.yml │ ├── version_3 │ ├── README.md │ ├── reset_k8s.yaml │ ├── roles │ │ ├── docker │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── k8s_cli │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ └── tasks │ │ │ │ ├── install_kube_prompt_tasks.yml │ │ │ │ └── main.yml │ │ ├── kube_master │ │ │ ├── defaults │ │ │ │ └── main.yml │ │ │ ├── tasks │ │ │ │ └── main.yml │ │ │ └── templates │ │ │ │ ├── cloud-config.j2 │ │ │ │ └── kubeadm.conf.j2 │ │ └── kube_node │ │ │ ├── defaults │ │ │ └── main.yml │ │ │ ├── tasks │ │ │ └── main.yml │ │ │ └── templates │ │ │ ├── cloud-config.j2 │ │ │ └── kubeadm.conf.j2 │ └── site.yaml │ └── version_4 │ ├── README.md │ ├── reset_k8s.yaml │ ├── roles │ ├── cfssl │ │ ├── defaults │ │ │ └── main.yaml │ │ └── tasks │ │ │ └── main.yaml │ ├── docker │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── etcd │ │ ├── defaults │ │ │ └── main.yaml │ │ ├── tasks │ │ │ └── main.yaml │ │ └── templates │ │ │ └── etc │ │ │ └── systemd │ │ │ └── system │ │ │ └── etcd.service.j2 │ ├── k8s_certs │ │ ├── defaults │ │ │ └── main.yaml │ │ ├── files │ │ │ ├── ca-config.json │ │ │ ├── ca-csr.json │ │ │ └── client.json │ │ ├── tasks │ │ │ └── main.yaml │ │ └── templates │ │ │ └── config.json.j2 │ ├── k8s_cli │ │ ├── defaults │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── install_kube_prompt_tasks.yml │ │ │ └── main.yml │ ├── kube_master │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── cloud-config.j2 │ │ │ └── config.yaml.j2 │ └── kube_node │ │ ├── defaults │ │ └── main.yml │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ └── cloud-config.j2 │ └── site.yml └── test ├── test_loadbalancer_type_service.sh └── test_nginx_ingress_controller.sh /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "v1" 2 | name: openstack-ingress-controller 3 | description: A Helm chart for OpenStack ingress controller 4 | version: 0.2.0 5 | home: https://github.com/kubernetes/cloud-provider-openstack 6 | sources: 7 | - https://github.com/kubernetes/cloud-provider-openstack 8 | keywords: 9 | - openstack 10 | - ingress 11 | - octavia 12 | maintainers: 13 | - email: anlin.kong@gmail.com 14 | name: Lingxian Kong -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/README.md: -------------------------------------------------------------------------------- 1 | # How to deploy 2 | 3 | 1. Prepare your customized values file like the following: 4 | 5 | ```shell 6 | cat < myvals.yaml 7 | clusterName: mycluster 8 | openstack: 9 | username: demo 10 | password: password 11 | projectID: b83c654b664042818707291acff230df 12 | authURL: http://10.52.0.6/identity/v3 13 | region: RegionOne 14 | lbSubnetID: 100a129c-8f53-47fa-adf8-9f04849cb00b 15 | lbFipNetwork: a5d4e91c-7982-4b0a-8122-ffb3b38af42a 16 | EOF 17 | ``` 18 | 19 | 2. Install the chart using helm CLI. 20 | 21 | ```shell 22 | helm install --name os-ingress-controller \ 23 | -f myvals.yaml \ 24 | https://github.com/lingxiankong/kubernetes-study/releases/download/v0.2.0/openstack-ingress-controller-0.2.0.tgz 25 | ``` -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | Thank you for installing {{ .Chart.Name }}. 2 | Your release is named {{ .Release.Name }}. 3 | To learn more about the release, try: 4 | 5 | $ helm status {{ .Release.Name }} 6 | $ helm get {{ .Release.Name }} -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ .Values.configmapName }} 5 | namespace: kube-system 6 | data: 7 | openstack-ingress-controller.yaml: | 8 | clusterName: {{ .Values.clusterName }} 9 | openstack: 10 | username: {{ .Values.openstack.username }} 11 | password: {{ .Values.openstack.password }} 12 | project_id: {{ .Values.openstack.projectID }} 13 | auth_url: {{ .Values.openstack.authURL }} 14 | region: {{ .Values.openstack.region }} 15 | octavia: 16 | subnet_id: {{ .Values.openstack.lbSubnetID }} 17 | fip_network: {{ .Values.openstack.lbFipNetwork }} -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: openstack-ingress-controller 5 | namespace: kube-system 6 | labels: 7 | app: {{ .Chart.Name }} 8 | release: {{ .Release.Name }} 9 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 10 | spec: 11 | replicas: 1 12 | selector: 13 | matchLabels: 14 | app: {{ .Chart.Name }} 15 | template: 16 | metadata: 17 | annotations: 18 | scheduler.alpha.kubernetes.io/critical-pod: "" 19 | labels: 20 | app: {{ .Chart.Name }} 21 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 22 | spec: 23 | serviceAccountName: openstack-ingress-controller 24 | containers: 25 | - name: openstack-ingress-controller 26 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 27 | imagePullPolicy: {{ .Values.image.pullPolicy }} 28 | command: 29 | - /bin/octavia-ingress-controller 30 | - --config=/etc/kubernetes/openstack-ingress-controller.yaml 31 | volumeMounts: 32 | - name: ingress-config 33 | mountPath: /etc/kubernetes 34 | volumes: 35 | - name: ingress-config 36 | configMap: 37 | name: {{ .Values.configmapName }} -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: openstack-ingress-controller 6 | namespace: kube-system 7 | # --- 8 | # kind: ClusterRole 9 | # apiVersion: rbac.authorization.k8s.io/v1 10 | # metadata: 11 | # name: openstack-ingress-controller 12 | # rules: 13 | # - apiGroups: [""] 14 | # resources: ["ResourceAll"] 15 | # verbs: ["VerbAll"] 16 | --- 17 | kind: ClusterRoleBinding 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | metadata: 20 | name: openstack-ingress-controller-binding 21 | subjects: 22 | - kind: ServiceAccount 23 | name: openstack-ingress-controller 24 | namespace: kube-system 25 | roleRef: 26 | kind: ClusterRole 27 | name: cluster-admin 28 | apiGroup: rbac.authorization.k8s.io 29 | -------------------------------------------------------------------------------- /helm/openstack-ingress-controller/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for openstack-ingress-controller. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | configmapName: "openstack-ingress-controller" 6 | image: 7 | repository: "k8scloudprovider/octavia-ingress-controller" 8 | tag: "v0.3.0" 9 | pullPolicy: IfNotPresent 10 | clusterName: mycluster 11 | openstack: 12 | username: 13 | password: 14 | projectID: 15 | authURL: 16 | region: RegionOne 17 | lbSubnetID: 18 | lbFipNetwork: -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/README.md: -------------------------------------------------------------------------------- 1 | # Create k8s cluster(>= 1.13) using ansible on top of OpenStack 2 | 3 | This script can only create kubernetes cluster including 1 master and multiple nodes. Kubernetes cluster will use OpenStack as cloud provider and will deploy cloud-provider-openstack as a separate cloud controller manager. 4 | 5 | ## How to run 6 | ### Prepare your local environment 7 | ```shell 8 | mkvirtualenv test_k8s 9 | pip install ansible shade 10 | ``` 11 | 12 | ### Install 13 | Take a look at the variables in `site.yaml` file, you need to define your own as needed or pass those as ansible-playbook vars. Here is an script example that I used in my **devstack** environment. 14 | 15 | ```bash 16 | echo 'alias source_adm="source ~/devstack/openrc admin admin"' >> ~/.bashrc 17 | echo 'alias source_demo="source ~/devstack/openrc demo demo"' >> ~/.bashrc 18 | echo 'alias source_altdemo="source ~/devstack/openrc alt_demo alt_demo"' >> ~/.bashrc 19 | echo 'alias o="openstack"' >> ~/.bashrc 20 | echo 'alias k="kubectl"' >> ~/.bashrc 21 | echo 'alias lb="openstack loadbalancer"' >> ~/.bashrc 22 | echo 'export PYTHONWARNINGS="ignore"' >> ~/.bashrc 23 | sed -i "/alias ll/c alias ll='ls -l'" ~/.bashrc 24 | cat <<'EOF' >> ~/.bashrc 25 | show_openstack_credential() { 26 | if [ "x$OS_USERNAME" == "x" ]; then 27 | echo "" 28 | else 29 | echo "$OS_USERNAME@$OS_REGION_NAME " 30 | fi 31 | } 32 | export PS1='\[\033[1;34m\]\u@\h $(show_openstack_credential)[\w]\[\033[00m\]\n\[\033[01;31m\]$\[\033[00m\] ' 33 | EOF 34 | source ~/.bashrc 35 | sudo -s 36 | 37 | cd ~ 38 | cat << EOF > pre.sh 39 | set -e 40 | pushd ~/devstack 41 | # create a new flavor 42 | source openrc admin admin 43 | openstack flavor create --id 6 --ram 8196 --disk 20 --vcpus 8 --public k8s 44 | # create keypair 45 | source openrc demo demo 46 | ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa 47 | openstack keypair create --public-key ~/.ssh/id_rsa.pub testkey 48 | # set up the default security group rules 49 | openstack security group rule create --proto icmp default 50 | openstack security group rule create --protocol tcp --dst-port 1:65535 default 51 | # register ubuntu image 52 | source openrc admin admin 53 | curl -SOL http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img 54 | glance image-create --name ubuntu-xenial \ 55 | --visibility public \ 56 | --container-format bare \ 57 | --disk-format qcow2 \ 58 | --file xenial-server-cloudimg-amd64-disk1.img 59 | rm -f xenial-server-cloudimg-amd64-disk1.img 60 | # modify quota 61 | openstack quota set --instances 100 --cores 50 --secgroups 100 --secgroup-rules 500 demo 62 | # install and config ansible 63 | sudo apt-add-repository ppa:ansible/ansible -y && sudo apt-get update -y && sudo apt-get install -y ansible qemu-kvm && sudo pip install shade 64 | sed -i '/stdout_callback/c stdout_callback=debug' /etc/ansible/ansible.cfg 65 | # config ssh 66 | cat <> /etc/ssh/ssh_config 67 | StrictHostKeyChecking no 68 | UserKnownHostsFile=/dev/null 69 | END 70 | popd 71 | EOF 72 | bash pre.sh 73 | 74 | cd ~; git clone https://github.com/lingxiankong/kubernetes_study.git 75 | cd ~/kubernetes_study/installation/ansible/external-cloud-provider/ 76 | source_demo 77 | image=$(openstack image list --name ubuntu-xenial -c ID -f value); echo $image 78 | network=$(openstack network list --name private -c ID -f value); echo $network 79 | subnet_id=$(openstack subnet list --network private --name private-subnet -c ID -f value); echo $subnet_id 80 | auth_url=$(export | grep OS_AUTH_URL | awk -F '"' '{print $2}'); echo $auth_url 81 | source_adm 82 | user_id=$(openstack user show demo -c id -f value) 83 | tenant_id=$(openstack project show demo -c id -f value) 84 | source_demo 85 | ansible-playbook site.yaml -e "node_prefix=lingxian rebuild=false flavor=d3 image=$image network=$network key_name=testkey private_key=$HOME/.ssh/id_rsa auth_url=$auth_url user_id=$user_id password=password tenant_id=$tenant_id region=RegionOne subnet_id=$subnet_id bootstrap=true pod_subnet=10.53.0.0/16 ccm_image=lingxiankong/openstack-cloud-controller-manager:1.13.1-rc k8s_version=1.13.2" 86 | ``` 87 | 88 | If anything unexpected happened during the installation, just re-run the command but changing `rebuild=true`. 89 | 90 | ## Clean up 91 | 92 | - Delete the instances 93 | - Disassociate/delete allocated floatingips if needed 94 | - Delete ports after instances deletion, using `neutron port-list -- --device-id ` to get ports attached to the instance. 95 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/reset_k8s.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Drain the node 3 | hosts: k8s_master 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | rebuild: false 10 | 11 | tasks: 12 | - name: Drain the node 13 | shell: kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets 14 | when: rebuild | bool 15 | ignore_errors: True 16 | with_items: "{{ groups['k8s_nodes'] }}" 17 | 18 | - name: Delete node 19 | shell: kubectl delete node {{ item }} 20 | when: rebuild | bool 21 | ignore_errors: True 22 | with_items: "{{ groups['k8s_nodes'] }}" 23 | 24 | - name: Reset k8s on each node 25 | hosts: k8s_hosts 26 | become: true 27 | become_method: sudo 28 | gather_facts: false 29 | 30 | vars: 31 | rebuild: false 32 | 33 | tasks: 34 | - name: Reset 35 | shell: kubeadm reset -f 36 | when: rebuild | bool 37 | ignore_errors: True -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | docker_version: 17.03.2~ce-0~ubuntu-xenial -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # apt-get install -y apt-transport-https ca-certificates curl 3 | - name: Install packages 4 | apt: 5 | name: "{{ packages }}" 6 | state: present 7 | update_cache: yes 8 | vars: 9 | packages: 10 | - apt-transport-https 11 | - ca-certificates 12 | - curl 13 | 14 | # curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 15 | - name: Add Docker’s official GPG key 16 | apt_key: 17 | url: https://download.docker.com/linux/ubuntu/gpg 18 | state: present 19 | 20 | # add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable" 21 | - name: Set up the stable repository 22 | apt_repository: repo='deb https://download.docker.com/linux/ubuntu xenial stable' state=present 23 | 24 | # apt-get update; apt install -y docker-ce=17.03.2~ce-0~ubuntu-xenial 25 | - name: Install docker-ce 26 | apt: 27 | name: docker-ce={{ docker_version }} 28 | state: present 29 | update_cache: yes -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/k8s_cli/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k8s_version: "1.13.2" 3 | kube_prompt_version: "v1.0.5" -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create temporary directory 3 | file: 4 | state: directory 5 | path: ~/kube-prompt 6 | register: dir 7 | 8 | - name: Download kube-prompt 9 | get_url: 10 | url: https://github.com/c-bata/kube-prompt/releases/download/{{ kube_prompt_version }}/kube-prompt_{{ kube_prompt_version }}_linux_amd64.zip 11 | dest: "{{ dir.path }}/kube-prompt.zip" 12 | force: no 13 | 14 | - name: Unarchive kube-prompt 15 | unarchive: 16 | src: "{{ dir.path }}/kube-prompt.zip" 17 | dest: "{{ dir.path }}" 18 | remote_src: yes 19 | creates: "{{ dir.path }}/kube-prompt" 20 | 21 | - name: copy kube-prompt to /usr/bin 22 | copy: 23 | src: "{{ dir.path }}/kube-prompt" 24 | dest: /usr/bin/kube-prompt 25 | remote_src: yes 26 | force: no 27 | mode: 0755 28 | 29 | - name: create alias for kube-prompt 30 | lineinfile: 31 | path: /etc/bash.bashrc 32 | create: true 33 | line: 'alias kprompt="/usr/bin/kube-prompt"' 34 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/k8s_cli/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - 3 | - name: Add Kubernetes GPG key 4 | apt_key: 5 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 6 | state: present 7 | 8 | # add-apt-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main" 9 | - name: Set up Kubernetes repository 10 | apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present 11 | 12 | # apt update; apt install -y kubelet=1.11.1-00 kubectl=1.11.1-00 kubeadm=1.11.1-00 13 | - name: Install kube CLIs 14 | apt: 15 | name: "{{ packages }}" 16 | state: present 17 | update_cache: yes 18 | vars: 19 | packages: 20 | - kubelet={{ k8s_version }}-00 21 | - kubectl={{ k8s_version }}-00 22 | - kubeadm={{ k8s_version }}-00 23 | 24 | - import_tasks: install_kube_prompt_tasks.yml 25 | 26 | - name: create alias for cli 27 | lineinfile: 28 | path: /etc/bash.bashrc 29 | create: true 30 | line: 'alias k="/usr/bin/kubectl"' 31 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_cloud_controller_manager/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ccm_image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:latest" 3 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_cloud_controller_manager/files/cloud-controller-manager-role-bindings.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | items: 3 | - apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRoleBinding 5 | metadata: 6 | name: system:cloud-node-controller 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: ClusterRole 10 | name: system:cloud-node-controller 11 | subjects: 12 | - kind: ServiceAccount 13 | name: cloud-node-controller 14 | namespace: kube-system 15 | - apiVersion: rbac.authorization.k8s.io/v1 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: system:pvl-controller 19 | roleRef: 20 | apiGroup: rbac.authorization.k8s.io 21 | kind: ClusterRole 22 | name: system:pvl-controller 23 | subjects: 24 | - kind: ServiceAccount 25 | name: pvl-controller 26 | namespace: kube-system 27 | - apiVersion: rbac.authorization.k8s.io/v1 28 | kind: ClusterRoleBinding 29 | metadata: 30 | name: system:cloud-controller-manager 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: system:cloud-controller-manager 35 | subjects: 36 | - kind: ServiceAccount 37 | name: cloud-controller-manager 38 | namespace: kube-system 39 | kind: List 40 | metadata: {} -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_cloud_controller_manager/files/cloud-controller-manager-roles.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | items: 3 | - apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | name: system:cloud-controller-manager 7 | rules: 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - events 12 | verbs: 13 | - create 14 | - patch 15 | - update 16 | - apiGroups: 17 | - "" 18 | resources: 19 | - nodes 20 | verbs: 21 | - '*' 22 | - apiGroups: 23 | - "" 24 | resources: 25 | - nodes/status 26 | verbs: 27 | - patch 28 | - apiGroups: 29 | - "" 30 | resources: 31 | - services 32 | verbs: 33 | - list 34 | - patch 35 | - update 36 | - watch 37 | - apiGroups: 38 | - "" 39 | resources: 40 | - serviceaccounts 41 | verbs: 42 | - create 43 | - get 44 | - apiGroups: 45 | - "" 46 | resources: 47 | - persistentvolumes 48 | verbs: 49 | - '*' 50 | - apiGroups: 51 | - "" 52 | resources: 53 | - endpoints 54 | verbs: 55 | - create 56 | - get 57 | - list 58 | - watch 59 | - update 60 | - apiGroups: 61 | - "" 62 | resources: 63 | - configmaps 64 | verbs: 65 | - get 66 | - list 67 | - watch 68 | - apiGroups: 69 | - "" 70 | resources: 71 | - secrets 72 | verbs: 73 | - list 74 | - get 75 | - apiVersion: rbac.authorization.k8s.io/v1 76 | kind: ClusterRole 77 | metadata: 78 | name: system:cloud-node-controller 79 | rules: 80 | - apiGroups: 81 | - "" 82 | resources: 83 | - nodes 84 | verbs: 85 | - delete 86 | - get 87 | - patch 88 | - update 89 | - list 90 | - apiGroups: 91 | - "" 92 | resources: 93 | - nodes/status 94 | verbs: 95 | - patch 96 | - apiGroups: 97 | - "" 98 | resources: 99 | - events 100 | verbs: 101 | - create 102 | - patch 103 | - update 104 | - apiVersion: rbac.authorization.k8s.io/v1 105 | kind: ClusterRole 106 | metadata: 107 | name: system:pvl-controller 108 | rules: 109 | - apiGroups: 110 | - "" 111 | resources: 112 | - persistentvolumes 113 | verbs: 114 | - '*' 115 | - apiGroups: 116 | - "" 117 | resources: 118 | - events 119 | verbs: 120 | - create 121 | - patch 122 | - update 123 | kind: List 124 | metadata: {} 125 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_cloud_controller_manager/files/initializer_config_manifest.yaml: -------------------------------------------------------------------------------- 1 | kind: InitializerConfiguration 2 | apiVersion: admissionregistration.k8s.io/v1alpha1 3 | metadata: 4 | name: pvlabel.kubernetes.io 5 | initializers: 6 | - name: pvlabel.kubernetes.io 7 | rules: 8 | - apiGroups: 9 | - "" 10 | apiVersions: 11 | - "*" 12 | resources: 13 | - persistentvolumes -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_cloud_controller_manager/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create cloud-config configmap for ccm 3 | shell: kubectl create configmap cloud-config --from-file=/etc/kubernetes/cloud-config -n kube-system 4 | ignore_errors: True 5 | 6 | - name: Create CCM manifests directory 7 | file: path={{ ansible_env.HOME }}/openstack-cloud-controller-manager state=directory mode=0755 8 | 9 | - name: Create CCM manifest file 10 | template: 11 | src: openstack-cloud-controller-manager-ds.yaml.j2 12 | dest: "{{ ansible_env.HOME }}/openstack-cloud-controller-manager/openstack-cloud-controller-manager-ds.yaml" 13 | 14 | - name: Create CCM role manifest file 15 | copy: 16 | src: cloud-controller-manager-roles.yaml 17 | dest: "{{ ansible_env.HOME }}/openstack-cloud-controller-manager/cloud-controller-manager-roles.yaml" 18 | 19 | - name: Create CCM rolebinding manifest file 20 | copy: 21 | src: cloud-controller-manager-role-bindings.yaml 22 | dest: "{{ ansible_env.HOME }}/openstack-cloud-controller-manager/cloud-controller-manager-role-bindings.yaml" 23 | 24 | - name: Create InitializerConfiguration manifest file 25 | copy: 26 | src: initializer_config_manifest.yaml 27 | dest: "{{ ansible_env.HOME }}/openstack-cloud-controller-manager/initializer_config_manifest.yaml" 28 | 29 | - name: Install cloud-controller-manager 30 | shell: kubectl apply -f {{ ansible_env.HOME }}/openstack-cloud-controller-manager/ 31 | 32 | - name: Download cinder-csi-plugin manifest zip 33 | get_url: 34 | url: https://github.com/kubernetes/cloud-provider-openstack/archive/master.zip 35 | dest: "{{ ansible_env.HOME }}/cloud-provider-openstack.zip" 36 | 37 | - name: Unzip manifests 38 | shell: unzip -qq -u {{ ansible_env.HOME }}/cloud-provider-openstack.zip 'cloud-provider-openstack-master/manifests/cinder-csi-plugin/*' -d {{ ansible_env.HOME }} 39 | 40 | - name: Encode /etc/kubernetes/cloud-config and create cloud-config secret for csi 41 | shell: kubectl create secret -n kube-system generic cloud-config --from-literal=cloud.conf="$(cat /etc/kubernetes/cloud-config)" --dry-run -o yaml > {{ ansible_env.HOME }}/cloud-provider-openstack-master/manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml 42 | 43 | - name: Install cinder-csi-plugin 44 | shell: kubectl apply -f {{ ansible_env.HOME }}/cloud-provider-openstack-master/manifests/cinder-csi-plugin/ 45 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_cloud_controller_manager/templates/openstack-cloud-controller-manager-ds.yaml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: cloud-controller-manager 6 | namespace: kube-system 7 | --- 8 | apiVersion: extensions/v1beta1 9 | kind: DaemonSet 10 | metadata: 11 | name: openstack-cloud-controller-manager 12 | namespace: kube-system 13 | labels: 14 | k8s-app: openstack-cloud-controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | k8s-app: openstack-cloud-controller-manager 19 | updateStrategy: 20 | type: RollingUpdate 21 | template: 22 | metadata: 23 | labels: 24 | k8s-app: openstack-cloud-controller-manager 25 | spec: 26 | nodeSelector: 27 | node-role.kubernetes.io/master: "" 28 | securityContext: 29 | runAsUser: 1001 30 | tolerations: 31 | - key: node.cloudprovider.kubernetes.io/uninitialized 32 | value: "true" 33 | effect: NoSchedule 34 | - key: node-role.kubernetes.io/master 35 | effect: NoSchedule 36 | serviceAccountName: cloud-controller-manager 37 | containers: 38 | - name: openstack-cloud-controller-manager 39 | image: {{ ccm_image }} 40 | args: 41 | - /bin/openstack-cloud-controller-manager 42 | - --v=1 43 | - --cloud-config=/etc/cloud/cloud-config 44 | - --cloud-provider=openstack 45 | - --use-service-account-credentials=true 46 | - --address=127.0.0.1 47 | volumeMounts: 48 | - mountPath: /etc/kubernetes/pki 49 | name: k8s-certs 50 | readOnly: true 51 | - mountPath: /etc/ssl/certs 52 | name: ca-certs 53 | readOnly: true 54 | - mountPath: /etc/cloud 55 | name: cloud-config-volume 56 | readOnly: true 57 | - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec 58 | name: flexvolume-dir 59 | resources: 60 | requests: 61 | cpu: 200m 62 | hostNetwork: true 63 | volumes: 64 | - hostPath: 65 | path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec 66 | type: DirectoryOrCreate 67 | name: flexvolume-dir 68 | - hostPath: 69 | path: /etc/kubernetes/pki 70 | type: DirectoryOrCreate 71 | name: k8s-certs 72 | - hostPath: 73 | path: /etc/ssl/certs 74 | type: DirectoryOrCreate 75 | name: ca-certs 76 | - name: cloud-config-volume 77 | configMap: 78 | name: cloud-config -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k8s_version: "1.13.2" 3 | pod_subnet: "192.168.0.0/16" 4 | 5 | auth_url: http://192.168.121.15/identity/v3/ 6 | user_id: 3fd32295751a41aca601021eba681a29 7 | password: password 8 | tenant_id: 84ce53873abe4d22a6f2491879e76048 9 | region: RegionOne 10 | subnet_id: 1740cd57-62ab-4f7f-a853-2038561060d1 11 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare kubeadm.conf 3 | template: 4 | src: kubeadm.conf.j2 5 | dest: "{{ ansible_env.HOME }}/kubeadm.conf" 6 | 7 | - name: Make sure /etc/kubernetes exists 8 | file: 9 | path: /etc/kubernetes 10 | state: directory 11 | 12 | - name: Prepare openstack config file 13 | template: 14 | src: cloud-config.j2 15 | dest: /etc/kubernetes/cloud-config 16 | 17 | - name: Check if kubelet config file needs change 18 | shell: grep -q "external" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 19 | register: check_kubelet 20 | ignore_errors: True 21 | 22 | - name: Modify kubelet arguments 23 | replace: 24 | path: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 25 | regexp: '(.*)KUBELET_KUBECONFIG_ARGS=(.*)$' 26 | replace: '\1KUBELET_KUBECONFIG_ARGS=--cloud-provider=external \2' 27 | when: check_kubelet.rc != 0 28 | 29 | - name: restart kubelet service 30 | systemd: 31 | state: restarted 32 | daemon_reload: yes 33 | name: kubelet 34 | 35 | - name: Init master 36 | shell: kubeadm init --config {{ ansible_env.HOME }}/kubeadm.conf 37 | register: kubeadm_init 38 | 39 | - fail: 40 | msg: "kubeadm init command failed." 41 | when: kubeadm_init.stdout.find ("kubeadm join") == -1 42 | 43 | - name: Store kubeadm join string 44 | set_fact: 45 | join_str: "{{ kubeadm_init.stdout | regex_search('kubeadm join(.*)$') }}" 46 | 47 | - name: Prepare kube config 48 | shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config 49 | loop: 50 | - "{{ ansible_env.HOME }}" 51 | - /home/ubuntu 52 | 53 | - name: Allow pod on master 54 | shell: kubectl taint nodes --all node-role.kubernetes.io/master- 55 | 56 | - name: Install Calico RBAC 57 | shell: kubectl apply -f {{ item }} 58 | loop: 59 | - https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml 60 | 61 | - name: Download calico manifest file if needed 62 | get_url: 63 | url: https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml 64 | dest: "{{ ansible_env.HOME }}/calico.yaml" 65 | force: no 66 | 67 | - name: Modify the pod cidr in calico manifest file 68 | replace: 69 | path: "{{ ansible_env.HOME }}/calico.yaml" 70 | regexp: '(^.*value: )"192.168.*' 71 | replace: '\1"{{ pod_subnet }}"' 72 | 73 | - name: Install Calico 74 | shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml 75 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_master/templates/cloud-config.j2: -------------------------------------------------------------------------------- 1 | [Global] 2 | auth-url={{ auth_url }} 3 | user-id={{ user_id }} 4 | password={{ password }} 5 | tenant-id={{ tenant_id }} 6 | region={{ region }} 7 | [LoadBalancer] 8 | use-octavia=yes 9 | subnet-id={{ subnet_id }} 10 | create-monitor=no 11 | [BlockStorage] 12 | bs-version=v2 13 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_master/templates/kubeadm.conf.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubeadm.k8s.io/v1beta1 3 | kind: ClusterConfiguration 4 | kubernetesVersion: v{{ k8s_version }} 5 | networking: 6 | podSubnet: {{ pod_subnet }} 7 | unifiedControlPlaneImage: "gcr.io/google_containers/hyperkube-amd64:v{{ k8s_version }}" 8 | apiServer: 9 | certSANs: 10 | - {{ ansible_host }} 11 | extraArgs: 12 | cloud-provider: "external" 13 | enable-admission-plugins: NodeRestriction,Initializers 14 | runtime-config: admissionregistration.k8s.io/v1alpha1=true 15 | controllerManager: 16 | extraArgs: 17 | cloud-provider: "external" 18 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_node/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | auth_url: http://192.168.121.15/identity/v3/ 3 | user_id: 3fd32295751a41aca601021eba681a29 4 | password: password 5 | tenant_id: 84ce53873abe4d22a6f2491879e76048 6 | region: RegionOne 7 | subnet_id: 1740cd57-62ab-4f7f-a853-2038561060d1 8 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Make sure /etc/kubernetes exists 3 | file: 4 | path: /etc/kubernetes 5 | state: directory 6 | 7 | - name: Prepare openstack config file 8 | template: 9 | src: cloud-config.j2 10 | dest: /etc/kubernetes/cloud-config 11 | 12 | - name: Check if kubelet config file needs change 13 | shell: grep -q "external" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 14 | register: check_kubelet 15 | ignore_errors: True 16 | 17 | - name: Modify kubelet arguments 18 | replace: 19 | path: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 20 | regexp: '(.*)KUBELET_KUBECONFIG_ARGS=(.*)$' 21 | replace: '\1KUBELET_KUBECONFIG_ARGS=--cloud-provider=external \2' 22 | when: check_kubelet.rc != 0 23 | 24 | - name: restart kubelet service 25 | systemd: 26 | state: restarted 27 | daemon_reload: yes 28 | name: kubelet 29 | 30 | - name: Join to cluster 31 | shell: "{{ hostvars[groups['k8s_master'][0]]['join_str'] }}" 32 | when: bootstrap | bool 33 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/roles/kube_node/templates/cloud-config.j2: -------------------------------------------------------------------------------- 1 | [Global] 2 | auth-url={{ auth_url }} 3 | user-id={{ user_id }} 4 | password={{ password }} 5 | tenant-id={{ tenant_id }} 6 | region={{ region }} 7 | [LoadBalancer] 8 | use-octavia=yes 9 | subnet-id={{ subnet_id }} 10 | create-monitor=no 11 | [BlockStorage] 12 | bs-version=v2 13 | -------------------------------------------------------------------------------- /installation/ansible/external-cloud-provider/site.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | --- 3 | - name: Deploy and prepare instances in OpenStack 4 | hosts: localhost 5 | 6 | vars: 7 | image: 9136e86b-14d9-4127-b3dd-f1cdb18b0dc6 8 | network: network1 9 | public_network: public 10 | key_name: lingxian_pc 11 | private_key: /home/vagrant/.ssh/id_rsa 12 | flavor: 2ba44887-5fe3-44cc-8bca-e2b60a206a66 13 | boot_from_vol: false 14 | security_groups: 15 | - default 16 | node_prefix: lingxiantest 17 | # Mater hostname must contain 'master', node hostname contains 'node' 18 | # Make sure the vm name complies with k8s naming convention, should not 19 | # contain '_' 20 | node_names: 21 | - master 22 | - node1 23 | pod_subnet: 192.168.0.0/16 24 | 25 | tasks: 26 | # Create ports separately in order to setup allowed_address_pairs 27 | - name: Create ports 28 | os_port: 29 | state: present 30 | name: "{{ node_prefix }}-k8s-{{ item }}" 31 | network: "{{ network }}" 32 | security_groups: "{{ security_groups }}" 33 | allowed_address_pairs: 34 | - ip_address: "{{ pod_subnet }}" 35 | with_items: "{{ node_names }}" 36 | 37 | - name: Create VMs 38 | os_server: 39 | state: present 40 | name: "{{ node_prefix }}-k8s-{{ item }}" 41 | boot_from_volume: "{{ boot_from_vol | bool }}" 42 | image: "{{ image }}" 43 | volume_size: 15 44 | key_name: "{{ key_name }}" 45 | nics: port-name={{ node_prefix }}-k8s-{{ item }} 46 | flavor: "{{ flavor }}" 47 | terminate_volume: true 48 | # Do not allocate floating ip 49 | auto_ip: false 50 | with_items: "{{ node_names }}" 51 | 52 | - name: Associate floating ip if needed 53 | os_floating_ip: 54 | state: present 55 | reuse: yes 56 | network: "{{ public_network }}" 57 | server: "{{ node_prefix }}-k8s-{{ item }}" 58 | with_items: "{{ node_names }}" 59 | when: public_network is defined 60 | 61 | - name: Gather facts about vms 62 | os_server_facts: 63 | server: "{{ node_prefix }}-k8s-*" 64 | 65 | - name: Wait for ssh ready 66 | wait_for: port=22 host="{{ item.private_v4 if public_network is undefined else item.public_v4 }}" search_regex=OpenSSH timeout=300 delay=5 67 | with_items: "{{ openstack_servers }}" 68 | 69 | - name: Add VMs to inventory 70 | add_host: 71 | name: "{{ item.name }}" 72 | groups: k8s_hosts 73 | ansible_user: ubuntu 74 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 75 | ansible_ssh_private_key_file: "{{ private_key }}" 76 | ansible_ssh_common_args: "-o StrictHostKeyChecking=no" 77 | with_items: "{{ openstack_servers }}" 78 | 79 | - name: Setup master group 80 | add_host: 81 | name: "{{ item.name }}" 82 | groups: k8s_master 83 | ansible_user: ubuntu 84 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 85 | ansible_ssh_private_key_file: "{{ private_key }}" 86 | when: item.name | regex_search('(master)') 87 | with_items: "{{ openstack_servers }}" 88 | 89 | - name: Setup nodes group 90 | add_host: 91 | name: "{{ item.name }}" 92 | groups: k8s_nodes 93 | ansible_user: ubuntu 94 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 95 | ansible_ssh_private_key_file: "{{ private_key }}" 96 | when: item.name | regex_search('(node)') 97 | with_items: "{{ openstack_servers }}" 98 | 99 | - name: Prepare k8s hosts 100 | hosts: k8s_hosts 101 | become: true 102 | become_method: sudo 103 | gather_facts: false 104 | 105 | pre_tasks: 106 | - name: Wait for cloud init to finish 107 | raw: cloud-init status --wait 108 | 109 | - name: Install python 2 110 | raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) 111 | 112 | - name: Install pip 113 | shell: which pip || (curl https://bootstrap.pypa.io/get-pip.py | python - && rm -f get-pip.py) 114 | 115 | - name: Pin pip to 9.0.3 116 | shell: pip install pip==9.0.3 117 | 118 | - name: Gather facts after python installation 119 | setup: filter=ansible_* 120 | 121 | - name: Install packages 122 | apt: 123 | name: "{{ packages }}" 124 | state: present 125 | update_cache: yes 126 | vars: 127 | packages: 128 | - ansible 129 | - unzip 130 | - build-essential 131 | - python-dev 132 | - python-setuptools 133 | - libffi-dev 134 | - libxslt1-dev 135 | - libxml2-dev 136 | - libyaml-dev 137 | - libssl-dev 138 | - zlib1g-dev 139 | - jq 140 | 141 | - name: Install pypi libraries 142 | pip: 143 | name: "{{ libraries }}" 144 | ignore_errors: True 145 | vars: 146 | libraries: 147 | - python-openstackclient 148 | - httpie 149 | 150 | - name: Build hosts file 151 | lineinfile: dest=/etc/hosts line="{{ ansible_host }} {{ ansible_hostname }}" state=present 152 | 153 | roles: 154 | - docker 155 | - k8s_cli 156 | 157 | - import_playbook: reset_k8s.yaml rebuild={{ rebuild }} 158 | 159 | - name: Set up k8s master 160 | hosts: k8s_master 161 | become: true 162 | become_method: sudo 163 | gather_facts: true 164 | 165 | roles: 166 | - kube_master 167 | 168 | - name: Set up k8s node 169 | hosts: k8s_nodes 170 | become: true 171 | become_method: sudo 172 | gather_facts: false 173 | 174 | roles: 175 | - kube_node 176 | 177 | - name: Set up cloud-controller-manager and cinder-csi-plugin 178 | hosts: k8s_master 179 | become: true 180 | become_method: sudo 181 | gather_facts: true 182 | 183 | roles: 184 | - kube_cloud_controller_manager 185 | -------------------------------------------------------------------------------- /installation/ansible/version_1/README.md: -------------------------------------------------------------------------------- 1 | # Create k8s cluster using Ansible on top of OpenStack 2 | 3 | This script can only create kubernetes cluster including 1 master and multiple 4 | nodes. 5 | 6 | ## How to run 7 | First, you need to download all the ansible scripts here, then do the 8 | following: 9 | 10 | ```shell 11 | mkvirtualenv test_k8s 12 | pip install ansible shade 13 | source 14 | ``` 15 | 16 | Take a look at the variables in `deploy_k8s.yml` file, you need to define your 17 | own as needed or pass those as ansible-playbook vars. 18 | 19 | ```shell 20 | ansible-playbook deploy_k8s.yml -e "rebuild=false" 21 | ``` 22 | 23 | If anything unexpected happened during the installation, just re-run using: 24 | ```shell 25 | ansible-playbook deploy_k8s.yml -e "rebuild=true" 26 | ``` 27 | 28 | ## Destroy 29 | 30 | - Delete the instances 31 | - Delete ports after instances deletion, using `neutron port-list -- --device-id ` to get port attached to the instance. 32 | -------------------------------------------------------------------------------- /installation/ansible/version_1/deploy_k8s.yml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | --- 3 | - name: Deploy and prepare instances in OpenStack 4 | hosts: localhost 5 | 6 | vars: 7 | image: 9136e86b-14d9-4127-b3dd-f1cdb18b0dc6 8 | network: network1 9 | key_name: lingxian_pc 10 | flavor: 5ff0b09b-684c-4212-8edc-826f26f9ab78 11 | security_groups: 12 | - default 13 | node_prefix: lingxian 14 | node_names: 15 | - master 16 | - node1 17 | - node2 18 | 19 | tasks: 20 | - name: Create ports 21 | os_port: 22 | state: present 23 | name: "{{ node_prefix }}-k8s-{{ item }}" 24 | network: "{{ network }}" 25 | security_groups: "{{ security_groups }}" 26 | allowed_address_pairs: 27 | - ip_address: 192.168.0.0/16 28 | with_items: "{{ node_names }}" 29 | 30 | - name: Create VMs 31 | os_server: 32 | state: present 33 | name: "{{ node_prefix }}-k8s-{{ item }}" 34 | boot_from_volume: true 35 | image: "{{ image }}" 36 | volume_size: 15 37 | key_name: "{{ key_name }}" 38 | nics: port-name={{ node_prefix }}-k8s-{{ item }} 39 | flavor: "{{ flavor }}" 40 | terminate_volume: true 41 | auto_ip: false 42 | with_items: "{{ node_names }}" 43 | 44 | - name: Gather facts about vms 45 | os_server_facts: 46 | server: "{{ node_prefix }}-k8s-*" 47 | 48 | - name: Wait for ssh ready 49 | wait_for: port=22 host="{{ item.private_v4 }}" search_regex=OpenSSH timeout=300 delay=5 50 | with_items: "{{ openstack_servers }}" 51 | 52 | - name: Add VMs to inventory 53 | add_host: 54 | name: "{{ item.name }}" 55 | groups: k8s_hosts 56 | ansible_user: ubuntu 57 | ansible_host: "{{ item.private_v4 }}" 58 | with_items: "{{ openstack_servers }}" 59 | 60 | - name: Setup master group 61 | add_host: 62 | name: "{{ item.name }}" 63 | groups: k8s_master 64 | ansible_user: ubuntu 65 | ansible_host: "{{ item.private_v4 }}" 66 | when: item.name | regex_search('(master)') 67 | with_items: "{{ openstack_servers }}" 68 | 69 | - name: Setup nodes group 70 | add_host: 71 | name: "{{ item.name }}" 72 | groups: k8s_nodes 73 | ansible_user: ubuntu 74 | ansible_host: "{{ item.private_v4 }}" 75 | when: item.name | regex_search('(node)') 76 | with_items: "{{ openstack_servers }}" 77 | 78 | - name: Prepare k8s hosts 79 | hosts: k8s_hosts 80 | become: true 81 | become_method: sudo 82 | gather_facts: true 83 | 84 | tasks: 85 | - name: Install packages 86 | apt: name={{ item }} state=latest update_cache=yes 87 | with_items: 88 | - python-minimal 89 | - ansible 90 | - iperf3 91 | - unzip 92 | 93 | - name: Build hosts file 94 | lineinfile: dest=/etc/hosts line="{{ ansible_host }} {{ ansible_hostname }}" state=present 95 | 96 | - import_playbook: install_docker.yml rebuild={{ rebuild }} 97 | 98 | - import_playbook: reset_k8s.yml rebuild={{ rebuild }} 99 | 100 | - import_playbook: install_k8s_cli.yml rebuild={{ rebuild }} 101 | 102 | - import_playbook: install_kube_prompt.yml 103 | 104 | - name: Set up k8s master 105 | hosts: k8s_master 106 | become: true 107 | become_method: sudo 108 | gather_facts: true 109 | 110 | tasks: 111 | - name: Init master 112 | shell: kubeadm init --pod-network-cidr=192.168.0.0/16 113 | register: kubeadm_init 114 | 115 | - fail: 116 | msg: "kubeadm init command failed." 117 | when: kubeadm_init.stdout.find ("kubeadm join") == -1 118 | 119 | - name: Store kubeadm join string 120 | set_fact: 121 | join_str: "{{ kubeadm_init.stdout | regex_search('kubeadm join(.*)$') }}" 122 | 123 | - name: Prepare kube config 124 | shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config 125 | with_items: 126 | - "{{ ansible_env.HOME }}" 127 | - /home/ubuntu 128 | 129 | - name: Allow pod on master 130 | shell: kubectl taint nodes --all node-role.kubernetes.io/master- 131 | 132 | - name: Download calico yaml 133 | get_url: 134 | url: https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml 135 | dest: "{{ ansible_env.HOME }}/calico.yaml" 136 | 137 | - name: Change calico config 138 | replace: 139 | path: "{{ ansible_env.HOME }}/calico.yaml" 140 | regexp: 'value: "always"' 141 | replace: 'value: "off"\n - name: CALICO_IPV4POOL_NAT_OUTGOING\n value: "true"' 142 | backup: yes 143 | 144 | - name: Install network plugin 145 | shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml 146 | 147 | - name: Set up k8s node 148 | hosts: k8s_nodes 149 | become: true 150 | become_method: sudo 151 | gather_facts: false 152 | 153 | tasks: 154 | - name: Join to cluster 155 | shell: "{{ hostvars[groups['k8s_master'][0]]['join_str'] }}" -------------------------------------------------------------------------------- /installation/ansible/version_1/install_docker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install docker 3 | hosts: k8s_hosts 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | docker_version: 17.03.2~ce-0~ubuntu-xenial 10 | rebuild: false 11 | 12 | tasks: 13 | - name: Install packages 14 | apt: name={{ item }} state=latest update_cache=yes 15 | with_items: 16 | - apt-transport-https 17 | - ca-certificates 18 | - curl 19 | when: not rebuild | bool 20 | 21 | - name: Add Docker’s official GPG key 22 | apt_key: 23 | url: https://download.docker.com/linux/ubuntu/gpg 24 | state: present 25 | 26 | - name: Set up the stable repository 27 | apt_repository: repo='deb https://download.docker.com/linux/ubuntu xenial stable' state=present 28 | 29 | - name: Install docker-ce 30 | apt: 31 | name: docker-ce={{ docker_version }} 32 | state: present 33 | update_cache: yes 34 | when: not rebuild | bool -------------------------------------------------------------------------------- /installation/ansible/version_1/install_k8s_cli.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install kube CLIs 3 | hosts: k8s_hosts 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | rebuild: false 10 | 11 | tasks: 12 | - name: Add Kubernetes GPG key 13 | apt_key: 14 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 15 | state: present 16 | 17 | - name: Set up Kubernetes repository 18 | apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present 19 | 20 | - name: Install kube CLIs 21 | apt: 22 | name: "{{ item }}" 23 | state: latest 24 | update_cache: yes 25 | with_items: 26 | - kubelet 27 | - kubeadm 28 | - kubectl 29 | when: not rebuild | bool -------------------------------------------------------------------------------- /installation/ansible/version_1/install_kube_prompt.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install kube-prompt 3 | hosts: k8s_master 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | tasks: 9 | - name: Create temporary directory 10 | file: 11 | state: directory 12 | path: ~/kube-prompt 13 | register: dir 14 | 15 | - name: Download kube-prompt 16 | get_url: 17 | url: https://github.com/c-bata/kube-prompt/releases/download/v1.0.3/kube-prompt_v1.0.3_linux_amd64.zip 18 | dest: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" 19 | 20 | - name: Unarchive kube-prompt 21 | unarchive: 22 | src: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" 23 | dest: "{{ dir.path }}" 24 | remote_src: yes 25 | creates: "{{ dir.path }}/kube-prompt" 26 | 27 | - name: Copy kube-prompt to bin 28 | copy: 29 | dest: /usr/local/bin/kprompt 30 | src: "{{ dir.path }}/kube-prompt" 31 | mode: u+x 32 | remote_src: yes -------------------------------------------------------------------------------- /installation/ansible/version_1/reset_k8s.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Drain the node 3 | hosts: k8s_master 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | rebuild: false 10 | 11 | tasks: 12 | - name: Drain the node 13 | shell: kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets 14 | when: rebuild | bool 15 | ignore_errors: True 16 | with_items: "{{ groups['k8s_nodes'] }}" 17 | 18 | - name: Delete node 19 | shell: kubectl delete node {{ item }} 20 | when: rebuild | bool 21 | ignore_errors: True 22 | with_items: "{{ groups['k8s_nodes'] }}" 23 | 24 | - name: Reset k8s on each node 25 | hosts: k8s_hosts 26 | become: true 27 | become_method: sudo 28 | gather_facts: false 29 | 30 | vars: 31 | rebuild: false 32 | 33 | tasks: 34 | - name: Reset 35 | shell: kubeadm reset 36 | when: rebuild | bool 37 | ignore_errors: True -------------------------------------------------------------------------------- /installation/ansible/version_2/README.md: -------------------------------------------------------------------------------- 1 | # Create k8s cluster using Ansible on top of OpenStack 2 | 3 | This script can only create kubernetes cluster including 1 master and multiple nodes. Version 2 playbook doesn't take openstack as k8s cloud provider, if you need integration with openstack, please refer to version 3. Version 2 is better than Version 1 because it's using roles. 4 | 5 | ## How to run 6 | ### Prepare your local environment 7 | ```shell 8 | mkvirtualenv test_k8s 9 | pip install ansible shade 10 | ``` 11 | 12 | ### Install 13 | Take a look at the variables in `site.yml` file, you need to define your own as needed or pass those as ansible-playbook vars. 14 | 15 | ```bash 16 | source openrc 17 | ansible-playbook site.yml -e "rebuild=false image=$image flavor=$flavor" 18 | ``` 19 | 20 | If anything unexpected happened during the installation, just re-run using: 21 | ```shell 22 | ansible-playbook site.yml -e "rebuild=true" 23 | ``` 24 | 25 | ## Destroy 26 | 27 | - Delete the instances 28 | - Delete ports after instances deletion, using `neutron port-list -- --device-id ` to get port attached to the instance. 29 | -------------------------------------------------------------------------------- /installation/ansible/version_2/reset_k8s.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Drain the node 3 | hosts: k8s_master 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | rebuild: false 10 | 11 | tasks: 12 | - name: Drain the node 13 | shell: kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets 14 | when: rebuild | bool 15 | ignore_errors: True 16 | with_items: "{{ groups['k8s_nodes'] }}" 17 | 18 | - name: Delete node 19 | shell: kubectl delete node {{ item }} 20 | when: rebuild | bool 21 | ignore_errors: True 22 | with_items: "{{ groups['k8s_nodes'] }}" 23 | 24 | - name: Reset k8s on each node 25 | hosts: k8s_hosts 26 | become: true 27 | become_method: sudo 28 | gather_facts: false 29 | 30 | vars: 31 | rebuild: false 32 | 33 | tasks: 34 | - name: Reset 35 | shell: kubeadm reset 36 | when: rebuild | bool 37 | ignore_errors: True -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | docker_version: 17.03.2~ce-0~ubuntu-xenial -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/docker/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install packages 3 | apt: name={{ item }} state=present update_cache=yes 4 | with_items: 5 | - apt-transport-https 6 | - ca-certificates 7 | - curl 8 | 9 | - name: Add Docker’s official GPG key 10 | apt_key: 11 | url: https://download.docker.com/linux/ubuntu/gpg 12 | state: present 13 | 14 | - name: Set up the stable repository 15 | apt_repository: repo='deb https://download.docker.com/linux/ubuntu xenial stable' state=present 16 | 17 | - name: Install docker-ce 18 | apt: 19 | name: docker-ce={{ docker_version }} 20 | state: present 21 | update_cache: yes -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/k8s_cli/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k8s_version: 1.11.1-00 3 | -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create temporary directory 3 | file: 4 | state: directory 5 | path: ~/kube-prompt 6 | register: dir 7 | 8 | - name: Download kube-prompt 9 | get_url: 10 | url: https://github.com/c-bata/kube-prompt/releases/download/v1.0.3/kube-prompt_v1.0.3_linux_amd64.zip 11 | dest: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" 12 | force: no 13 | 14 | - name: Unarchive kube-prompt 15 | unarchive: 16 | src: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" 17 | dest: "{{ dir.path }}" 18 | remote_src: yes 19 | creates: "{{ dir.path }}/kube-prompt" 20 | 21 | - name: copy kube-prompt to /usr/bin 22 | copy: 23 | src: "{{ dir.path }}/kube-prompt" 24 | dest: /usr/bin/kube-prompt 25 | remote_src: yes 26 | force: no 27 | mode: 0755 28 | 29 | - name: create alias for kube-prompt 30 | lineinfile: 31 | path: /etc/bash.bashrc 32 | create: true 33 | line: 'alias kp="/usr/bin/kube-prompt"' 34 | -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/k8s_cli/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: add Kubernetes GPG key 3 | apt_key: 4 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 5 | state: present 6 | 7 | - name: set up Kubernetes repository 8 | apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present 9 | 10 | - name: install kube CLIs 11 | apt: 12 | name: "{{ item }}" 13 | state: present 14 | update_cache: yes 15 | with_items: 16 | - kubelet={{ k8s_version }} 17 | - kubectl={{ k8s_version }} 18 | - kubeadm={{ k8s_version }} 19 | 20 | - name: create alias for cli 21 | lineinfile: 22 | path: /etc/bash.bashrc 23 | create: true 24 | line: 'alias kc="/usr/bin/kubectl"' 25 | -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/kube_master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k8s_version: v1.11.1 3 | -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/kube_master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Init master 3 | shell: kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version={{ k8s_version }} 4 | register: kubeadm_init 5 | 6 | - fail: 7 | msg: "kubeadm init command failed." 8 | when: kubeadm_init.stdout.find ("kubeadm join") == -1 9 | 10 | - name: Store kubeadm join string 11 | set_fact: 12 | join_str: "{{ kubeadm_init.stdout | regex_search('kubeadm join(.*)$') }}" 13 | 14 | - name: Prepare kube config 15 | shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config 16 | with_items: 17 | - "{{ ansible_env.HOME }}" 18 | - /home/ubuntu 19 | 20 | - name: Allow pod on master 21 | shell: kubectl taint nodes --all node-role.kubernetes.io/master- 22 | 23 | - name: Download calico yaml 24 | get_url: 25 | url: https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml 26 | dest: "{{ ansible_env.HOME }}/calico.yaml" 27 | force: no 28 | 29 | - name: Change calico config 30 | replace: 31 | path: "{{ ansible_env.HOME }}/calico.yaml" 32 | regexp: 'value: "always"' 33 | replace: 'value: "off"\n - name: CALICO_IPV4POOL_NAT_OUTGOING\n value: "true"' 34 | backup: yes 35 | 36 | - name: Install network plugin 37 | shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml 38 | -------------------------------------------------------------------------------- /installation/ansible/version_2/roles/kube_node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Join to cluster 3 | shell: "{{ hostvars[groups['k8s_master'][0]]['join_str'] }}" -------------------------------------------------------------------------------- /installation/ansible/version_2/site.yml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | # Usage: 3 | # source openrc 4 | # flavor=$(nova flavor-list | grep c2r4 | awk '{print $2}') && echo "flavor: $flavor" 5 | # image=$(glance image-list | grep ubuntu-16.04-x86_64 | awk '$4 ~ /ubuntu-16.04-x86_64$/ {print $2}') && echo "image: $image" 6 | # ansible-playbook site.yml -e "rebuild=false image=$image flavor=$flavor" 7 | # Don't forget to delete the ports when clearing up. 8 | --- 9 | - name: Deploy and prepare instances in OpenStack 10 | hosts: localhost 11 | 12 | vars: 13 | image: 9136e86b-14d9-4127-b3dd-f1cdb18b0dc6 14 | network: network1 15 | key_name: lingxian_pc 16 | flavor: 2ba44887-5fe3-44cc-8bca-e2b60a206a66 17 | security_groups: 18 | - lingxian 19 | node_prefix: lingxian 20 | # Mater hostname must contain 'master', node hostname contains 'node' 21 | node_names: 22 | - master 23 | - node1 24 | - node2 25 | 26 | tasks: 27 | # Create ports separately in order to setup allowed_address_pairs 28 | - name: Create ports 29 | os_port: 30 | state: present 31 | name: "{{ node_prefix }}-k8s-{{ item }}" 32 | network: "{{ network }}" 33 | security_groups: "{{ security_groups }}" 34 | allowed_address_pairs: 35 | - ip_address: 192.168.0.0/16 36 | with_items: "{{ node_names }}" 37 | 38 | - name: Create VMs 39 | os_server: 40 | state: present 41 | name: "{{ node_prefix }}-k8s-{{ item }}" 42 | boot_from_volume: true 43 | image: "{{ image }}" 44 | volume_size: 15 45 | key_name: "{{ key_name }}" 46 | nics: port-name={{ node_prefix }}-k8s-{{ item }} 47 | flavor: "{{ flavor }}" 48 | terminate_volume: true 49 | auto_ip: false 50 | with_items: "{{ node_names }}" 51 | 52 | - name: Gather facts about vms 53 | os_server_facts: 54 | server: "{{ node_prefix }}-k8s-*" 55 | 56 | - name: Wait for ssh ready 57 | wait_for: port=22 host="{{ item.private_v4 }}" search_regex=OpenSSH timeout=300 delay=5 58 | with_items: "{{ openstack_servers }}" 59 | 60 | - name: Add VMs to inventory 61 | add_host: 62 | name: "{{ item.name }}" 63 | groups: k8s_hosts 64 | ansible_user: ubuntu 65 | ansible_host: "{{ item.private_v4 }}" 66 | with_items: "{{ openstack_servers }}" 67 | 68 | - name: Setup master group 69 | add_host: 70 | name: "{{ item.name }}" 71 | groups: k8s_master 72 | ansible_user: ubuntu 73 | ansible_host: "{{ item.private_v4 }}" 74 | when: item.name | regex_search('(master)') 75 | with_items: "{{ openstack_servers }}" 76 | 77 | - name: Setup nodes group 78 | add_host: 79 | name: "{{ item.name }}" 80 | groups: k8s_nodes 81 | ansible_user: ubuntu 82 | ansible_host: "{{ item.private_v4 }}" 83 | when: item.name | regex_search('(node)') 84 | with_items: "{{ openstack_servers }}" 85 | 86 | - name: Prepare k8s hosts 87 | hosts: k8s_hosts 88 | become: true 89 | become_method: sudo 90 | gather_facts: false 91 | 92 | pre_tasks: 93 | - name: install python 2 94 | raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) 95 | 96 | - name: Install pip 97 | shell: which pip || (curl https://bootstrap.pypa.io/get-pip.py | python - && rm -f get-pip.py) 98 | 99 | - name: Gather facts after python installation 100 | setup: filter=ansible_* 101 | 102 | - name: Install packages 103 | apt: name={{ item }} state=present update_cache=yes 104 | with_items: 105 | - ansible 106 | - unzip 107 | - build-essential 108 | - python-dev 109 | - python-setuptools 110 | - libffi-dev 111 | - libxslt1-dev 112 | - libxml2-dev 113 | - libyaml-dev 114 | - libssl-dev 115 | - zlib1g-dev 116 | 117 | - name: Install pypi libraries 118 | pip: name={{ item }} 119 | with_items: 120 | - httpie 121 | - kube-shell 122 | 123 | - name: Build hosts file 124 | lineinfile: dest=/etc/hosts line="{{ ansible_host }} {{ ansible_hostname }}" state=present 125 | 126 | roles: 127 | - docker 128 | - k8s_cli 129 | 130 | - import_playbook: reset_k8s.yml rebuild={{ rebuild }} 131 | 132 | - name: Set up k8s master 133 | hosts: k8s_master 134 | become: true 135 | become_method: sudo 136 | gather_facts: true 137 | 138 | roles: 139 | - kube_master 140 | 141 | - name: Set up k8s node 142 | hosts: k8s_nodes 143 | become: true 144 | become_method: sudo 145 | gather_facts: false 146 | 147 | roles: 148 | - kube_node 149 | -------------------------------------------------------------------------------- /installation/ansible/version_3/README.md: -------------------------------------------------------------------------------- 1 | # Create k8s cluster using ansible on top of OpenStack 2 | 3 | This script can only create kubernetes cluster including 1 master and multiple nodes. Kubernetes cluster will use OpenStack as cloud provider, VMs(and related cloud resources) are also created. If you want to use separate openstack cloud controller manager, please see [here](https://github.com/lingxiankong/kubernetes-study/tree/master/installation/ansible/external-cloud-provider) 4 | 5 | ## How to run 6 | ### Prepare your local environment 7 | ```shell 8 | mkvirtualenv test_k8s 9 | pip install ansible shade 10 | ``` 11 | 12 | ### Install 13 | Take a look at the variables in `site.yaml` file, you need to define your own as needed or pass those as ansible-playbook vars. Here is an script example that I used in my **devstack** environment. 14 | 15 | ```bash 16 | echo 'alias source_adm="source ~/devstack/openrc admin admin"' >> ~/.bashrc 17 | echo 'alias source_demo="source ~/devstack/openrc demo demo"' >> ~/.bashrc 18 | echo 'alias source_altdemo="source ~/devstack/openrc alt_demo alt_demo"' >> ~/.bashrc 19 | echo 'alias o="openstack"' >> ~/.bashrc 20 | echo 'alias k="kubectl"' >> ~/.bashrc 21 | echo 'alias lb="openstack loadbalancer"' >> ~/.bashrc 22 | echo 'export PYTHONWARNINGS="ignore"' >> ~/.bashrc 23 | sed -i "/alias ll/c alias ll='ls -l'" ~/.bashrc 24 | source ~/.bashrc 25 | sudo -s 26 | 27 | cd ~ 28 | cat << EOF > pre.sh 29 | set -e 30 | pushd ~/devstack 31 | # create a new flavor 32 | source openrc admin admin 33 | openstack flavor create --id 6 --ram 8196 --disk 20 --vcpus 8 --public k8s 34 | # create keypair 35 | source openrc demo demo 36 | ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa 37 | openstack keypair create --public-key ~/.ssh/id_rsa.pub lingxian_key 38 | # set up the default security group rules 39 | openstack security group rule create --proto icmp default 40 | openstack security group rule create --protocol tcp --dst-port 1:65535 default 41 | # register ubuntu image 42 | source openrc admin admin 43 | curl -SOL http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img 44 | glance image-create --name ubuntu-xenial \ 45 | --visibility public \ 46 | --container-format bare \ 47 | --disk-format qcow2 \ 48 | --file xenial-server-cloudimg-amd64-disk1.img 49 | rm -f xenial-server-cloudimg-amd64-disk1.img 50 | # modify quota 51 | openstack quota set --instances 100 --cores 50 --secgroups 100 --secgroup-rules 500 demo 52 | # install and config ansible 53 | sudo apt-add-repository ppa:ansible/ansible -y && sudo apt-get update -y && sudo apt-get install -y ansible qemu-kvm && sudo pip install shade 54 | sed -i '/stdout_callback/c stdout_callback=debug' /etc/ansible/ansible.cfg 55 | # config ssh 56 | cat <> /etc/ssh/ssh_config 57 | StrictHostKeyChecking no 58 | UserKnownHostsFile=/dev/null 59 | END 60 | popd 61 | EOF 62 | bash pre.sh 63 | 64 | cd ~; git clone https://github.com/lingxiankong/kubernetes_study.git 65 | cd ~/kubernetes_study/installation/ansible/version_3/ 66 | source_demo 67 | image=$(openstack image list --name ubuntu-xenial -c ID -f value); echo $image 68 | network=$(openstack network list --name private -c ID -f value); echo $network 69 | subnet_id=$(openstack subnet list --network private --name private-subnet -c ID -f value); echo $subnet_id 70 | auth_url=$(export | grep OS_AUTH_URL | awk -F '"' '{print $2}'); echo $auth_url 71 | source_adm 72 | user_id=$(openstack user show demo -c id -f value) 73 | tenant_id=$(openstack project show demo -c id -f value) 74 | source_demo 75 | 76 | ansible-playbook site.yaml -v -e \ 77 | "node_prefix=test \ 78 | rebuild=false \ 79 | flavor=6 \ 80 | image=$image \ 81 | network=$network \ 82 | key_name=testkey \ 83 | private_key=$HOME/.ssh/id_rsa \ 84 | auth_url=$auth_url \ 85 | user_id=$user_id \ 86 | password=password \ 87 | tenant_id=$tenant_id \ 88 | region=RegionOne \ 89 | subnet_id=$subnet_id \ 90 | k8s_version=1.17.3" 91 | ``` 92 | 93 | If anything unexpected happened during the installation, just re-run the command but changing `rebuild=true`. 94 | 95 | ## Clean up 96 | 97 | - Delete the instances 98 | - Disassociate/delete allocated floatingips if needed 99 | - Delete ports after instances deletion, using `neutron port-list -- --device-id ` to get ports attached to the instance. 100 | -------------------------------------------------------------------------------- /installation/ansible/version_3/reset_k8s.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Drain the node 3 | hosts: k8s_master 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | rebuild: false 10 | 11 | tasks: 12 | - name: Drain the node 13 | shell: kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets 14 | when: rebuild | bool 15 | ignore_errors: True 16 | with_items: "{{ groups['k8s_nodes'] }}" 17 | 18 | - name: Delete node 19 | shell: kubectl delete node {{ item }} 20 | when: rebuild | bool 21 | ignore_errors: True 22 | with_items: "{{ groups['k8s_nodes'] }}" 23 | 24 | - name: Reset k8s on each node 25 | hosts: k8s_hosts 26 | become: true 27 | become_method: sudo 28 | gather_facts: false 29 | 30 | vars: 31 | rebuild: false 32 | 33 | tasks: 34 | - name: Reset 35 | shell: kubeadm reset -f 36 | when: rebuild | bool 37 | ignore_errors: True -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | docker_version: 5:19.03.4~3-0~ubuntu-bionic -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # apt-get install -y apt-transport-https ca-certificates curl 3 | - name: Install packages 4 | apt: 5 | name: "{{ packages }}" 6 | state: present 7 | update_cache: yes 8 | vars: 9 | packages: 10 | - apt-transport-https 11 | - ca-certificates 12 | - curl 13 | 14 | # curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 15 | - name: Add Docker’s official GPG key 16 | apt_key: 17 | url: https://download.docker.com/linux/ubuntu/gpg 18 | state: present 19 | 20 | # add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable" 21 | - name: Set up the stable repository 22 | apt_repository: repo='deb https://download.docker.com/linux/ubuntu {{ hostvars[inventory_hostname].ansible_distribution_release }} stable' state=present 23 | 24 | # apt-get update; apt install -y docker-ce=17.03.2~ce-0~ubuntu-xenial 25 | - name: Install docker-ce 26 | apt: 27 | name: docker-ce={{ docker_version }} 28 | state: present 29 | update_cache: yes -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/k8s_cli/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k8s_version: "1.17.3" 3 | kube_prompt_version: "v1.0.5" -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create temporary directory 3 | file: 4 | state: directory 5 | path: ~/kube-prompt 6 | register: dir 7 | 8 | - name: Download kube-prompt 9 | get_url: 10 | url: https://github.com/c-bata/kube-prompt/releases/download/{{ kube_prompt_version }}/kube-prompt_{{ kube_prompt_version }}_linux_amd64.zip 11 | dest: "{{ dir.path }}/kube-prompt.zip" 12 | force: no 13 | 14 | - name: Unarchive kube-prompt 15 | unarchive: 16 | src: "{{ dir.path }}/kube-prompt.zip" 17 | dest: "{{ dir.path }}" 18 | remote_src: yes 19 | creates: "{{ dir.path }}/kube-prompt" 20 | 21 | - name: copy kube-prompt to /usr/bin 22 | copy: 23 | src: "{{ dir.path }}/kube-prompt" 24 | dest: /usr/bin/kube-prompt 25 | remote_src: yes 26 | force: no 27 | mode: 0755 28 | 29 | - name: create alias for kube-prompt 30 | lineinfile: 31 | path: /etc/bash.bashrc 32 | create: true 33 | line: 'alias kprompt="/usr/bin/kube-prompt"' 34 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/k8s_cli/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - 3 | - name: Add Kubernetes GPG key 4 | apt_key: 5 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 6 | state: present 7 | 8 | # add-apt-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main" 9 | # At the time of writing, Ubuntu 18.04 (Bionic) packages for Kubernetes are not available. 10 | # Check here https://packages.cloud.google.com/apt/dists 11 | - name: Set up Kubernetes repository 12 | apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present 13 | 14 | # apt update; apt install -y kubelet=1.11.1-00 kubectl=1.11.1-00 kubeadm=1.11.1-00 15 | - name: Install kube CLIs 16 | apt: 17 | name: "{{ packages }}" 18 | state: present 19 | update_cache: yes 20 | vars: 21 | packages: 22 | - kubelet={{ k8s_version }}-00 23 | - kubectl={{ k8s_version }}-00 24 | - kubeadm={{ k8s_version }}-00 25 | 26 | # - import_tasks: install_kube_prompt_tasks.yml 27 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k8s_version: "1.13.2" 3 | pod_subnet: "192.168.0.0/16" 4 | extra_api_endpoints: [] 5 | 6 | auth_url: http://192.168.121.15/identity/v3/ 7 | user_id: 3fd32295751a41aca601021eba681a29 8 | password: password 9 | tenant_id: 84ce53873abe4d22a6f2491879e76048 10 | region: RegionOne 11 | subnet_id: 1740cd57-62ab-4f7f-a853-2038561060d1 12 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare kubeadm.conf 3 | template: 4 | src: kubeadm.conf.j2 5 | dest: "{{ ansible_env.HOME }}/kubeadm.conf" 6 | 7 | - name: Make sure /etc/kubernetes/pki exists 8 | file: 9 | path: /etc/kubernetes/pki 10 | state: directory 11 | 12 | - name: Prepare openstack config file 13 | template: 14 | src: cloud-config.j2 15 | dest: /etc/kubernetes/pki/cloud-config 16 | 17 | - name: Check if kubelet config file needs change 18 | shell: grep -q "/etc/kubernetes/pki/cloud-config" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 19 | register: check_kubelet 20 | ignore_errors: True 21 | 22 | - name: Modify kubelet arguments 23 | replace: 24 | path: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 25 | regexp: '(.*)KUBELET_KUBECONFIG_ARGS=(.*)$' 26 | replace: '\1KUBELET_KUBECONFIG_ARGS=--cloud-provider=openstack --cloud-config=/etc/kubernetes/pki/cloud-config \2' 27 | when: check_kubelet.rc != 0 28 | 29 | - name: restart kubelet service 30 | systemd: 31 | state: restarted 32 | daemon_reload: yes 33 | name: kubelet 34 | 35 | - name: Init master 36 | shell: kubeadm init --config {{ ansible_env.HOME }}/kubeadm.conf 37 | register: kubeadm_init 38 | 39 | - fail: 40 | msg: "kubeadm init command failed." 41 | when: kubeadm_init.stdout.find ("kubeadm join") == -1 42 | 43 | - name: Prepare kube config 44 | shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config 45 | with_items: 46 | - "{{ ansible_env.HOME }}" 47 | - /home/ubuntu 48 | 49 | - name: Allow pod on master 50 | shell: kubectl taint nodes --all node-role.kubernetes.io/master- 51 | 52 | - name: Download calico manifest file if needed 53 | get_url: 54 | url: https://docs.projectcalico.org/v3.11/manifests/calico.yaml 55 | dest: "{{ ansible_env.HOME }}/calico.yaml" 56 | force: no 57 | 58 | - name: Modify the pod cidr in calico manifest file 59 | replace: 60 | path: "{{ ansible_env.HOME }}/calico.yaml" 61 | regexp: '(^.*value: )"192.168.*' 62 | replace: '\1"{{ pod_subnet }}"' 63 | 64 | - name: Install Calico 65 | shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml 66 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_master/templates/cloud-config.j2: -------------------------------------------------------------------------------- 1 | [Global] 2 | auth-url={{ auth_url }} 3 | user-id={{ user_id }} 4 | password={{ password }} 5 | tenant-id={{ tenant_id }} 6 | region={{ region }} 7 | [LoadBalancer] 8 | use-octavia=yes 9 | subnet-id={{ subnet_id }} 10 | create-monitor=no 11 | [BlockStorage] 12 | bs-version=v2 13 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_master/templates/kubeadm.conf.j2: -------------------------------------------------------------------------------- 1 | #jinja2: lstrip_blocks: "True", trim_blocks: "True" 2 | --- 3 | apiVersion: kubeadm.k8s.io/v1beta2 4 | kind: InitConfiguration 5 | bootstrapTokens: 6 | - token: "9a08jv.c0izixklcxtmnze7" 7 | description: "kubeadm bootstrap token" 8 | ttl: "24h" 9 | 10 | --- 11 | apiVersion: kubeadm.k8s.io/v1beta2 12 | kind: ClusterConfiguration 13 | kubernetesVersion: v{{ k8s_version }} 14 | networking: 15 | podSubnet: {{ pod_subnet }} 16 | imageRepository: "k8s.gcr.io" 17 | apiServer: 18 | certSANs: 19 | - {{ ansible_local_ip }} 20 | - {{ ansible_public_ip }} 21 | {% for ip in extra_api_endpoints %} 22 | - {{ ip }} 23 | {% endfor %} 24 | extraArgs: 25 | cloud-provider: "openstack" 26 | cloud-config: /etc/kubernetes/pki/cloud-config 27 | controllerManager: 28 | extraArgs: 29 | cloud-provider: "openstack" 30 | cloud-config: /etc/kubernetes/pki/cloud-config 31 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_node/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | auth_url: http://192.168.121.15/identity/v3/ 3 | user_id: 3fd32295751a41aca601021eba681a29 4 | password: password 5 | tenant_id: 84ce53873abe4d22a6f2491879e76048 6 | region: RegionOne 7 | subnet_id: 1740cd57-62ab-4f7f-a853-2038561060d1 8 | 9 | bootstrap: true 10 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare kubeadm.conf 3 | template: 4 | src: kubeadm.conf.j2 5 | dest: "{{ ansible_env.HOME }}/kubeadm.conf" 6 | 7 | - name: Make sure /etc/kubernetes/pki exists 8 | file: 9 | path: /etc/kubernetes/pki 10 | state: directory 11 | 12 | - name: Prepare openstack config file 13 | template: 14 | src: cloud-config.j2 15 | dest: /etc/kubernetes/pki/cloud-config 16 | 17 | - name: Modify kubelet arguments 18 | shell: sed -i -E 's/(.*)KUBELET_KUBECONFIG_ARGS=(.*)$/\1KUBELET_KUBECONFIG_ARGS=--cloud-provider=openstack --cloud-config=\/etc\/kubernetes\/pki\/cloud-config \2/' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 19 | 20 | - name: restart kubelet service 21 | systemd: 22 | state: restarted 23 | daemon_reload: yes 24 | name: kubelet 25 | 26 | - name: Join to cluster 27 | shell: kubeadm join --config {{ ansible_env.HOME }}/kubeadm.conf 28 | when: bootstrap | bool 29 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_node/templates/cloud-config.j2: -------------------------------------------------------------------------------- 1 | [Global] 2 | auth-url={{ auth_url }} 3 | user-id={{ user_id }} 4 | password={{ password }} 5 | tenant-id={{ tenant_id }} 6 | region={{ region }} 7 | [LoadBalancer] 8 | use-octavia=yes 9 | subnet-id={{ subnet_id }} 10 | create-monitor=no 11 | [BlockStorage] 12 | bs-version=v2 13 | -------------------------------------------------------------------------------- /installation/ansible/version_3/roles/kube_node/templates/kubeadm.conf.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubeadm.k8s.io/v1beta2 3 | kind: JoinConfiguration 4 | discovery: 5 | bootstrapToken: 6 | apiServerEndpoint: {{ hostvars[groups['k8s_master'][0]]['ansible_local_ip'] }}:6443 7 | token: 9a08jv.c0izixklcxtmnze7 8 | unsafeSkipCAVerification: true -------------------------------------------------------------------------------- /installation/ansible/version_3/site.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | # Usage: 3 | # source openrc 4 | # ansible-playbook site.yml -e "rebuild=false" 5 | # Don't forget to delete the ports when clearing up. 6 | --- 7 | - name: Deploy and prepare instances in OpenStack 8 | hosts: localhost 9 | 10 | vars: 11 | image: 9136e86b-14d9-4127-b3dd-f1cdb18b0dc6 12 | network: network1 13 | public_network: public 14 | key_name: lingxian_pc 15 | private_key: /home/vagrant/.ssh/id_rsa 16 | flavor: 2ba44887-5fe3-44cc-8bca-e2b60a206a66 17 | boot_from_vol: false 18 | security_groups: 19 | - default 20 | node_prefix: lingxiantest 21 | # Mater hostname must contain 'master', node hostname contains 'node' 22 | # Make sure the vm name complies with k8s naming convention, should not 23 | # contain '_' 24 | node_names: 25 | - master 26 | - node1 27 | pod_subnet: 192.168.0.0/16 28 | 29 | tasks: 30 | # Create ports separately in order to setup allowed_address_pairs 31 | - name: Create ports 32 | os_port: 33 | state: present 34 | name: "{{ node_prefix }}-k8s-{{ item }}" 35 | network: "{{ network }}" 36 | security_groups: "{{ security_groups }}" 37 | allowed_address_pairs: 38 | - ip_address: "{{ pod_subnet }}" 39 | with_items: "{{ node_names }}" 40 | 41 | - name: Create VMs 42 | os_server: 43 | state: present 44 | name: "{{ node_prefix }}-k8s-{{ item }}" 45 | boot_from_volume: "{{ boot_from_vol | bool }}" 46 | image: "{{ image }}" 47 | volume_size: 15 48 | key_name: "{{ key_name }}" 49 | nics: port-name={{ node_prefix }}-k8s-{{ item }} 50 | flavor: "{{ flavor }}" 51 | terminate_volume: true 52 | # Do not allocate floating ip 53 | auto_ip: false 54 | with_items: "{{ node_names }}" 55 | 56 | - name: Associate floating ip if needed 57 | os_floating_ip: 58 | state: present 59 | reuse: yes 60 | network: "{{ public_network }}" 61 | server: "{{ node_prefix }}-k8s-{{ item }}" 62 | with_items: "{{ node_names }}" 63 | when: public_network is defined 64 | 65 | - name: Gather facts about vms 66 | os_server_facts: 67 | server: "{{ node_prefix }}-k8s-*" 68 | 69 | - name: Wait for ssh ready 70 | wait_for: port=22 host="{{ item.private_v4 if public_network is undefined else item.public_v4 }}" search_regex=OpenSSH timeout=300 delay=5 71 | with_items: "{{ openstack_servers }}" 72 | 73 | - name: Add VMs to inventory 74 | add_host: 75 | name: "{{ item.name }}" 76 | groups: k8s_hosts 77 | ansible_user: ubuntu 78 | ansible_host: "{{ item.public_v4 }}" 79 | ansible_local_ip: "{{ item.private_v4 }}" 80 | ansible_public_ip: "{{ item.public_v4 }}" 81 | ansible_ssh_private_key_file: "{{ private_key }}" 82 | ansible_ssh_common_args: "-o StrictHostKeyChecking=no" 83 | with_items: "{{ openstack_servers }}" 84 | 85 | - name: Setup master group 86 | add_host: 87 | name: "{{ item.name }}" 88 | groups: k8s_master 89 | ansible_user: ubuntu 90 | ansible_host: "{{ item.public_v4 }}" 91 | ansible_local_ip: "{{ item.private_v4 }}" 92 | ansible_public_ip: "{{ item.public_v4 }}" 93 | ansible_ssh_private_key_file: "{{ private_key }}" 94 | when: item.name | regex_search('(master)') 95 | with_items: "{{ openstack_servers }}" 96 | 97 | - name: Setup nodes group 98 | add_host: 99 | name: "{{ item.name }}" 100 | groups: k8s_nodes 101 | ansible_user: ubuntu 102 | ansible_host: "{{ item.public_v4 }}" 103 | ansible_local_ip: "{{ item.private_v4 }}" 104 | ansible_public_ip: "{{ item.public_v4 }}" 105 | ansible_ssh_private_key_file: "{{ private_key }}" 106 | when: item.name | regex_search('(node)') 107 | with_items: "{{ openstack_servers }}" 108 | 109 | - name: Prepare k8s hosts 110 | hosts: k8s_hosts 111 | become: true 112 | become_method: sudo 113 | gather_facts: false 114 | 115 | pre_tasks: 116 | - name: Wait for cloud init to finish 117 | raw: cloud-init status --wait 118 | 119 | - name: Install python 3 120 | raw: test -e /usr/bin/python || (apt -y update && apt install -y python3-minimal) 121 | 122 | - name: Install packages 123 | apt: 124 | name: "{{ packages }}" 125 | state: present 126 | update_cache: yes 127 | vars: 128 | packages: 129 | - python3-pip 130 | - ansible 131 | - unzip 132 | - build-essential 133 | - python3-dev 134 | - python3-setuptools 135 | - libffi-dev 136 | - libxslt1-dev 137 | - libxml2-dev 138 | - libyaml-dev 139 | - libssl-dev 140 | - zlib1g-dev 141 | - jq 142 | - net-tools 143 | 144 | - name: Pin pip to 9.0.3 145 | shell: pip3 install pip==9.0.3 146 | 147 | - name: Install pypi libraries 148 | pip: 149 | name: "{{ libraries }}" 150 | ignore_errors: True 151 | vars: 152 | libraries: 153 | - python-openstackclient 154 | - python-octaviaclient 155 | - httpie 156 | 157 | - name: Gather facts after python installation 158 | setup: filter=ansible_* 159 | 160 | - name: Build hosts file 161 | lineinfile: dest=/etc/hosts line="{{ ansible_host }} {{ ansible_hostname }}" state=present 162 | 163 | - name: Disable swap memory 164 | shell: swapoff -a 165 | 166 | roles: 167 | - docker 168 | - k8s_cli 169 | 170 | - import_playbook: reset_k8s.yaml rebuild={{ rebuild }} 171 | 172 | - name: Set up k8s master 173 | hosts: k8s_master 174 | become: true 175 | become_method: sudo 176 | gather_facts: true 177 | 178 | roles: 179 | - kube_master 180 | 181 | - name: Set up k8s node 182 | hosts: k8s_nodes 183 | become: true 184 | become_method: sudo 185 | gather_facts: false 186 | 187 | roles: 188 | - kube_node 189 | -------------------------------------------------------------------------------- /installation/ansible/version_4/README.md: -------------------------------------------------------------------------------- 1 | # Create k8s cluster using Ansible on top of OpenStack 2 | 3 | This script can create multi-master kubernetes cluster. Kubernetes cluster will use OpenStack as cloud provider. 4 | 5 | ## How to run 6 | ### Prepare your local environment 7 | ```shell 8 | mkvirtualenv test_k8s 9 | pip install ansible shade 10 | ``` 11 | 12 | ### Install 13 | Take a look at the variables in `site.yml` file, you need to define your own as needed or pass those as ansible-playbook vars. Here is an script example that I used in my devstack environment. 14 | 15 | ```bash 16 | cat << EOF >> ~/.bashrc 17 | alias source_adm="cd ~/devstack; source openrc admin admin; cd -" 18 | alias source_demo="cd ~/devstack; source openrc demo demo; cd -" 19 | alias source_altdemo="cd ~/devstack; source openrc alt_demo alt_demo; cd -" 20 | alias os="openstack" 21 | alias ll='ls -l' 22 | EOF 23 | 24 | cat << EOF > pre.sh 25 | set -e 26 | pushd ~/devstack 27 | # create keypair 28 | source openrc demo demo 29 | ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa 30 | openstack keypair create --public-key ~/.ssh/id_rsa.pub testkey 31 | # set up the security group rules 32 | openstack security group rule create --proto icmp default 33 | openstack security group rule create --protocol tcp --dst-port 1:65535 default 34 | # register ubuntu image 35 | source openrc admin admin 36 | curl -SOL http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img 37 | glance image-create --name ubuntu-xenial \ 38 | --visibility public \ 39 | --container-format bare \ 40 | --disk-format qcow2 \ 41 | --file xenial-server-cloudimg-amd64-disk1.img 42 | rm -f xenial-server-cloudimg-amd64-disk1.img 43 | # modify quota 44 | openstack quota set --instances 100 --cores 50 --secgroups 100 --secgroup-rules 500 demo 45 | # update ansible and install shade 46 | sudo apt-add-repository ppa:ansible/ansible -y && sudo apt-get update -y && sudo apt-get install -y ansible && sudo pip install shade 47 | popd 48 | EOF 49 | 50 | bash pre.sh 51 | 52 | cd ~; git clone https://github.com/lingxiankong/kubernetes_study.git 53 | cd ~/kubernetes_study/installation/ansible/version_4/ 54 | pushd ~/devstack && source openrc demo demo && popd 55 | image=$(openstack image list --name ubuntu-xenial -c ID -f value); echo $image 56 | network=$(openstack network list --name private -c ID -f value); echo $network 57 | subnet_id=$(openstack subnet list --network private --name private-subnet -c ID -f value); echo $subnet_id 58 | auth_url=$(export | grep OS_AUTH_URL | awk -F '"' '{print $2}'); echo $auth_url 59 | pushd ~/devstack && source openrc admin admin && popd 60 | user_id=$(openstack user show demo -c id -f value) 61 | tenant_id=$(openstack project show demo -c id -f value) 62 | cat < roles/kube_master/defaults/main.yml 63 | auth_url: $auth_url 64 | user_id: $user_id 65 | password: password 66 | tenant_id: $tenant_id 67 | region: RegionOne 68 | subnet_id: $subnet_id 69 | master_0: "true" 70 | EOF 71 | cp roles/kube_master/defaults/main.yml roles/kube_node/defaults/main.yml 72 | cat roles/kube_node/defaults/main.yml 73 | 74 | pushd ~/devstack && source openrc demo demo && popd 75 | ansible-playbook site.yml -e "rebuild=false flavor=6 image=$image network=$network subnet=8c51132b-e0b2-4ba8-8e6b-685063fd5e71 key_name=testkey private_key=$HOME/.ssh/id_rsa" 76 | popd 77 | ``` 78 | 79 | If anything unexpected happened during the installation, just re-run the ansible-playbook command but using `rebuild=false`: 80 | ble-playbook site.yml -e "rebuild=true". 81 | 82 | ## Destroy 83 | 84 | - Delete the load balancer(using `--cascade`) 85 | - Delete the instances 86 | - Disassociate/delete allocated floatingips if needed 87 | - Delete ports after instances deletion, using `neutron port-list -- --device-id ` to get port attached to the instance. 88 | -------------------------------------------------------------------------------- /installation/ansible/version_4/reset_k8s.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Drain the node 3 | hosts: k8s_master_0 4 | become: true 5 | become_method: sudo 6 | gather_facts: false 7 | 8 | vars: 9 | rebuild: false 10 | 11 | tasks: 12 | - name: Drain the node 13 | shell: kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets 14 | when: rebuild | bool 15 | ignore_errors: True 16 | with_items: "{{ groups['k8s_nodes'] }}" 17 | 18 | - name: Delete node 19 | shell: kubectl delete node {{ item }} 20 | when: rebuild | bool 21 | ignore_errors: True 22 | with_items: "{{ groups['k8s_nodes'] }}" 23 | 24 | - name: Reset k8s 25 | hosts: k8s_master 26 | become: true 27 | become_method: sudo 28 | gather_facts: false 29 | 30 | vars: 31 | rebuild: false 32 | 33 | tasks: 34 | - name: Reset 35 | shell: kubeadm reset 36 | when: rebuild | bool 37 | ignore_errors: True -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/cfssl/defaults/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | #Specifies the version of CFSSL toolkit we want to download and use 3 | cfssl_version: "R1.2" -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/cfssl/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Download statically linked cfssl binary 3 | get_url: 4 | url: https://pkg.cfssl.org/{{cfssl_version}}/cfssl_linux-amd64 5 | dest: /usr/local/bin/cfssl 6 | mode: 0755 7 | tags: 8 | - cfssl 9 | 10 | - name: Download statically linked cfssljson binary 11 | get_url: 12 | url: https://pkg.cfssl.org/{{cfssl_version}}/cfssljson_linux-amd64 13 | dest: /usr/local/bin/cfssljson 14 | mode: 0755 15 | tags: 16 | - cfssl -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | docker_version: 17.03.2~ce-0~ubuntu-xenial -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/docker/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install packages 3 | apt: name={{ item }} state=present update_cache=yes 4 | with_items: 5 | - apt-transport-https 6 | - ca-certificates 7 | - curl 8 | 9 | - name: Add Docker’s official GPG key 10 | apt_key: 11 | url: https://download.docker.com/linux/ubuntu/gpg 12 | state: present 13 | 14 | - name: Set up the stable repository 15 | apt_repository: repo='deb https://download.docker.com/linux/ubuntu xenial stable' state=present 16 | 17 | - name: Install docker-ce 18 | apt: 19 | name: docker-ce={{ docker_version }} 20 | state: present 21 | update_cache: yes -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/etcd/defaults/main.yaml: -------------------------------------------------------------------------------- 1 | etcd_version: "v3.1.12" 2 | etcd_data_dir: "/var/lib/etcd" -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/etcd/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Download etcd binaries 2 | shell: curl -sSL https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/ 3 | args: 4 | creates: /usr/local/bin/etcd 5 | 6 | - name: Remove data dir for etcd 7 | file: 8 | path: "{{ etcd_data_dir }}" 9 | state: absent 10 | 11 | - name: Create data dir for etcd 12 | file: 13 | path: "{{ etcd_data_dir }}" 14 | state: directory 15 | 16 | - name: Copy the systemd unit file 17 | template: 18 | src: etc/systemd/system/etcd.service.j2 19 | dest: /etc/systemd/system/etcd.service 20 | 21 | - name: Start etcd service 22 | systemd: 23 | name: etcd 24 | state: started 25 | daemon_reload: yes -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/etcd/templates/etc/systemd/system/etcd.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=etcd 3 | Documentation=https://github.com/coreos/etcd 4 | Conflicts=etcd.service 5 | Conflicts=etcd2.service 6 | 7 | [Service] 8 | Type=notify 9 | Restart=on-failure 10 | RestartSec=5s 11 | LimitNOFILE=40000 12 | TimeoutStartSec=0 13 | 14 | {% set local_ip = hostvars[inventory_hostname]["ansible_customized_private_ip"] %} 15 | {% set local_host = inventory_hostname + "=https://" + local_ip + ":2380" %} 16 | {% set init_list = [local_host] %} 17 | {%- for host in groups.k8s_master|difference([inventory_hostname]) -%} 18 | {% set init_str = host + "=https://" + hostvars[host]["ansible_customized_private_ip"] + ":2380" %} 19 | {%- if init_list.append(init_str) -%} {%- endif -%} 20 | {%- endfor -%} 21 | 22 | ExecStart=/usr/local/bin/etcd \ 23 | --name="{{ inventory_hostname }}" \ 24 | --data-dir="{{ etcd_data_dir }}" \ 25 | --listen-client-urls="https://{{ local_ip }}:2379,https://127.0.0.1:2379" \ 26 | --advertise-client-urls="https://{{ local_ip }}:2379" \ 27 | --listen-peer-urls="https://{{ local_ip }}:2380" \ 28 | --initial-advertise-peer-urls="https://{{ local_ip }}:2380" \ 29 | --cert-file="/etc/kubernetes/pki/etcd/server.pem" \ 30 | --key-file="/etc/kubernetes/pki/etcd/server-key.pem" \ 31 | --client-cert-auth="true" \ 32 | --trusted-ca-file="/etc/kubernetes/pki/etcd/ca.pem" \ 33 | --peer-cert-file="/etc/kubernetes/pki/etcd/peer.pem" \ 34 | --peer-key-file="/etc/kubernetes/pki/etcd/peer-key.pem" \ 35 | --peer-client-cert-auth="true" \ 36 | --peer-trusted-ca-file="/etc/kubernetes/pki/etcd/ca.pem"\ 37 | --initial-cluster="{{ init_list | join(",") }}" \ 38 | --initial-cluster-token="my-etcd-token" \ 39 | --initial-cluster-state="new" \ 40 | --heartbeat-interval=1000 \ 41 | --election-timeout=5000 42 | 43 | [Install] 44 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_certs/defaults/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_certs/files/ca-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "43800h" 5 | }, 6 | "profiles": { 7 | "server": { 8 | "expiry": "43800h", 9 | "usages": [ 10 | "signing", 11 | "key encipherment", 12 | "server auth", 13 | "client auth" 14 | ] 15 | }, 16 | "client": { 17 | "expiry": "43800h", 18 | "usages": [ 19 | "signing", 20 | "key encipherment", 21 | "client auth" 22 | ] 23 | }, 24 | "peer": { 25 | "expiry": "43800h", 26 | "usages": [ 27 | "signing", 28 | "key encipherment", 29 | "server auth", 30 | "client auth" 31 | ] 32 | } 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_certs/files/ca-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "etcd", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | } 7 | } -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_certs/files/client.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "client", 3 | "key": { 4 | "algo": "ecdsa", 5 | "size": 256 6 | } 7 | } -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_certs/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Generate CA on local 2 | local_action: shell chdir={{ role_path }}/files creates={{ role_path }}/files/ca.pem cfssl gencert -initca ca-csr.json | cfssljson -bare ca 3 | run_once: True 4 | 5 | - name: Generate etcd client cert on local 6 | local_action: shell chdir={{ role_path }}/files creates={{ role_path }}/files/client.pem cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client 7 | run_once: True 8 | 9 | - name: Copy all the existing certs to remote 10 | copy: 11 | src: "{{ role_path }}/files/" 12 | dest: /etc/kubernetes/pki/etcd/ 13 | 14 | - name: Copy the server cert config to remote 15 | template: 16 | src: config.json.j2 17 | dest: /etc/kubernetes/pki/etcd/config.json 18 | 19 | - name: Generate etcd server cert 20 | shell: cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server 21 | args: 22 | chdir: /etc/kubernetes/pki/etcd/ 23 | 24 | - name: Generate etcd peer cert 25 | shell: cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer 26 | args: 27 | chdir: /etc/kubernetes/pki/etcd/ -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_certs/templates/config.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "{{ ansible_hostname }}", 3 | "hosts": [ 4 | "{{ ansible_hostname }}", 5 | "{{ ansible_customized_private_ip }}" 6 | ], 7 | "key": { 8 | "algo": "ecdsa", 9 | "size": 256 10 | }, 11 | "names": [ 12 | { 13 | "C": "NZ", 14 | "ST": "Wellington", 15 | "L": "Wellington" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_cli/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for k8s_cli -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_cli/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_cli/tasks/install_kube_prompt_tasks.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create temporary directory 3 | file: 4 | state: directory 5 | path: ~/kube-prompt 6 | register: dir 7 | 8 | - name: Download kube-prompt 9 | get_url: 10 | url: https://github.com/c-bata/kube-prompt/releases/download/v1.0.3/kube-prompt_v1.0.3_linux_amd64.zip 11 | dest: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" 12 | force: no 13 | 14 | - name: Unarchive kube-prompt 15 | unarchive: 16 | src: "{{ dir.path }}/kube-prompt_v1.0.3_linux_amd64.zip" 17 | dest: "{{ dir.path }}" 18 | remote_src: yes 19 | creates: "{{ dir.path }}/kube-prompt" 20 | 21 | - name: copy kube-prompt to /usr/bin 22 | copy: 23 | src: "{{ dir.path }}/kube-prompt" 24 | dest: /usr/bin/kube-prompt 25 | remote_src: yes 26 | force: no 27 | mode: 0755 28 | 29 | - name: create alias for kube-prompt 30 | lineinfile: 31 | path: /etc/bash.bashrc 32 | create: true 33 | line: 'alias kp="/usr/bin/kube-prompt"' 34 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/k8s_cli/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add Kubernetes GPG key 3 | apt_key: 4 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 5 | state: present 6 | 7 | - name: Set up Kubernetes repository 8 | apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present 9 | 10 | - name: Install kube CLIs 11 | apt: 12 | name: "{{ item }}" 13 | state: present 14 | update_cache: yes 15 | with_items: 16 | - kubelet=1.10.4-00 17 | - kubectl=1.10.4-00 18 | - kubeadm=1.10.4-00 19 | 20 | # - import_tasks: install_kube_prompt_tasks.yml 21 | 22 | - name: create alias for cli 23 | lineinfile: 24 | path: /etc/bash.bashrc 25 | create: true 26 | line: 'alias kc="/usr/bin/kubectl"' 27 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for kube_master 3 | master_0: "true" 4 | auth_url: http://192.168.121.15/identity/v3/ 5 | user_id: 3fd32295751a41aca601021eba681a29 6 | password: password 7 | tenant_id: 84ce53873abe4d22a6f2491879e76048 8 | region: RegionOne 9 | subnet_id: 1740cd57-62ab-4f7f-a853-2038561060d1 10 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Copy k8s certs to slave masters 3 | copy: 4 | src: /tmp/k8s_certs/ 5 | dest: /etc/kubernetes/pki/ 6 | when: not (master_0 | bool) 7 | 8 | - name: sleep for 300 seconds to wait for etcd service to be healthy(hopefully) 9 | wait_for: timeout=300 10 | 11 | - name: Prepare kubeadm config 12 | template: 13 | src: config.yaml.j2 14 | dest: "{{ ansible_env.HOME }}/kubeadm.conf" 15 | 16 | - name: Prepare openstack config file 17 | template: 18 | src: cloud-config.j2 19 | dest: /etc/kubernetes/cloud-config 20 | 21 | - name: Modify kubelet arguments 22 | shell: sed -i -E 's/(.*)KUBELET_KUBECONFIG_ARGS=(.*)$/\1KUBELET_KUBECONFIG_ARGS=--cloud-provider=openstack --cloud-config=\/etc\/kubernetes\/cloud-config \2/' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 23 | 24 | - name: restart kubelet service 25 | systemd: 26 | state: restarted 27 | daemon_reload: yes 28 | name: kubelet 29 | 30 | - name: Init master 31 | shell: kubeadm init --config {{ ansible_env.HOME }}/kubeadm.conf 32 | register: kubeadm_init 33 | 34 | - fail: 35 | msg: "kubeadm init command failed." 36 | when: kubeadm_init.stdout.find ("kubeadm join") == -1 37 | 38 | - name: Store kubeadm join string 39 | set_fact: 40 | join_str: "{{ kubeadm_init.stdout | regex_search('kubeadm join(.*)$') }}" 41 | 42 | - name: Prepare kube config 43 | shell: mkdir -p {{ item }}/.kube && cp -a /etc/kubernetes/admin.conf {{ item }}/.kube/config 44 | with_items: 45 | - "{{ ansible_env.HOME }}" 46 | - /home/ubuntu 47 | 48 | - name: Allow pod on master 49 | shell: kubectl taint nodes {{ inventory_hostname }} node-role.kubernetes.io/master- 50 | 51 | - name: Copy k8s certs to local 52 | fetch: 53 | src: "{{ item }}" 54 | dest: /tmp/k8s_certs/ 55 | flat: yes 56 | with_items: 57 | - /etc/kubernetes/pki/ca.crt 58 | - /etc/kubernetes/pki/ca.key 59 | - /etc/kubernetes/pki/sa.pub 60 | - /etc/kubernetes/pki/sa.key 61 | when: master_0 | bool -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_master/templates/cloud-config.j2: -------------------------------------------------------------------------------- 1 | [Global] 2 | auth-url={{ auth_url }} 3 | user-id={{ user_id }} 4 | password={{ password }} 5 | tenant-id={{ tenant_id }} 6 | region={{ region }} 7 | [LoadBalancer] 8 | use-octavia=yes 9 | subnet-id={{ subnet_id }} 10 | create-monitor=no 11 | [BlockStorage] 12 | bs-version=v2 13 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_master/templates/config.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kubeadm.k8s.io/v1alpha1 2 | kind: MasterConfiguration 3 | api: 4 | advertiseAddress: {{ hostvars["localhost"]["vip_address"] }} 5 | controlPlaneEndpoint: {{ hostvars["localhost"]["vip_address"] }} 6 | etcd: 7 | endpoints: 8 | - https://{{ ansible_customized_private_ip }}:2379 9 | {% for host in groups.k8s_master|difference([inventory_hostname]) %} 10 | - https://{{ hostvars[host]["ansible_customized_private_ip"] }}:2379 11 | {% endfor %} 12 | caFile: /etc/kubernetes/pki/etcd/ca.pem 13 | certFile: /etc/kubernetes/pki/etcd/client.pem 14 | keyFile: /etc/kubernetes/pki/etcd/client-key.pem 15 | networking: 16 | podSubnet: 192.168.0.0/16 17 | apiServerCertSANs: 18 | - {{ hostvars["localhost"]["vip_address"] }} 19 | - {{ ansible_customized_private_ip }} 20 | apiServerExtraArgs: 21 | apiserver-count: "3" 22 | cloudProvider: openstack 23 | kubernetesVersion: 1.10.0 -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_node/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for kube_master 3 | auth_url: http://192.168.121.15/identity/v3/ 4 | user_id: 3fd32295751a41aca601021eba681a29 5 | password: password 6 | tenant_id: 84ce53873abe4d22a6f2491879e76048 7 | region: RegionOne 8 | subnet_id: 1740cd57-62ab-4f7f-a853-2038561060d1 9 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare openstack config file 3 | template: 4 | src: cloud-config.j2 5 | dest: /etc/kubernetes/cloud-config 6 | 7 | - name: Modify kubelet arguments 8 | shell: sed -i -E 's/(.*)KUBELET_KUBECONFIG_ARGS=(.*)$/\1KUBELET_KUBECONFIG_ARGS=--cloud-provider=openstack --cloud-config=\/etc\/kubernetes\/cloud-config \2/' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 9 | 10 | - name: restart kubelet service 11 | systemd: 12 | state: restarted 13 | daemon_reload: yes 14 | name: kubelet 15 | 16 | - name: Join to cluster 17 | shell: "{{ hostvars[groups['k8s_master_0'][0]]['join_str'] }}" 18 | -------------------------------------------------------------------------------- /installation/ansible/version_4/roles/kube_node/templates/cloud-config.j2: -------------------------------------------------------------------------------- 1 | [Global] 2 | auth-url={{ auth_url }} 3 | user-id={{ user_id }} 4 | password={{ password }} 5 | tenant-id={{ tenant_id }} 6 | region={{ region }} 7 | [LoadBalancer] 8 | use-octavia=yes 9 | subnet-id={{ subnet_id }} 10 | create-monitor=no 11 | [BlockStorage] 12 | bs-version=v2 13 | -------------------------------------------------------------------------------- /installation/ansible/version_4/site.yml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | # Usage: 3 | # source openrc 4 | # ansible-playbook site.yml -e "rebuild=false" 5 | # Don't forget to delete the ports when clearing up. 6 | --- 7 | - name: Deploy and prepare instances in OpenStack 8 | hosts: localhost 9 | 10 | vars: 11 | image: 9136e86b-14d9-4127-b3dd-f1cdb18b0dc6 12 | network: network1 13 | subnet: subnet 14 | public_network: public 15 | key_name: lingxian_pc 16 | private_key: /home/vagrant/.ssh/id_rsa 17 | flavor: 2ba44887-5fe3-44cc-8bca-e2b60a206a66 18 | boot_from_vol: false 19 | security_groups: 20 | - default 21 | node_prefix: lingxian-k8s 22 | # Mater hostname must contain 'master', node hostname contains 'node' 23 | # Make sure the vm name complies with k8s naming convention, should not 24 | # contain '_' 25 | node_names: 26 | - master-0 27 | - master-1 28 | - master-2 29 | - node-1 30 | master_node: "{{ node_prefix }}-master-0" 31 | 32 | tasks: 33 | # Create ports separately in order to setup allowed_address_pairs 34 | - name: Create ports 35 | os_port: 36 | state: present 37 | name: "{{ node_prefix }}-{{ item }}" 38 | network: "{{ network }}" 39 | security_groups: "{{ security_groups }}" 40 | allowed_address_pairs: 41 | - ip_address: 192.168.0.0/16 42 | with_items: "{{ node_names }}" 43 | 44 | - name: Create VMs 45 | os_server: 46 | state: present 47 | name: "{{ node_prefix }}-{{ item }}" 48 | boot_from_volume: "{{ boot_from_vol | bool }}" 49 | image: "{{ image }}" 50 | volume_size: 15 51 | key_name: "{{ key_name }}" 52 | nics: port-name={{ node_prefix }}-{{ item }} 53 | flavor: "{{ flavor }}" 54 | terminate_volume: true 55 | auto_ip: false 56 | with_items: "{{ node_names }}" 57 | 58 | - name: Associate floating ip if needed 59 | os_floating_ip: 60 | state: present 61 | reuse: yes 62 | network: "{{ public_network }}" 63 | server: "{{ node_prefix }}-{{ item }}" 64 | with_items: "{{ node_names }}" 65 | when: public_network is defined 66 | 67 | - name: Gather facts about vms 68 | os_server_facts: 69 | server: "{{ node_prefix }}-*" 70 | 71 | - name: Wait for ssh ready 72 | wait_for: port=22 host="{{ item.private_v4 if public_network is undefined else item.public_v4 }}" search_regex=OpenSSH timeout=300 delay=5 73 | with_items: "{{ openstack_servers }}" 74 | 75 | - name: Add VMs to inventory 76 | add_host: 77 | name: "{{ item.name }}" 78 | groups: k8s_hosts 79 | ansible_user: ubuntu 80 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 81 | ansible_ssh_private_key_file: "{{ private_key }}" 82 | ansible_ssh_common_args: "-o StrictHostKeyChecking=no" 83 | ansible_customized_private_ip: "{{ item.private_v4 }}" 84 | with_items: "{{ openstack_servers }}" 85 | 86 | - name: Setup master group 87 | add_host: 88 | name: "{{ item.name }}" 89 | groups: k8s_master 90 | ansible_user: ubuntu 91 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 92 | ansible_ssh_private_key_file: "{{ private_key }}" 93 | ansible_customized_private_ip: "{{ item.private_v4 }}" 94 | when: item.name | regex_search('(master)') 95 | with_items: "{{ openstack_servers }}" 96 | 97 | - name: Setup master 0 group 98 | add_host: 99 | name: "{{ item.name }}" 100 | groups: k8s_master_0 101 | ansible_user: ubuntu 102 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 103 | ansible_ssh_private_key_file: "{{ private_key }}" 104 | ansible_customized_private_ip: "{{ item.private_v4 }}" 105 | when: '"master-0" in item.name' 106 | with_items: "{{ openstack_servers }}" 107 | 108 | - name: Setup slave masters group 109 | add_host: 110 | name: "{{ item.name }}" 111 | groups: k8s_slave_masters 112 | ansible_user: ubuntu 113 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 114 | ansible_ssh_private_key_file: "{{ private_key }}" 115 | ansible_customized_private_ip: "{{ item.private_v4 }}" 116 | when: item.name | regex_search('(master)') and "master-0" not in item.name 117 | with_items: "{{ openstack_servers }}" 118 | 119 | - name: Setup nodes group 120 | add_host: 121 | name: "{{ item.name }}" 122 | groups: k8s_nodes 123 | ansible_user: ubuntu 124 | ansible_host: "{{ item.private_v4 if public_network is undefined else item.public_v4 }}" 125 | ansible_ssh_private_key_file: "{{ private_key }}" 126 | ansible_customized_private_ip: "{{ item.private_v4 }}" 127 | when: item.name | regex_search('(node)') 128 | with_items: "{{ openstack_servers }}" 129 | 130 | - name: Delete load balancer if exists 131 | shell: openstack loadbalancer delete k8s_master_lb --cascade 132 | ignore_errors: True 133 | 134 | - name: Create load balancer 135 | shell: openstack loadbalancer create --name k8s_master_lb --vip-subnet-id {{ subnet }} -f json 136 | register: k8s_lb 137 | 138 | - name: Wait for the load balancer to be active 139 | shell: openstack loadbalancer show {{ (k8s_lb.stdout | from_json)['id'] }} -f value -c provisioning_status 140 | register: lb_status 141 | until: lb_status.stdout == "ACTIVE" 142 | retries: 20 143 | delay: 15 144 | 145 | - name: Get vip address 146 | set_fact: vip_address={{ (k8s_lb.stdout | from_json)['vip_address'] }} 147 | 148 | - name: Create listener 149 | shell: openstack loadbalancer listener create --name k8s_listener --protocol TCP --protocol-port 6443 {{ (k8s_lb.stdout | from_json)['id'] }} -f value -c id 150 | register: listener_id 151 | 152 | - name: Wait for the load balancer to be active 153 | shell: openstack loadbalancer show {{ (k8s_lb.stdout | from_json)['id'] }} -f value -c provisioning_status 154 | register: lb_status 155 | until: lb_status.stdout == "ACTIVE" 156 | retries: 10 157 | delay: 5 158 | 159 | - name: Create pool 160 | shell: openstack loadbalancer pool create --protocol TCP --listener {{ listener_id.stdout }} --lb-algorithm ROUND_ROBIN -f value -c id 161 | register: pool_id 162 | 163 | - name: Wait for the load balancer to be active 164 | shell: openstack loadbalancer show {{ (k8s_lb.stdout | from_json)['id'] }} -f value -c provisioning_status 165 | register: lb_status 166 | until: lb_status.stdout == "ACTIVE" 167 | retries: 10 168 | delay: 5 169 | 170 | - name: Add master 0 to the pool 171 | shell: openstack loadbalancer member create --address {{ hostvars[master_node]["ansible_customized_private_ip"] }} --subnet-id {{ subnet }} --protocol-port 6443 {{ pool_id.stdout }} 172 | 173 | - name: Install cfssl tool on local 174 | hosts: localhost 175 | gather_facts: false 176 | 177 | roles: 178 | - cfssl 179 | 180 | - name: Prepare k8s hosts 181 | hosts: k8s_hosts 182 | become: true 183 | become_method: sudo 184 | gather_facts: false 185 | 186 | pre_tasks: 187 | - name: Install python 2 188 | raw: test -e /usr/bin/python || (apt -y update && apt -y upgrade && apt install -y python-minimal) 189 | 190 | - name: Install pip 191 | shell: which pip || (curl https://bootstrap.pypa.io/get-pip.py | python - && rm -f get-pip.py) 192 | 193 | - name: Gather facts after python installation 194 | setup: filter=ansible_* 195 | 196 | - name: Install packages 197 | apt: name={{ item }} state=present update_cache=yes 198 | with_items: 199 | - ansible 200 | - unzip 201 | - build-essential 202 | - python-dev 203 | - python-setuptools 204 | - libffi-dev 205 | - libxslt1-dev 206 | - libxml2-dev 207 | - libyaml-dev 208 | - libssl-dev 209 | - zlib1g-dev 210 | 211 | - name: Install pypi libraries 212 | pip: name={{ item }} 213 | ignore_errors: True 214 | with_items: 215 | - httpie 216 | - kube-shell 217 | 218 | - name: Build hosts file 219 | lineinfile: dest=/etc/hosts line="{{ ansible_customized_private_ip }} {{ ansible_hostname }}" state=present 220 | 221 | roles: 222 | - docker 223 | - k8s_cli 224 | - cfssl 225 | 226 | # This must run before we generate certs because it will delete the cert folder. 227 | - import_playbook: reset_k8s.yaml rebuild={{ rebuild }} 228 | 229 | - name: Certs and etcd 230 | hosts: k8s_master 231 | become: true 232 | gather_facts: true 233 | 234 | pre_tasks: 235 | - name: Make sure masters can reach each other by hostname 236 | lineinfile: dest=/etc/hosts line="{{ hostvars[item]["ansible_customized_private_ip"] }} {{ item }} " state=present 237 | loop: "{{ groups['k8s_master'] }}" 238 | 239 | roles: 240 | - k8s_certs 241 | - etcd 242 | 243 | - name: Set up master 0 244 | hosts: k8s_master_0 245 | become: true 246 | become_method: sudo 247 | 248 | roles: 249 | - { role: kube_master, master_0: True } 250 | 251 | - name: Set up slave masters 252 | hosts: k8s_slave_masters 253 | become: true 254 | become_method: sudo 255 | 256 | roles: 257 | - { role: kube_master, master_0: False } 258 | 259 | - name: Add slave masters to load balancer 260 | hosts: localhost 261 | 262 | tasks: 263 | - name: Add master 0 to the pool 264 | shell: openstack loadbalancer member create --address {{ hostvars[item]["ansible_customized_private_ip"] }} --subnet-id {{ subnet }} --protocol-port 6443 {{ pool_id.stdout }}; sleep 15 265 | loop: "{{ groups['k8s_slave_masters'] }}" 266 | 267 | - name: Install calico on master 0 268 | hosts: k8s_master_0 269 | become: true 270 | become_method: sudo 271 | 272 | tasks: 273 | - name: Download calico yaml 274 | get_url: 275 | url: https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml 276 | dest: "{{ ansible_env.HOME }}/calico.yaml" 277 | force: no 278 | 279 | - name: Change calico config 280 | replace: 281 | path: "{{ ansible_env.HOME }}/calico.yaml" 282 | regexp: 'value: "always"' 283 | replace: 'value: "off"\n - name: CALICO_IPV4POOL_NAT_OUTGOING\n value: "true"' 284 | backup: yes 285 | 286 | - name: Install network plugin 287 | shell: kubectl apply -f {{ ansible_env.HOME }}/calico.yaml 288 | 289 | - name: Set up k8s node 290 | hosts: k8s_nodes 291 | become: true 292 | become_method: sudo 293 | gather_facts: false 294 | 295 | roles: 296 | - kube_node 297 | -------------------------------------------------------------------------------- /test/test_loadbalancer_type_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Create a service of LoadBalancer type 4 | cat <> /etc/hosts 58 | $ip $url 59 | $ip $url1 60 | $ip $url2 61 | EOF 62 | curl http://$url/index 63 | curl http://$url1/index 64 | curl http://$url2 65 | --------------------------------------------------------------------------------