├── LICENSE.md ├── README.md └── aws ├── fargate └── cloud-custodian │ ├── Dockerfile │ ├── Makefile │ ├── docker-compose.yml │ ├── ecs.tf │ ├── iam.tf │ ├── outputs.tf │ ├── rules │ └── offhours.yml │ ├── run.sh │ ├── sources.tf │ └── variables.tf └── lambda ├── cloudformation └── ebs_snapshotter_lambda.json ├── elasticache_snapshot_copier ├── .gitignore ├── .terraform-version ├── Makefile ├── README.md ├── docker-compose.yml ├── elasticache_snapshot_copier.tf ├── iam.tf ├── lambda_function.py ├── tox.ini └── variables.tf ├── rds_logs_to_s3 ├── .gitignore ├── .python-version ├── .terraform-version ├── Makefile ├── README.md ├── docker-compose.yml ├── iam.tf ├── lambda_function.py ├── outputs.tf ├── rds_logs_to_s3.tf ├── requirements.txt ├── resources │ └── event.json ├── sources.tf └── test.bats ├── rds_snapshot_copier ├── .gitignore ├── .terraform-version ├── Makefile ├── README.md ├── docker-compose.yml ├── iam.tf ├── lambda_function.py ├── rds_snapshot_copier.tf └── variables.tf ├── redshift_snapshot_copier ├── .gitignore ├── .terraform-version ├── Makefile ├── README.md ├── docker-compose.yml ├── iam.tf ├── lambda_function.py ├── redshift_snapshot_copier.tf └── variables.tf └── s3_to_elasticsearch ├── .gitignore ├── .python-version ├── .terraform-version ├── Makefile ├── README.md ├── docker-compose.yml ├── iam.tf ├── lambda_function.py ├── outputs.tf ├── requirements.txt ├── resources └── event.json ├── s3_to_elasticsearch.tf ├── sources.tf └── test.bats /LICENSE.md: -------------------------------------------------------------------------------- 1 | # GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 [Free Software Foundation, Inc.](http://fsf.org/) 5 | 6 | Everyone is permitted to copy and distribute verbatim copies of this license 7 | document, but changing it is not allowed. 8 | 9 | ## Preamble 10 | 11 | The GNU General Public License is a free, copyleft license for software and 12 | other kinds of works. 13 | 14 | The licenses for most software and other practical works are designed to take 15 | away your freedom to share and change the works. By contrast, the GNU General 16 | Public License is intended to guarantee your freedom to share and change all 17 | versions of a program--to make sure it remains free software for all its users. 18 | We, the Free Software Foundation, use the GNU General Public License for most 19 | of our software; it applies also to any other work released this way by its 20 | authors. You can apply it to your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not price. Our 23 | General Public Licenses are designed to make sure that you have the freedom to 24 | distribute copies of free software (and charge for them if you wish), that you 25 | receive source code or can get it if you want it, that you can change the 26 | software or use pieces of it in new free programs, and that you know you can do 27 | these things. 28 | 29 | To protect your rights, we need to prevent others from denying you these rights 30 | or asking you to surrender the rights. Therefore, you have certain 31 | responsibilities if you distribute copies of the software, or if you modify it: 32 | responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether gratis or for 35 | a fee, you must pass on to the recipients the same freedoms that you received. 36 | You must make sure that they, too, receive or can get the source code. And you 37 | must show them these terms so they know their rights. 38 | 39 | Developers that use the GNU GPL protect your rights with two steps: 40 | 41 | 1. assert copyright on the software, and 42 | 2. offer you this License giving you legal permission to copy, distribute 43 | and/or modify it. 44 | 45 | For the developers' and authors' protection, the GPL clearly explains that 46 | there is no warranty for this free software. For both users' and authors' sake, 47 | the GPL requires that modified versions be marked as changed, so that their 48 | problems will not be attributed erroneously to authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run modified 51 | versions of the software inside them, although the manufacturer can do so. This 52 | is fundamentally incompatible with the aim of protecting users' freedom to 53 | change the software. The systematic pattern of such abuse occurs in the area of 54 | products for individuals to use, which is precisely where it is most 55 | unacceptable. Therefore, we have designed this version of the GPL to prohibit 56 | the practice for those products. If such problems arise substantially in other 57 | domains, we stand ready to extend this provision to those domains in future 58 | versions of the GPL, as needed to protect the freedom of users. 59 | 60 | Finally, every program is threatened constantly by software patents. States 61 | should not allow patents to restrict development and use of software on 62 | general-purpose computers, but in those that do, we wish to avoid the special 63 | danger that patents applied to a free program could make it effectively 64 | proprietary. To prevent this, the GPL assures that patents cannot be used to 65 | render the program non-free. 66 | 67 | The precise terms and conditions for copying, distribution and modification 68 | follow. 69 | 70 | ## TERMS AND CONDITIONS 71 | 72 | ### 0. Definitions. 73 | 74 | *This License* refers to version 3 of the GNU General Public License. 75 | 76 | *Copyright* also means copyright-like laws that apply to other kinds of works, 77 | such as semiconductor masks. 78 | 79 | *The Program* refers to any copyrightable work licensed under this License. 80 | Each licensee is addressed as *you*. *Licensees* and *recipients* may be 81 | individuals or organizations. 82 | 83 | To *modify* a work means to copy from or adapt all or part of the work in a 84 | fashion requiring copyright permission, other than the making of an exact copy. 85 | The resulting work is called a *modified version* of the earlier work or a work 86 | *based on* the earlier work. 87 | 88 | A *covered work* means either the unmodified Program or a work based on the 89 | Program. 90 | 91 | To *propagate* a work means to do anything with it that, without permission, 92 | would make you directly or secondarily liable for infringement under applicable 93 | copyright law, except executing it on a computer or modifying a private copy. 94 | Propagation includes copying, distribution (with or without modification), 95 | making available to the public, and in some countries other activities as well. 96 | 97 | To *convey* a work means any kind of propagation that enables other parties to 98 | make or receive copies. Mere interaction with a user through a computer 99 | network, with no transfer of a copy, is not conveying. 100 | 101 | An interactive user interface displays *Appropriate Legal Notices* to the 102 | extent that it includes a convenient and prominently visible feature that 103 | 104 | 1. displays an appropriate copyright notice, and 105 | 2. tells the user that there is no warranty for the work (except to the 106 | extent that warranties are provided), that licensees may convey the work 107 | under this License, and how to view a copy of this License. 108 | 109 | If the interface presents a list of user commands or options, such as a menu, a 110 | prominent item in the list meets this criterion. 111 | 112 | ### 1. Source Code. 113 | 114 | The *source code* for a work means the preferred form of the work for making 115 | modifications to it. *Object code* means any non-source form of a work. 116 | 117 | A *Standard Interface* means an interface that either is an official standard 118 | defined by a recognized standards body, or, in the case of interfaces specified 119 | for a particular programming language, one that is widely used among developers 120 | working in that language. 121 | 122 | The *System Libraries* of an executable work include anything, other than the 123 | work as a whole, that (a) is included in the normal form of packaging a Major 124 | Component, but which is not part of that Major Component, and (b) serves only 125 | to enable use of the work with that Major Component, or to implement a Standard 126 | Interface for which an implementation is available to the public in source code 127 | form. A *Major Component*, in this context, means a major essential component 128 | (kernel, window system, and so on) of the specific operating system (if any) on 129 | which the executable work runs, or a compiler used to produce the work, or an 130 | object code interpreter used to run it. 131 | 132 | The *Corresponding Source* for a work in object code form means all the source 133 | code needed to generate, install, and (for an executable work) run the object 134 | code and to modify the work, including scripts to control those activities. 135 | However, it does not include the work's System Libraries, or general-purpose 136 | tools or generally available free programs which are used unmodified in 137 | performing those activities but which are not part of the work. For example, 138 | Corresponding Source includes interface definition files associated with source 139 | files for the work, and the source code for shared libraries and dynamically 140 | linked subprograms that the work is specifically designed to require, such as 141 | by intimate data communication or control flow between those subprograms and 142 | other parts of the work. 143 | 144 | The Corresponding Source need not include anything that users can regenerate 145 | automatically from other parts of the Corresponding Source. 146 | 147 | The Corresponding Source for a work in source code form is that same work. 148 | 149 | ### 2. Basic Permissions. 150 | 151 | All rights granted under this License are granted for the term of copyright on 152 | the Program, and are irrevocable provided the stated conditions are met. This 153 | License explicitly affirms your unlimited permission to run the unmodified 154 | Program. The output from running a covered work is covered by this License only 155 | if the output, given its content, constitutes a covered work. This License 156 | acknowledges your rights of fair use or other equivalent, as provided by 157 | copyright law. 158 | 159 | You may make, run and propagate covered works that you do not convey, without 160 | conditions so long as your license otherwise remains in force. You may convey 161 | covered works to others for the sole purpose of having them make modifications 162 | exclusively for you, or provide you with facilities for running those works, 163 | provided that you comply with the terms of this License in conveying all 164 | material for which you do not control copyright. Those thus making or running 165 | the covered works for you must do so exclusively on your behalf, under your 166 | direction and control, on terms that prohibit them from making any copies of 167 | your copyrighted material outside their relationship with you. 168 | 169 | Conveying under any other circumstances is permitted solely under the 170 | conditions stated below. Sublicensing is not allowed; section 10 makes it 171 | unnecessary. 172 | 173 | ### 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 174 | 175 | No covered work shall be deemed part of an effective technological measure 176 | under any applicable law fulfilling obligations under article 11 of the WIPO 177 | copyright treaty adopted on 20 December 1996, or similar laws prohibiting or 178 | restricting circumvention of such measures. 179 | 180 | When you convey a covered work, you waive any legal power to forbid 181 | circumvention of technological measures to the extent such circumvention is 182 | effected by exercising rights under this License with respect to the covered 183 | work, and you disclaim any intention to limit operation or modification of the 184 | work as a means of enforcing, against the work's users, your or third parties' 185 | legal rights to forbid circumvention of technological measures. 186 | 187 | ### 4. Conveying Verbatim Copies. 188 | 189 | You may convey verbatim copies of the Program's source code as you receive it, 190 | in any medium, provided that you conspicuously and appropriately publish on 191 | each copy an appropriate copyright notice; keep intact all notices stating that 192 | this License and any non-permissive terms added in accord with section 7 apply 193 | to the code; keep intact all notices of the absence of any warranty; and give 194 | all recipients a copy of this License along with the Program. 195 | 196 | You may charge any price or no price for each copy that you convey, and you may 197 | offer support or warranty protection for a fee. 198 | 199 | ### 5. Conveying Modified Source Versions. 200 | 201 | You may convey a work based on the Program, or the modifications to produce it 202 | from the Program, in the form of source code under the terms of section 4, 203 | provided that you also meet all of these conditions: 204 | 205 | - a) The work must carry prominent notices stating that you modified it, and 206 | giving a relevant date. 207 | - b) The work must carry prominent notices stating that it is released under 208 | this License and any conditions added under section 7. This requirement 209 | modifies the requirement in section 4 to *keep intact all notices*. 210 | - c) You must license the entire work, as a whole, under this License to 211 | anyone who comes into possession of a copy. This License will therefore 212 | apply, along with any applicable section 7 additional terms, to the whole 213 | of the work, and all its parts, regardless of how they are packaged. This 214 | License gives no permission to license the work in any other way, but it 215 | does not invalidate such permission if you have separately received it. 216 | - d) If the work has interactive user interfaces, each must display 217 | Appropriate Legal Notices; however, if the Program has interactive 218 | interfaces that do not display Appropriate Legal Notices, your work need 219 | not make them do so. 220 | 221 | A compilation of a covered work with other separate and independent works, 222 | which are not by their nature extensions of the covered work, and which are not 223 | combined with it such as to form a larger program, in or on a volume of a 224 | storage or distribution medium, is called an *aggregate* if the compilation and 225 | its resulting copyright are not used to limit the access or legal rights of the 226 | compilation's users beyond what the individual works permit. Inclusion of a 227 | covered work in an aggregate does not cause this License to apply to the other 228 | parts of the aggregate. 229 | 230 | ### 6. Conveying Non-Source Forms. 231 | 232 | You may convey a covered work in object code form under the terms of sections 4 233 | and 5, provided that you also convey the machine-readable Corresponding Source 234 | under the terms of this License, in one of these ways: 235 | 236 | - a) Convey the object code in, or embodied in, a physical product (including 237 | a physical distribution medium), accompanied by the Corresponding Source 238 | fixed on a durable physical medium customarily used for software 239 | interchange. 240 | - b) Convey the object code in, or embodied in, a physical product (including 241 | a physical distribution medium), accompanied by a written offer, valid for 242 | at least three years and valid for as long as you offer spare parts or 243 | customer support for that product model, to give anyone who possesses the 244 | object code either 245 | 1. a copy of the Corresponding Source for all the software in the product 246 | that is covered by this License, on a durable physical medium 247 | customarily used for software interchange, for a price no more than your 248 | reasonable cost of physically performing this conveying of source, or 249 | 2. access to copy the Corresponding Source from a network server at no 250 | charge. 251 | - c) Convey individual copies of the object code with a copy of the written 252 | offer to provide the Corresponding Source. This alternative is allowed only 253 | occasionally and noncommercially, and only if you received the object code 254 | with such an offer, in accord with subsection 6b. 255 | - d) Convey the object code by offering access from a designated place 256 | (gratis or for a charge), and offer equivalent access to the Corresponding 257 | Source in the same way through the same place at no further charge. You 258 | need not require recipients to copy the Corresponding Source along with the 259 | object code. If the place to copy the object code is a network server, the 260 | Corresponding Source may be on a different server operated by you or a 261 | third party) that supports equivalent copying facilities, provided you 262 | maintain clear directions next to the object code saying where to find the 263 | Corresponding Source. Regardless of what server hosts the Corresponding 264 | Source, you remain obligated to ensure that it is available for as long as 265 | needed to satisfy these requirements. 266 | - e) Convey the object code using peer-to-peer transmission, provided you 267 | inform other peers where the object code and Corresponding Source of the 268 | work are being offered to the general public at no charge under subsection 269 | 6d. 270 | 271 | A separable portion of the object code, whose source code is excluded from the 272 | Corresponding Source as a System Library, need not be included in conveying the 273 | object code work. 274 | 275 | A *User Product* is either 276 | 277 | 1. a *consumer product*, which means any tangible personal property which is 278 | normally used for personal, family, or household purposes, or 279 | 2. anything designed or sold for incorporation into a dwelling. 280 | 281 | In determining whether a product is a consumer product, doubtful cases shall be 282 | resolved in favor of coverage. For a particular product received by a 283 | particular user, *normally used* refers to a typical or common use of that 284 | class of product, regardless of the status of the particular user or of the way 285 | in which the particular user actually uses, or expects or is expected to use, 286 | the product. A product is a consumer product regardless of whether the product 287 | has substantial commercial, industrial or non-consumer uses, unless such uses 288 | represent the only significant mode of use of the product. 289 | 290 | *Installation Information* for a User Product means any methods, procedures, 291 | authorization keys, or other information required to install and execute 292 | modified versions of a covered work in that User Product from a modified 293 | version of its Corresponding Source. The information must suffice to ensure 294 | that the continued functioning of the modified object code is in no case 295 | prevented or interfered with solely because modification has been made. 296 | 297 | If you convey an object code work under this section in, or with, or 298 | specifically for use in, a User Product, and the conveying occurs as part of a 299 | transaction in which the right of possession and use of the User Product is 300 | transferred to the recipient in perpetuity or for a fixed term (regardless of 301 | how the transaction is characterized), the Corresponding Source conveyed under 302 | this section must be accompanied by the Installation Information. But this 303 | requirement does not apply if neither you nor any third party retains the 304 | ability to install modified object code on the User Product (for example, the 305 | work has been installed in ROM). 306 | 307 | The requirement to provide Installation Information does not include a 308 | requirement to continue to provide support service, warranty, or updates for a 309 | work that has been modified or installed by the recipient, or for the User 310 | Product in which it has been modified or installed. Access to a network may be 311 | denied when the modification itself materially and adversely affects the 312 | operation of the network or violates the rules and protocols for communication 313 | across the network. 314 | 315 | Corresponding Source conveyed, and Installation Information provided, in accord 316 | with this section must be in a format that is publicly documented (and with an 317 | implementation available to the public in source code form), and must require 318 | no special password or key for unpacking, reading or copying. 319 | 320 | ### 7. Additional Terms. 321 | 322 | *Additional permissions* are terms that supplement the terms of this License by 323 | making exceptions from one or more of its conditions. Additional permissions 324 | that are applicable to the entire Program shall be treated as though they were 325 | included in this License, to the extent that they are valid under applicable 326 | law. If additional permissions apply only to part of the Program, that part may 327 | be used separately under those permissions, but the entire Program remains 328 | governed by this License without regard to the additional permissions. 329 | 330 | When you convey a copy of a covered work, you may at your option remove any 331 | additional permissions from that copy, or from any part of it. (Additional 332 | permissions may be written to require their own removal in certain cases when 333 | you modify the work.) You may place additional permissions on material, added 334 | by you to a covered work, for which you have or can give appropriate copyright 335 | permission. 336 | 337 | Notwithstanding any other provision of this License, for material you add to a 338 | covered work, you may (if authorized by the copyright holders of that material) 339 | supplement the terms of this License with terms: 340 | 341 | - a) Disclaiming warranty or limiting liability differently from the terms of 342 | sections 15 and 16 of this License; or 343 | - b) Requiring preservation of specified reasonable legal notices or author 344 | attributions in that material or in the Appropriate Legal Notices displayed 345 | by works containing it; or 346 | - c) Prohibiting misrepresentation of the origin of that material, or 347 | requiring that modified versions of such material be marked in reasonable 348 | ways as different from the original version; or 349 | - d) Limiting the use for publicity purposes of names of licensors or authors 350 | of the material; or 351 | - e) Declining to grant rights under trademark law for use of some trade 352 | names, trademarks, or service marks; or 353 | - f) Requiring indemnification of licensors and authors of that material by 354 | anyone who conveys the material (or modified versions of it) with 355 | contractual assumptions of liability to the recipient, for any liability 356 | that these contractual assumptions directly impose on those licensors and 357 | authors. 358 | 359 | All other non-permissive additional terms are considered *further restrictions* 360 | within the meaning of section 10. If the Program as you received it, or any 361 | part of it, contains a notice stating that it is governed by this License along 362 | with a term that is a further restriction, you may remove that term. If a 363 | license document contains a further restriction but permits relicensing or 364 | conveying under this License, you may add to a covered work material governed 365 | by the terms of that license document, provided that the further restriction 366 | does not survive such relicensing or conveying. 367 | 368 | If you add terms to a covered work in accord with this section, you must place, 369 | in the relevant source files, a statement of the additional terms that apply to 370 | those files, or a notice indicating where to find the applicable terms. 371 | 372 | Additional terms, permissive or non-permissive, may be stated in the form of a 373 | separately written license, or stated as exceptions; the above requirements 374 | apply either way. 375 | 376 | ### 8. Termination. 377 | 378 | You may not propagate or modify a covered work except as expressly provided 379 | under this License. Any attempt otherwise to propagate or modify it is void, 380 | and will automatically terminate your rights under this License (including any 381 | patent licenses granted under the third paragraph of section 11). 382 | 383 | However, if you cease all violation of this License, then your license from a 384 | particular copyright holder is reinstated 385 | 386 | - a) provisionally, unless and until the copyright holder explicitly and 387 | finally terminates your license, and 388 | - b) permanently, if the copyright holder fails to notify you of the 389 | violation by some reasonable means prior to 60 days after the cessation. 390 | 391 | Moreover, your license from a particular copyright holder is reinstated 392 | permanently if the copyright holder notifies you of the violation by some 393 | reasonable means, this is the first time you have received notice of violation 394 | of this License (for any work) from that copyright holder, and you cure the 395 | violation prior to 30 days after your receipt of the notice. 396 | 397 | Termination of your rights under this section does not terminate the licenses 398 | of parties who have received copies or rights from you under this License. If 399 | your rights have been terminated and not permanently reinstated, you do not 400 | qualify to receive new licenses for the same material under section 10. 401 | 402 | ### 9. Acceptance Not Required for Having Copies. 403 | 404 | You are not required to accept this License in order to receive or run a copy 405 | of the Program. Ancillary propagation of a covered work occurring solely as a 406 | consequence of using peer-to-peer transmission to receive a copy likewise does 407 | not require acceptance. However, nothing other than this License grants you 408 | permission to propagate or modify any covered work. These actions infringe 409 | copyright if you do not accept this License. Therefore, by modifying or 410 | propagating a covered work, you indicate your acceptance of this License to do 411 | so. 412 | 413 | ### 10. Automatic Licensing of Downstream Recipients. 414 | 415 | Each time you convey a covered work, the recipient automatically receives a 416 | license from the original licensors, to run, modify and propagate that work, 417 | subject to this License. You are not responsible for enforcing compliance by 418 | third parties with this License. 419 | 420 | An *entity transaction* is a transaction transferring control of an 421 | organization, or substantially all assets of one, or subdividing an 422 | organization, or merging organizations. If propagation of a covered work 423 | results from an entity transaction, each party to that transaction who receives 424 | a copy of the work also receives whatever licenses to the work the party's 425 | predecessor in interest had or could give under the previous paragraph, plus a 426 | right to possession of the Corresponding Source of the work from the 427 | predecessor in interest, if the predecessor has it or can get it with 428 | reasonable efforts. 429 | 430 | You may not impose any further restrictions on the exercise of the rights 431 | granted or affirmed under this License. For example, you may not impose a 432 | license fee, royalty, or other charge for exercise of rights granted under this 433 | License, and you may not initiate litigation (including a cross-claim or 434 | counterclaim in a lawsuit) alleging that any patent claim is infringed by 435 | making, using, selling, offering for sale, or importing the Program or any 436 | portion of it. 437 | 438 | ### 11. Patents. 439 | 440 | A *contributor* is a copyright holder who authorizes use under this License of 441 | the Program or a work on which the Program is based. The work thus licensed is 442 | called the contributor's *contributor version*. 443 | 444 | A contributor's *essential patent claims* are all patent claims owned or 445 | controlled by the contributor, whether already acquired or hereafter acquired, 446 | that would be infringed by some manner, permitted by this License, of making, 447 | using, or selling its contributor version, but do not include claims that would 448 | be infringed only as a consequence of further modification of the contributor 449 | version. For purposes of this definition, *control* includes the right to grant 450 | patent sublicenses in a manner consistent with the requirements of this 451 | License. 452 | 453 | Each contributor grants you a non-exclusive, worldwide, royalty-free patent 454 | license under the contributor's essential patent claims, to make, use, sell, 455 | offer for sale, import and otherwise run, modify and propagate the contents of 456 | its contributor version. 457 | 458 | In the following three paragraphs, a *patent license* is any express agreement 459 | or commitment, however denominated, not to enforce a patent (such as an express 460 | permission to practice a patent or covenant not to sue for patent 461 | infringement). To *grant* such a patent license to a party means to make such 462 | an agreement or commitment not to enforce a patent against the party. 463 | 464 | If you convey a covered work, knowingly relying on a patent license, and the 465 | Corresponding Source of the work is not available for anyone to copy, free of 466 | charge and under the terms of this License, through a publicly available 467 | network server or other readily accessible means, then you must either 468 | 469 | 1. cause the Corresponding Source to be so available, or 470 | 2. arrange to deprive yourself of the benefit of the patent license for this 471 | particular work, or 472 | 3. arrange, in a manner consistent with the requirements of this License, to 473 | extend the patent license to downstream recipients. 474 | 475 | *Knowingly relying* means you have actual knowledge that, but for the patent 476 | license, your conveying the covered work in a country, or your recipient's use 477 | of the covered work in a country, would infringe one or more identifiable 478 | patents in that country that you have reason to believe are valid. 479 | 480 | If, pursuant to or in connection with a single transaction or arrangement, you 481 | convey, or propagate by procuring conveyance of, a covered work, and grant a 482 | patent license to some of the parties receiving the covered work authorizing 483 | them to use, propagate, modify or convey a specific copy of the covered work, 484 | then the patent license you grant is automatically extended to all recipients 485 | of the covered work and works based on it. 486 | 487 | A patent license is *discriminatory* if it does not include within the scope of 488 | its coverage, prohibits the exercise of, or is conditioned on the non-exercise 489 | of one or more of the rights that are specifically granted under this License. 490 | You may not convey a covered work if you are a party to an arrangement with a 491 | third party that is in the business of distributing software, under which you 492 | make payment to the third party based on the extent of your activity of 493 | conveying the work, and under which the third party grants, to any of the 494 | parties who would receive the covered work from you, a discriminatory patent 495 | license 496 | 497 | - a) in connection with copies of the covered work conveyed by you (or copies 498 | made from those copies), or 499 | - b) primarily for and in connection with specific products or compilations 500 | that contain the covered work, unless you entered into that arrangement, or 501 | that patent license was granted, prior to 28 March 2007. 502 | 503 | Nothing in this License shall be construed as excluding or limiting any implied 504 | license or other defenses to infringement that may otherwise be available to 505 | you under applicable patent law. 506 | 507 | ### 12. No Surrender of Others' Freedom. 508 | 509 | If conditions are imposed on you (whether by court order, agreement or 510 | otherwise) that contradict the conditions of this License, they do not excuse 511 | you from the conditions of this License. If you cannot convey a covered work so 512 | as to satisfy simultaneously your obligations under this License and any other 513 | pertinent obligations, then as a consequence you may not convey it at all. For 514 | example, if you agree to terms that obligate you to collect a royalty for 515 | further conveying from those to whom you convey the Program, the only way you 516 | could satisfy both those terms and this License would be to refrain entirely 517 | from conveying the Program. 518 | 519 | ### 13. Use with the GNU Affero General Public License. 520 | 521 | Notwithstanding any other provision of this License, you have permission to 522 | link or combine any covered work with a work licensed under version 3 of the 523 | GNU Affero General Public License into a single combined work, and to convey 524 | the resulting work. The terms of this License will continue to apply to the 525 | part which is the covered work, but the special requirements of the GNU Affero 526 | General Public License, section 13, concerning interaction through a network 527 | will apply to the combination as such. 528 | 529 | ### 14. Revised Versions of this License. 530 | 531 | The Free Software Foundation may publish revised and/or new versions of the GNU 532 | General Public License from time to time. Such new versions will be similar in 533 | spirit to the present version, but may differ in detail to address new problems 534 | or concerns. 535 | 536 | Each version is given a distinguishing version number. If the Program specifies 537 | that a certain numbered version of the GNU General Public License *or any later 538 | version* applies to it, you have the option of following the terms and 539 | conditions either of that numbered version or of any later version published by 540 | the Free Software Foundation. If the Program does not specify a version number 541 | of the GNU General Public License, you may choose any version ever published by 542 | the Free Software Foundation. 543 | 544 | If the Program specifies that a proxy can decide which future versions of the 545 | GNU General Public License can be used, that proxy's public statement of 546 | acceptance of a version permanently authorizes you to choose that version for 547 | the Program. 548 | 549 | Later license versions may give you additional or different permissions. 550 | However, no additional obligations are imposed on any author or copyright 551 | holder as a result of your choosing to follow a later version. 552 | 553 | ### 15. Disclaimer of Warranty. 554 | 555 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE 556 | LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER 557 | PARTIES PROVIDE THE PROGRAM *AS IS* WITHOUT WARRANTY OF ANY KIND, EITHER 558 | EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 559 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE 560 | QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE 561 | DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR 562 | CORRECTION. 563 | 564 | ### 16. Limitation of Liability. 565 | 566 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY 567 | COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS 568 | PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, 569 | INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE 570 | THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED 571 | INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE 572 | PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY 573 | HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 574 | 575 | ### 17. Interpretation of Sections 15 and 16. 576 | 577 | If the disclaimer of warranty and limitation of liability provided above cannot 578 | be given local legal effect according to their terms, reviewing courts shall 579 | apply local law that most closely approximates an absolute waiver of all civil 580 | liability in connection with the Program, unless a warranty or assumption of 581 | liability accompanies a copy of the Program in return for a fee. 582 | 583 | ## END OF TERMS AND CONDITIONS ### 584 | 585 | ### How to Apply These Terms to Your New Programs 586 | 587 | If you develop a new program, and you want it to be of the greatest possible 588 | use to the public, the best way to achieve this is to make it free software 589 | which everyone can redistribute and change under these terms. 590 | 591 | To do so, attach the following notices to the program. It is safest to attach 592 | them to the start of each source file to most effectively state the exclusion 593 | of warranty; and each file should have at least the *copyright* line and a 594 | pointer to where the full notice is found. 595 | 596 | 597 | Copyright (C) 598 | 599 | This program is free software: you can redistribute it and/or modify 600 | it under the terms of the GNU General Public License as published by 601 | the Free Software Foundation, either version 3 of the License, or 602 | (at your option) any later version. 603 | 604 | This program is distributed in the hope that it will be useful, 605 | but WITHOUT ANY WARRANTY; without even the implied warranty of 606 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 607 | GNU General Public License for more details. 608 | 609 | You should have received a copy of the GNU General Public License 610 | along with this program. If not, see . 611 | 612 | Also add information on how to contact you by electronic and paper mail. 613 | 614 | If the program does terminal interaction, make it output a short notice like 615 | this when it starts in an interactive mode: 616 | 617 | Copyright (C) 618 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 619 | This is free software, and you are welcome to redistribute it 620 | under certain conditions; type `show c' for details. 621 | 622 | The hypothetical commands `show w` and `show c` should show the appropriate 623 | parts of the General Public License. Of course, your program's commands might 624 | be different; for a GUI interface, you would use an *about box*. 625 | 626 | You should also get your employer (if you work as a programmer) or school, if 627 | any, to sign a *copyright disclaimer* for the program, if necessary. For more 628 | information on this, and how to apply and follow the GNU GPL, see 629 | [http://www.gnu.org/licenses/](http://www.gnu.org/licenses/). 630 | 631 | The GNU General Public License does not permit incorporating your program into 632 | proprietary programs. If your program is a subroutine library, you may consider 633 | it more useful to permit linking proprietary applications with the library. If 634 | this is what you want to do, use the GNU Lesser General Public License instead 635 | of this License. But first, please read 636 | [http://www.gnu.org/philosophy/why-not-lgpl.html](http://www.gnu.org/philosophy/why-not-lgpl.html). 637 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DevOps 2 | Things I've written that I don't want to disappear 3 | 4 | # Files 5 | 6 | 1. [EBS Snapshotter in Lambda (CFT)](aws/lambda/cloudformation/ebs_snapshotter_lambda.json) - This is a CFT template to create 4 lambda tasks. 7 | 1. `EbsSnapshotCreatorLambdaFunction` - This is the real work. It basically searches Ec2 instances for a tag (defined as a set of CFT params) 8 | 1. `EbsSnapshotDailyLambdaFunction` - Runs daily with a retention period of 7 days 9 | 1. `EbsSnapshotWeeklyLambdaFunction` - Runs weekly with a retention period of 14 days 10 | 1. `EbsSnapshotJanitorLambdaFunction` - Runs daily and deletes ebs snapshots that were created by the `EbsSnapshotCreatorLambdaFunction` task with dates /usr/local/bin/dumb-init 5 | RUN chmod +x /usr/local/bin/dumb-init 6 | 7 | RUN mkdir /opt 8 | COPY run.sh /opt 9 | RUN chmod +x /opt/run.sh 10 | 11 | COPY rules /tmp 12 | 13 | RUN echo 'policies:' >/tmp/custodian.yml 14 | RUN for yml in $(find /tmp -name '*.yml'); do cat $yml; done | grep -v policies: >>/tmp/custodian.yml 15 | 16 | ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] 17 | CMD ["/opt/run.sh"] 18 | -------------------------------------------------------------------------------- /aws/fargate/cloud-custodian/Makefile: -------------------------------------------------------------------------------- 1 | TFENV_ROOT=$(HOME)/.tfenv 2 | export PATH := $(TFENV_ROOT)/bin:$(PATH) 3 | TFENV_VERSION := $(shell cat .terraform-version) 4 | 5 | all: setup plan deploy 6 | 7 | setup: 8 | if [ ! -d $(TFENV_ROOT) ]; then git clone https://github.com/kamatama41/tfenv.git $(TFENV_ROOT); fi 9 | $(TFENV_ROOT)/bin/tfenv install $(TFENV_VERSION) 10 | terraform init 11 | 12 | test: 13 | docker-compose build 14 | docker-compose run custodian 15 | 16 | plan: setup 17 | terraform plan 18 | 19 | build: setup 20 | terraform apply -target=aws_ecr_repository.cloud-custodian 21 | docker build -t cloud-custodian . 22 | 23 | deploy: build 24 | $(eval REPO_URL := $(shell terraform output cloud_custodian_repository_url | tr -d '\n')) 25 | $(eval CLUSTER := $(shell terraform output cluster | tr -d '\n')) 26 | docker tag cloud-custodian:latest $(REPO_URL):latest 27 | aws ecr get-login --region us-east-1 --no-include-email | sh - 28 | docker push $(REPO_URL):latest 29 | terraform apply 30 | aws ecs stop-task --task $(shell aws ecs describe-tasks --tasks $(shell aws ecs list-tasks --cluster $(CLUSTER) | jq -r .taskArns[] | xargs echo) --cluster $(CLUSTER) | jq -r '.tasks[] | select(.group == "service:cloud-custodian") | .taskArn') --cluster $(CLUSTER) 31 | 32 | clean: 33 | terraform destroy -force security_group_id=$(SECURITY_GROUP_ID) -var subnet_id=$(SUBNET_ID) -auto-approve 34 | -------------------------------------------------------------------------------- /aws/fargate/cloud-custodian/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | custodian: 5 | build: . 6 | environment: 7 | - AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 8 | - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 9 | - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 10 | volumes: 11 | - .:/app 12 | command: /opt/run.sh 13 | -------------------------------------------------------------------------------- /aws/fargate/cloud-custodian/ecs.tf: -------------------------------------------------------------------------------- 1 | resource "aws_ecr_repository" "cloud-custodian" { 2 | name = "cloud-custodian" 3 | } 4 | 5 | resource "aws_cloudwatch_log_group" "cloud-custodian" { 6 | name = "cloud-custodian" 7 | } 8 | 9 | resource "aws_ecs_cluster" "cloud-custodian" { 10 | name = "cloud-custodian" 11 | } 12 | 13 | resource "aws_ecs_service" "cloud-custodian" { 14 | name = "cloud-custodian" 15 | cluster = "${aws_ecs_cluster.cloud-custodian.arn}" 16 | task_definition = "${aws_ecs_task_definition.cloud-custodian.arn}" 17 | desired_count = "1" 18 | deployment_maximum_percent = "100" 19 | deployment_minimum_healthy_percent = "0" 20 | launch_type = "FARGATE" 21 | 22 | network_configuration { 23 | security_groups = ["${var.security_group_id}"] 24 | subnets = ["${var.subnet_id}"] 25 | } 26 | } 27 | 28 | resource "aws_ecs_task_definition" "cloud-custodian" { 29 | family = "cloud-custodian" 30 | network_mode = "awsvpc" 31 | cpu = 256 32 | memory = 512 33 | container_definitions = < 0 and "lambda:elasticache_snapshot_copier" in [tag['Value'] for tag in tags]): 22 | if ((datetime.now(timezone.utc) - snapshot['NodeSnapshots'][0]['SnapshotCreateTime']).days > 7): 23 | print('removing snapshot with name {}'.format(snapshot['SnapshotName'])) 24 | elasticache_client.delete_snapshot(SnapshotName=snapshot['SnapshotName']) 25 | 26 | 27 | def create_snapshot(elasticache_cluster): 28 | print('Creating {} snapshots'.format(elasticache_cluster['CacheClusterId'])) 29 | automated_snapshots = [s for s in elasticache_client.describe_snapshots(CacheClusterId=elasticache_cluster['CacheClusterId'])['Snapshots'] if (s['SnapshotSource'] != 'manual' and s['SnapshotStatus'] == 'available')] 30 | try: 31 | latest_automated_snapshot = sorted(automated_snapshots, key=lambda x: (x['NodeSnapshots'][0]['SnapshotCreateTime']), reverse=True)[0] 32 | except IndexError: 33 | return 34 | identifier = 'manual-{}'.format(re.search('.+?\.(.*)', latest_automated_snapshot['SnapshotName']).group(1)) 35 | print('creating {} from {}'.format(identifier, latest_automated_snapshot['SnapshotName'])) 36 | try: 37 | response = elasticache_client.copy_snapshot( 38 | SourceSnapshotName=latest_automated_snapshot['SnapshotName'], 39 | TargetSnapshotName=identifier, 40 | ) 41 | while elasticache_client.describe_snapshots(SnapshotName=identifier)['Snapshots'][0]['SnapshotStatus'] != 'available': 42 | print('Waiting for snapshot to be available...') 43 | time.sleep(10) 44 | 45 | region = re.search('([a-z]{2}-[a-z]+-[0-9]).*', response['Snapshot']['PreferredAvailabilityZone']).group(1) 46 | elasticache_client.add_tags_to_resource( 47 | ResourceName='arn:aws:elasticache:{}:{}:snapshot:{}'.format(region, account_id, identifier), 48 | Tags=[ 49 | { 50 | 'Key': 'Source', 51 | 'Value': elasticache_cluster['CacheClusterId'], 52 | }, 53 | { 54 | 'Key': 'Managed_by', 55 | 'Value': 'lambda:elasticache_snapshot_copier', 56 | }, 57 | ], 58 | ) 59 | except elasticache_client.exceptions.SnapshotAlreadyExistsFault: 60 | print('Skipping already created snapshot') 61 | pass 62 | 63 | 64 | def lambda_handler(event, context): 65 | for elasticache_cluster in elasticache_client.describe_cache_clusters()['CacheClusters']: 66 | create_snapshot(elasticache_cluster) 67 | clean_snapshots(elasticache_cluster) 68 | for replication_group in [replication_group for replication_group in elasticache_client.describe_replication_groups()['ReplicationGroups']]: 69 | for elasticache_node_group in [group['NodeGroupMembers'] for group in replication_group['NodeGroups']]: 70 | for elasticache_cluster in elasticache_node_group: 71 | create_snapshot(elasticache_cluster) 72 | clean_snapshots(elasticache_cluster) 73 | 74 | 75 | if __name__ == "__main__": 76 | lambda_handler(None, None) 77 | -------------------------------------------------------------------------------- /aws/lambda/elasticache_snapshot_copier/tox.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .tox,./build 3 | filename = *.py 4 | ignore = E501 5 | -------------------------------------------------------------------------------- /aws/lambda/elasticache_snapshot_copier/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | default = "us-east-1" 3 | } 4 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | venv 3 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/.python-version: -------------------------------------------------------------------------------- 1 | 3.6.0 2 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/.terraform-version: -------------------------------------------------------------------------------- 1 | 0.11.3 2 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/Makefile: -------------------------------------------------------------------------------- 1 | zip: 2 | rm -rf lambda_rds_logs_to_s3.zip 3 | zip -r lambda_rds_logs_to_s3.zip lambda_function.py 4 | 5 | test: 6 | bats test.bats 7 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/README.md: -------------------------------------------------------------------------------- 1 | RDS Logs to S3 2 | =========================================================== 3 | 4 | This lambda runs on a timer to move logs into S3 from RDS 5 | 6 | Make sure you [enable user logging first via this guide](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.PostgreSQL.html) 7 | 8 | ## To Test 9 | 10 | ``` 11 | make test 12 | ``` 13 | 14 | ## To Deploy 15 | 16 | ``` 17 | # verify changes look as expected 18 | make plan 19 | 20 | # actually do the deploy if the previous plan seems fine 21 | make deploy 22 | ``` 23 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | lambda: 5 | container_name: lambda 6 | image: lambci/lambda:python3.6 7 | environment: 8 | - AWS_REGION=$AWS_DEFAULT_REGION 9 | - AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 10 | - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 11 | - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 12 | - AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN 13 | - AWS_SECURITY_TOKEN=$AWS_SECURITY_TOKEN 14 | - S3BUCKET=$S3BUCKET 15 | - HOSTNAME=$HOSTNAME 16 | - ENDPOINT_URL=http://localstack:4572 17 | volumes: 18 | - .:/var/task 19 | networks: 20 | - mynetwork 21 | links: 22 | - localstack 23 | localstack: 24 | image: localstack/localstack 25 | ports: 26 | - 4572:4572 27 | environment: 28 | - DEFAULT_REGION=us-east-1 29 | - SERVICES=lambda,s3 30 | - DEBUG=1 31 | - LAMBDA_EXECUTOR=docker 32 | - DOCKER_HOST=unix:///var/run/docker.sock 33 | - HOSTNAME=localstack 34 | - HOSTNAME_EXTERNAL=localstack 35 | volumes: 36 | - /var/run/docker.sock:/var/run/docker.sock 37 | - /tmp/localstack:/tmp/localstack 38 | networks: 39 | - mynetwork 40 | 41 | networks: 42 | mynetwork: 43 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/iam.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "rds_logs_to_s3" { 2 | name = "rds_logs_to_s3" 3 | 4 | assume_role_policy = < last_written_time) or first_run: 38 | print("Downloading log file: %s found and with LastWritten value of: %s " % (db_log['LogFileName'], db_log['LastWritten'])) 39 | if int(db_log['LastWritten']) > last_written_this_run: 40 | last_written_this_run = int(db_log['LastWritten']) 41 | log_file = rds_client.download_db_log_file_portion(DBInstanceIdentifier=db_name, LogFileName=db_log['LogFileName'], Marker='0') 42 | log_file_data = log_file['LogFileData'] 43 | while log_file['AdditionalDataPending']: 44 | log_file = rds_client.download_db_log_file_portion(DBInstanceIdentifier=db_name, LogFileName=db_log['LogFileName'], Marker=log_file['Marker']) 45 | log_file_data += log_file['LogFileData'] 46 | byteData = str.encode(log_file_data) 47 | object_name = S3PREFIX + db_name + '/' + db_log['LogFileName'] 48 | print(object_name) 49 | s3_response = s3_client.put_object(Bucket=S3BUCKET, Key=object_name, Body=byteData) 50 | print("Writing log file %s to S3 bucket %s" % (object_name, S3BUCKET)) 51 | s3_response = s3_client.put_object(Bucket=S3BUCKET, Key=last_received_file, Body=str.encode(str(last_written_this_run))) 52 | print("Wrote new Last Written Marker to %s in Bucket %s" % (last_received_file, S3BUCKET)) 53 | print("Log file export complete") 54 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/outputs.tf: -------------------------------------------------------------------------------- 1 | output "function_arn" { 2 | value = "${aws_lambda_function.rds_logs_to_s3.arn}" 3 | } 4 | 5 | output "function_name" { 6 | value = "rds_logs_to_s3" 7 | } 8 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/rds_logs_to_s3.tf: -------------------------------------------------------------------------------- 1 | resource "aws_lambda_function" "rds_logs_to_s3" { 2 | filename = "lambda_rds_logs_to_s3.zip" 3 | function_name = "rds_logs_to_s3" 4 | role = "${data.terraform_remote_state.rds_logs_to_s3_iam_role.role_arn}" 5 | handler = "lambda_function.lambda_handler" 6 | source_code_hash = "${base64sha256(file("lambda_rds_logs_to_s3.zip"))}" 7 | runtime = "python3.6" 8 | timeout = "300" 9 | 10 | environment { 11 | variables = { 12 | S3BUCKET = "${data.terraform_remote_state.somebucket.bucket}" 13 | } 14 | } 15 | } 16 | 17 | resource "aws_cloudwatch_event_rule" "rds_logs_to_s3_schedule" { 18 | name = "rds_logs_to_s3_schedule" 19 | description = "Run every hour" 20 | schedule_expression = "rate(1 hour)" 21 | } 22 | 23 | resource "aws_cloudwatch_event_target" "rds_logs_to_s3_lambda" { 24 | rule = "${aws_cloudwatch_event_rule.rds_logs_to_s3_schedule.name}" 25 | arn = "${aws_lambda_function.rds_logs_to_s3.arn}" 26 | } 27 | 28 | resource "aws_lambda_permission" "allow_cloudwatch_to_call_rds_logs_to_s3" { 29 | statement_id = "AllowExecutionFromCloudWatchToRDSLogShipper" 30 | action = "lambda:InvokeFunction" 31 | function_name = "${aws_lambda_function.rds_logs_to_s3.function_name}" 32 | principal = "events.amazonaws.com" 33 | source_arn = "${aws_cloudwatch_event_rule.rds_logs_to_s3_schedule.arn}" 34 | } 35 | 36 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/resources/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "eventVersion": "2.0", 5 | "eventSource": "aws:s3", 6 | "awsRegion": "us-east-1", 7 | "eventTime": "2018-03-29T17:14:10.240Z", 8 | "eventName": "ObjectCreated:Post", 9 | "userIdentity": { 10 | "principalId": "AWS:HOORAH" 11 | }, 12 | "requestParameters": { 13 | "sourceIPAddress": "8.8.8.8" 14 | }, 15 | "responseElements": { 16 | "x-amz-request-id": "74EB086F3C821", 17 | "x-amz-id-2": "foo" 18 | }, 19 | "s3": { 20 | "s3SchemaVersion": "1.0", 21 | "configurationId": "tf-s3-topic-20180329164932817900000001", 22 | "bucket": { 23 | "name": "somebucket", 24 | "ownerIdentity": { 25 | "principalId": "WUT" 26 | }, 27 | "arn": "arn:aws:s3:::somebucket" 28 | }, 29 | "object": { 30 | "key": "foo.gz", 31 | "size": 146241, 32 | "eTag": "d85fef518f890cfd151178dbc77c54b9", 33 | "sequencer": "005ABD1EE21300EC8A" 34 | } 35 | } 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/sources.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | default = "us-east-1" 3 | } 4 | 5 | provider "aws" { 6 | version = "1.14" 7 | region = "${var.region}" 8 | } 9 | 10 | data "terraform_remote_state" "somebucket" { 11 | backend = "s3" 12 | 13 | config { 14 | bucket = "changeme" 15 | key = "setme" 16 | region = "us-east-1" 17 | } 18 | } 19 | 20 | 21 | -------------------------------------------------------------------------------- /aws/lambda/rds_logs_to_s3/test.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | function setup() { 4 | export S3BUCKET=somebucket 5 | export HOSTNAME=$HOSTNAME 6 | docker-compose up -d localstack 7 | sleep 5 8 | aws s3 mb s3://somebucket --endpoint http://localhost:4572 9 | } 10 | 11 | function teardown() { 12 | docker-compose down 13 | } 14 | 15 | @test "pulls all logs into s3" { 16 | docker-compose run lambda 17 | files_in_s3=$(aws s3 ls s3://somebucket/ --endpoint http://localhost:4572 --recursive | wc -l) 18 | [[ $files_in_s3 -ge 1 ]] 19 | 20 | docker-compose run lambda 21 | new_files_in_s3=$(aws s3 ls s3://somebucket/ --endpoint http://localhost:4572 --recursive | wc -l) 22 | [[ $new_files_in_s3 -ge 1 ]] 23 | [[ $files_in_s3 -eq $new_files_in_s3 ]] 24 | } 25 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | *.zip 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/.terraform-version: -------------------------------------------------------------------------------- 1 | 0.10.6 2 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/Makefile: -------------------------------------------------------------------------------- 1 | PYENV_ROOT=$(HOME)/.pyenv 2 | TFENV_ROOT=$(HOME)/.tfenv 3 | TFENV_VERSION := $(shell cat .terraform-version) 4 | export PATH := $(PYENV_ROOT)/bin:$(TFENV_ROOT)/bin:$(PATH) 5 | 6 | all: setup deploy 7 | 8 | setup: 9 | if [ ! -d $(PYENV_ROOT) ]; then git clone https://github.com/pyenv/pyenv.git $(PYENV_ROOT); fi 10 | if [ ! -d $(TFENV_ROOT) ]; then git clone https://github.com/kamatama41/tfenv.git $(TFENV_ROOT); fi 11 | eval "$(pyenv init -)" 12 | if [ ! "$(shell pyenv versions | grep 3.6.0)" ]; then pyenv install 3.6.0; pyenv local 3.6.0; fi 13 | $(TFENV_ROOT)/bin/tfenv install $(TFENV_VERSION) 14 | terraform init 15 | 16 | zip: 17 | rm -rf lambda_rds_snapshot_copier.zip 18 | zip -r lambda_rds_snapshot_copier.zip lambda_function.py 19 | 20 | test: 21 | docker run -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -v "$(PWD)":/var/task lambci/lambda:python3.6 22 | 23 | plan: zip 24 | terraform init 25 | terraform plan 26 | 27 | deploy: zip 28 | terraform init 29 | terraform apply 30 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/README.md: -------------------------------------------------------------------------------- 1 | RDS Snapshot Copier 2 | =================== 3 | 4 | This runs in lambda and copies automatic RDS snapshots so that theyre safe from terraform 5 | 6 | 7 | ## To Test 8 | 9 | ``` 10 | export AWS_ACCESS_KEY_ID=your_key_id 11 | export AWS_SECRET_ACCESS_KEY=your_secret_access_key 12 | 13 | docker-compose run lambda 14 | ``` 15 | 16 | ## To Deploy 17 | 18 | ``` 19 | # ensure you're using python 3.6 (pyenv should respect .python-version) 20 | make install 21 | 22 | # verify changes look as expected 23 | make plan 24 | 25 | # actually do the deploy if the previous plan seems fine 26 | make deploy 27 | ``` 28 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | lambda: 5 | container_name: lambda 6 | image: lambci/lambda:python3.6 7 | environment: 8 | - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 9 | - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 10 | volumes: 11 | - .:/var/task 12 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/iam.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "rds_snapshot_copier_lambda" { 2 | name = "rds_snapshot_copier_lambda" 3 | path = "/lambda/" 4 | 5 | assume_role_policy = < 0): 20 | latest_automated_snapshot = latest_automated_snapshots[0] 21 | identifier = re.search('.+?:(.*)', latest_automated_snapshot['DBSnapshotIdentifier']).group(1) 22 | print('creating {} from'.format(identifier, latest_automated_snapshot['DBSnapshotIdentifier'])) 23 | try: 24 | rds_client.copy_db_snapshot( 25 | SourceDBSnapshotIdentifier=latest_automated_snapshot['DBSnapshotIdentifier'], 26 | TargetDBSnapshotIdentifier='manual-{}'.format(identifier), 27 | Tags=[ 28 | { 29 | 'Key': 'Source', 30 | 'Value': db_instance['DBInstanceIdentifier'], 31 | }, 32 | { 33 | 'Key': 'Managed_by', 34 | 'Value': 'lambda:rds_snapshot_copier', 35 | }, 36 | ], 37 | CopyTags=True, 38 | ) 39 | except rds_client.exceptions.DBSnapshotAlreadyExistsFault: 40 | print('Skipping already created snapshot') 41 | pass 42 | 43 | print('Cleaning old snapshots') 44 | manual_snapshots = [s for s in rds_client.describe_db_snapshots(DBInstanceIdentifier=db_instance['DBInstanceIdentifier'])['DBSnapshots'] if (s['SnapshotType'] != 'automated' and s['Status'] == 'available')] 45 | for snapshot in manual_snapshots: 46 | tags = rds_client.list_tags_for_resource(ResourceName=snapshot['DBSnapshotArn'])['TagList'] 47 | if (len(tags) > 0 and "lambda:rds_snapshot_copier" in [tag['Value'] for tag in tags]): 48 | if ((datetime.now(timezone.utc) - snapshot['SnapshotCreateTime']).days > 7): 49 | print('removing snapshot with identifier {}'.format(snapshot['DBSnapshotIdentifier'])) 50 | rds_client.delete_db_snapshot(DBSnapshotIdentifier=snapshot['DBSnapshotIdentifier']) 51 | 52 | 53 | if __name__ == "__main__": 54 | lambda_handler(None, None) 55 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/rds_snapshot_copier.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | version = "1.2" 3 | region = "${var.region}" 4 | } 5 | 6 | resource "aws_lambda_function" "rds_snapshot_copier" { 7 | filename = "lambda_rds_snapshot_copier.zip" 8 | function_name = "rds_snapshot_copier" 9 | role = "${aws_iam_role.rds_snapshot_copier_lambda.arn}" 10 | handler = "lambda_function.lambda_handler" 11 | source_code_hash = "${base64sha256(file("lambda_rds_snapshot_copier.zip"))}" 12 | runtime = "python3.6" 13 | timeout = "30" 14 | } 15 | 16 | resource "aws_cloudwatch_event_rule" "rds_snapshot_copier_schedule" { 17 | name = "rds_snapshot_copier_schedule" 18 | description = "Run every day" 19 | schedule_expression = "rate(1 day)" 20 | } 21 | 22 | resource "aws_cloudwatch_event_target" "rds_snapshot_copier_lambda" { 23 | rule = "${aws_cloudwatch_event_rule.rds_snapshot_copier_schedule.name}" 24 | arn = "${aws_lambda_function.rds_snapshot_copier.arn}" 25 | } 26 | 27 | resource "aws_lambda_permission" "allow_cloudwatch_to_call_rds_snapshot_copier" { 28 | statement_id = "AllowExecutionFromCloudWatchToBeanstalkGarbageCollector" 29 | action = "lambda:InvokeFunction" 30 | function_name = "${aws_lambda_function.rds_snapshot_copier.function_name}" 31 | principal = "events.amazonaws.com" 32 | source_arn = "${aws_cloudwatch_event_rule.rds_snapshot_copier_schedule.arn}" 33 | } 34 | -------------------------------------------------------------------------------- /aws/lambda/rds_snapshot_copier/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | default = "us-east-1" 3 | } 4 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | *.zip 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/.terraform-version: -------------------------------------------------------------------------------- 1 | 0.10.6 2 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/Makefile: -------------------------------------------------------------------------------- 1 | PYENV_ROOT=$(HOME)/.pyenv 2 | TFENV_ROOT=$(HOME)/.tfenv 3 | TFENV_VERSION := $(shell cat .terraform-version) 4 | export PATH := $(PYENV_ROOT)/bin:$(TFENV_ROOT)/bin:$(PATH) 5 | 6 | all: setup deploy 7 | 8 | setup: 9 | if [ ! -d $(PYENV_ROOT) ]; then git clone https://github.com/pyenv/pyenv.git $(PYENV_ROOT); fi 10 | if [ ! -d $(TFENV_ROOT) ]; then git clone https://github.com/kamatama41/tfenv.git $(TFENV_ROOT); fi 11 | eval "$(pyenv init -)" 12 | if [ ! "$(shell pyenv versions | grep 3.6.0)" ]; then pyenv install 3.6.0; pyenv local 3.6.0; fi 13 | $(TFENV_ROOT)/bin/tfenv install $(TFENV_VERSION) 14 | terraform init 15 | 16 | zip: 17 | rm -rf lambda_redshift_snapshot_copier.zip 18 | zip -r lambda_redshift_snapshot_copier.zip lambda_function.py 19 | 20 | test: 21 | docker run -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -v "$(PWD)":/var/task lambci/lambda:python3.6 22 | 23 | plan: zip 24 | terraform init 25 | terraform plan 26 | 27 | deploy: zip 28 | terraform init 29 | terraform apply 30 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/README.md: -------------------------------------------------------------------------------- 1 | Redshift Snapshot Copier 2 | =================== 3 | 4 | This runs in lambda and copies automatic redshift snapshots so that theyre safe from terraform 5 | 6 | 7 | ## To Test 8 | 9 | ``` 10 | export AWS_ACCESS_KEY_ID=your_key_id 11 | export AWS_SECRET_ACCESS_KEY=your_secret_access_key 12 | 13 | docker-compose run lambda 14 | ``` 15 | 16 | ## To Deploy 17 | 18 | ``` 19 | # ensure you're using python 3.6 (pyenv should respect .python-version) 20 | make install 21 | 22 | # verify changes look as expected 23 | make plan 24 | 25 | # actually do the deploy if the previous plan seems fine 26 | make deploy 27 | ``` 28 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | lambda: 5 | container_name: lambda 6 | image: lambci/lambda:python3.6 7 | environment: 8 | - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 9 | - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 10 | volumes: 11 | - .:/var/task 12 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/iam.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "redshift_snapshot_copier_lambda" { 2 | name = "redshift_snapshot_copier_lambda" 3 | path = "/lambda/" 4 | 5 | assume_role_policy = < 0): 19 | latest_automated_snapshot = sorted_automated_snapshots[0] 20 | identifier = re.search('.+?:(.*)', latest_automated_snapshot['SnapshotIdentifier']).group(1) 21 | print('creating {} from {}'.format(identifier, latest_automated_snapshot['SnapshotIdentifier'])) 22 | try: 23 | create_response = redshift_client.copy_cluster_snapshot( 24 | SourceSnapshotIdentifier=latest_automated_snapshot['SnapshotIdentifier'], 25 | TargetSnapshotIdentifier='manual-{}'.format(identifier), 26 | )['Snapshot'] 27 | redshift_client.create_tags( 28 | ResourceName='arn:aws:redshift:{}:{}:snapshot:{}/{}'.format( 29 | create_response['AvailabilityZone'][:-1], 30 | create_response['OwnerAccount'], 31 | create_response['ClusterIdentifier'], 32 | 'manual-{}'.format(identifier) 33 | ), 34 | Tags=[{ 35 | 'Key': 'Managed_by', 36 | 'Value': 'lambda:redshift_snapshot_copier' 37 | }] 38 | ) 39 | except redshift_client.exceptions.ClusterSnapshotQuotaExceededFault: 40 | pass 41 | except redshift_client.exceptions.ClientError as e: 42 | if ('Cannot create more than' in str(e)): 43 | pass 44 | elif ('has already been copied' in str(e)): 45 | print('Skipping already created snapshot') 46 | pass 47 | else: 48 | raise(e) 49 | else: 50 | print('No automated snaphots found for cluster {}'.format(cluster['ClusterIdentifier'])) 51 | 52 | print('Clearing old snapshots for cluster {}'.format(cluster['ClusterIdentifier'])) 53 | manual_snapshots = [s for s in redshift_client.describe_cluster_snapshots(ClusterIdentifier=cluster['ClusterIdentifier'])['Snapshots'] if (s['SnapshotType'] != 'automated' and s['Status'] == 'available')] 54 | for snapshot in manual_snapshots: 55 | match = re.search('^(manual-).*', snapshot['SnapshotIdentifier']) 56 | if (match is not None and match.group(1) is not None): 57 | response = redshift_client.describe_tags( 58 | ResourceName='arn:aws:redshift:{}:{}:snapshot:{}/{}'.format( 59 | snapshot['AvailabilityZone'][:-1], 60 | snapshot['OwnerAccount'], 61 | snapshot['ClusterIdentifier'], 62 | snapshot['SnapshotIdentifier'] 63 | ), 64 | TagKeys=[ 65 | 'Managed_by', 66 | ], 67 | TagValues=[ 68 | 'lambda:redshift_snapshot_copier', 69 | ] 70 | )['TaggedResources'] 71 | 72 | if ((len(response) > 0) and (datetime.now(timezone.utc) - snapshot['SnapshotCreateTime']).days > 7): 73 | print('removing snapshot with identifier {}'.format(snapshot['SnapshotIdentifier'])) 74 | redshift_client.delete_cluster_snapshot(SnapshotIdentifier=snapshot['SnapshotIdentifier']) 75 | 76 | 77 | if __name__ == "__main__": 78 | lambda_handler(None, None) 79 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/redshift_snapshot_copier.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | version = "1.2" 3 | region = "${var.region}" 4 | } 5 | 6 | data "terraform_remote_state" "redshift_snapshot_copier_iam_role" { 7 | backend = "s3" 8 | 9 | config { 10 | bucket = "com-stratasan-terraform" 11 | key = "iam/roles/redshift_snapshot_copier/terraform.tfstate" 12 | region = "${var.region}" 13 | } 14 | } 15 | 16 | resource "aws_lambda_function" "redshift_snapshot_copier" { 17 | filename = "lambda_redshift_snapshot_copier.zip" 18 | function_name = "redshift_snapshot_copier" 19 | role = "${aws_iam_role.redshift_snapshot_copier_lambda.arn}" 20 | handler = "lambda_function.lambda_handler" 21 | source_code_hash = "${base64sha256(file("lambda_redshift_snapshot_copier.zip"))}" 22 | runtime = "python3.6" 23 | timeout = "30" 24 | } 25 | 26 | resource "aws_cloudwatch_event_rule" "redshift_snapshot_copier_schedule" { 27 | name = "redshift_snapshot_copier_schedule" 28 | description = "Run every day" 29 | schedule_expression = "rate(1 day)" 30 | } 31 | 32 | resource "aws_cloudwatch_event_target" "redshift_snapshot_copier_lambda" { 33 | rule = "${aws_cloudwatch_event_rule.redshift_snapshot_copier_schedule.name}" 34 | arn = "${aws_lambda_function.redshift_snapshot_copier.arn}" 35 | } 36 | 37 | resource "aws_lambda_permission" "allow_cloudwatch_to_call_redshift_snapshot_copier" { 38 | statement_id = "AllowExecutionFromCloudWatchToBeanstalkGarbageCollector" 39 | action = "lambda:InvokeFunction" 40 | function_name = "${aws_lambda_function.redshift_snapshot_copier.function_name}" 41 | principal = "events.amazonaws.com" 42 | source_arn = "${aws_cloudwatch_event_rule.redshift_snapshot_copier_schedule.arn}" 43 | } 44 | -------------------------------------------------------------------------------- /aws/lambda/redshift_snapshot_copier/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | default = "us-east-1" 3 | } 4 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | venv 3 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/.python-version: -------------------------------------------------------------------------------- 1 | 3.6.0 2 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/.terraform-version: -------------------------------------------------------------------------------- 1 | 0.11.3 2 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | if [ ! -d venv ]; then virtualenv venv; fi 3 | . venv/bin/activate 4 | pip3 install -r requirements.txt -t dist >/dev/null 5 | rm -rf venv 6 | 7 | zip: install 8 | rm -rf lambda_s3_to_elasticsearch.zip 9 | zip -r lambda_s3_to_elasticsearch.zip lambda_function.py dist >/dev/null 10 | 11 | test: zip 12 | bats test.bats 13 | 14 | clean: 15 | rm -rf venv dist 16 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/README.md: -------------------------------------------------------------------------------- 1 | S3 to Elasticsearch 2 | =========================================================== 3 | 4 | This lambda is triggered by S3 events to fork them into ELK 5 | 6 | ## To Test 7 | 8 | ``` 9 | make test 10 | ``` 11 | 12 | ## To Deploy 13 | 14 | ``` 15 | make zip 16 | terraform apply -auto-approve 17 | ``` 18 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | lambda: 5 | container_name: lambda 6 | image: lambci/lambda:python3.6 7 | environment: 8 | - AWS_REGION=$AWS_DEFAULT_REGION 9 | - AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 10 | - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 11 | - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 12 | - AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN 13 | - AWS_SECURITY_TOKEN=$AWS_SECURITY_TOKEN 14 | - AWS_LAMBDA_EVENT_BODY=$AWS_LAMBDA_EVENT_BODY 15 | - ES_HOST=elasticsearch 16 | - ES_AUTH_ENABLED=False 17 | - ES_PORT=9200 18 | - ES_PROTOCOL=http 19 | - HOSTNAME=$HOSTNAME 20 | - ENDPOINT_URL=http://localstack:4572 21 | volumes: 22 | - .:/var/task 23 | networks: 24 | - mynetwork 25 | links: 26 | - localstack 27 | - elasticsearch 28 | localstack: 29 | image: localstack/localstack 30 | ports: 31 | - 4572:4572 32 | environment: 33 | - DEFAULT_REGION=us-east-1 34 | - SERVICES=lambda,s3 35 | - DEBUG=1 36 | - LAMBDA_EXECUTOR=docker 37 | - DOCKER_HOST=unix:///var/run/docker.sock 38 | - HOSTNAME=localstack 39 | - HOSTNAME_EXTERNAL=localstack 40 | volumes: 41 | - /var/run/docker.sock:/var/run/docker.sock 42 | - /tmp/localstack:/tmp/localstack 43 | networks: 44 | - mynetwork 45 | elasticsearch: 46 | image: docker.elastic.co/elasticsearch/elasticsearch:6.2.3 47 | ports: 48 | - 9200:9200 49 | environment: 50 | - bootstrap.memory_lock=true 51 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 52 | - "discovery.type=single-node" 53 | networks: 54 | - mynetwork 55 | networks: 56 | mynetwork: 57 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/iam.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "s3_to_elasticsearch" { 2 | name = "s3_to_elasticsearch" 3 | 4 | assume_role_policy = <=6.0.0,<7.0.0 3 | urllib3 4 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/resources/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "eventVersion": "2.0", 5 | "eventSource": "aws:s3", 6 | "awsRegion": "us-east-1", 7 | "eventTime": "2018-03-29T17:14:10.240Z", 8 | "eventName": "ObjectCreated:Post", 9 | "userIdentity": { 10 | "principalId": "AWS:AIDAJYAQ4T2F64BJAWHGA" 11 | }, 12 | "requestParameters": { 13 | "sourceIPAddress": "76.123.194.220" 14 | }, 15 | "responseElements": { 16 | "x-amz-request-id": "74EB075686F3C821", 17 | "x-amz-id-2": "ce6FjO6PJVDFlQe4yc69+lPn+nOw65RFxWxJUoyQlM+m+C+RoDroDIucLc8rym8JVlp8Vj0AWfE=" 18 | }, 19 | "s3": { 20 | "s3SchemaVersion": "1.0", 21 | "configurationId": "tf-s3-topic-20180329164932817900000001", 22 | "bucket": { 23 | "name": "somebucket", 24 | "ownerIdentity": { 25 | "principalId": "AG1T3D3YZ1LT1" 26 | }, 27 | "arn": "arn:aws:s3:::somebucket" 28 | }, 29 | "object": { 30 | "key": "foo.gz", 31 | "size": 146241, 32 | "eTag": "d85fef518f890cfd151178dbc77c54b9", 33 | "sequencer": "005ABD1EE21300EC8A" 34 | } 35 | } 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/s3_to_elasticsearch.tf: -------------------------------------------------------------------------------- 1 | resource "aws_lambda_function" "s3_to_elasticsearch" { 2 | filename = "lambda_s3_to_elasticsearch.zip" 3 | function_name = "s3_to_elasticsearch" 4 | role = "${data.terraform_remote_state.s3_to_elasticsearch_iam_role.role_arn}" 5 | handler = "lambda_function.lambda_handler" 6 | source_code_hash = "${base64sha256(file("lambda_s3_to_elasticsearch.zip"))}" 7 | runtime = "python3.6" 8 | timeout = "60" 9 | 10 | environment { 11 | variables = { 12 | ES_HOST="elasticsearch" 13 | ES_AUTH_ENABLED="False" 14 | ES_PORT="9200" 15 | ES_PROTOCOL="http" 16 | } 17 | } 18 | } 19 | 20 | resource "aws_lambda_permission" "allow_s3_to_call_s3_to_elasticsearch" { 21 | statement_id = "AllowExecutionFromS3ToESShipperLambda" 22 | action = "lambda:InvokeFunction" 23 | function_name = "${aws_lambda_function.s3_to_elasticsearch.function_name}" 24 | principal = "s3.amazonaws.com" 25 | } 26 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/sources.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | default = "us-east-1" 3 | } 4 | 5 | provider "aws" { 6 | version = "1.14" 7 | region = "${var.region}" 8 | } 9 | -------------------------------------------------------------------------------- /aws/lambda/s3_to_elasticsearch/test.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | function setup() { 4 | docker-compose down 5 | export S3BUCKET=somebucket 6 | export HOSTNAME=$HOSTNAME 7 | docker-compose up -d localstack 8 | docker-compose up -d elasticsearch 9 | sleep 30 10 | aws s3 mb s3://somebucket --endpoint http://localhost:4572 11 | echo something >foo 12 | gzip foo 13 | aws s3 cp foo.gz s3://somebucket --endpoint http://localhost:4572 14 | } 15 | 16 | function teardown() { 17 | docker-compose down 18 | rm foo.gz 19 | unset VAULT_TOKEN 20 | unset AWS_LAMBDA_EVENT_BODY 21 | } 22 | 23 | @test "reads file from s3 into elasticsearch" { 24 | [[ $(aws s3 ls s3://somebucket/ --endpoint http://localhost:4572 --recursive | wc -l) -eq 1 ]] 25 | 26 | export AWS_LAMBDA_EVENT_BODY=$(cat resources/event.json | tr -d '\n' | sed 's/#/\\#/g') 27 | docker-compose run lambda 28 | sleep 10 29 | index="logstash-$(date +%Y.%m.%d)" 30 | data=$(curl -s "http://localhost:9200/${index}/_search?pretty=true&q=*:*" | jq .hits.hits[]._source) 31 | [[ $(echo $data | jq -r .source) == "s3" ]] 32 | [[ $(echo $data | jq -r .key) == "foo.gz" ]] 33 | [[ $(echo $data | jq -r .bucket) == "somebucket" ]] 34 | [[ $(echo $data | jq -r .message) == "something" ]] 35 | } 36 | --------------------------------------------------------------------------------