├── .gitignore ├── .gitmodules ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── bin ├── build.sh ├── build_cluster.sh ├── destroy.sh ├── launch.sh └── resolv_host.py ├── docker-compose.yml ├── ec2 ├── ec2.py └── fabfile.py ├── hadoop-base ├── Dockerfile ├── core-site.xml ├── hdfs-site.xml ├── log4j.properties ├── mapred-site.xml ├── ssh_config └── yarn-site.xml ├── hadoop-master ├── Dockerfile └── bootstrap.sh └── hadoop-slave ├── Dockerfile └── bootstrap.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.idea 2 | *.iml 3 | *.tar.gz 4 | bin/*.hosts 5 | 6 | *.pyc 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "hadoop"] 2 | path = hadoop 3 | url = git://git.apache.org/hadoop.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | sudo: required 3 | 4 | env: 5 | global: 6 | - MAVEN_OPTS="-Xmx512M" 7 | - PATH="$HOME/build/Lewuathe/docker-hadoop-cluster/apache-maven-3.5.2/bin:$PATH" 8 | 9 | services: 10 | - docker 11 | 12 | before_install: 13 | - docker info 14 | - git submodule update --init 15 | 16 | install: 17 | - sudo apt-get install autoconf automake libtool curl make g++ unzip 18 | - wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz 19 | - tar -xzvf protobuf-2.5.0.tar.gz 20 | - cd protobuf-2.5.0 21 | - ./configure && make && sudo make install && sudo ldconfig 22 | - cd .. 23 | - wget http://ftp.jaist.ac.jp/pub/apache/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.zip 24 | - unzip apache-maven-3.5.4-bin.zip 25 | - mvn -version 26 | - cd hadoop 27 | - mvn clean package -DskipTests -Pdist -Dtar | grep "Building Apache Hadoop" 28 | - cp hadoop-dist/target/hadoop-3.2.0-SNAPSHOT.tar.gz ../hadoop-base 29 | 30 | script: 31 | - make 32 | 33 | after_success: 34 | - if [ "$TRAVIS_BRANCH" == "master" ]; then 35 | docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; 36 | docker push lewuathe/hadoop-base:latest; 37 | docker push lewuathe/hadoop-master:latest; 38 | docker push lewuathe/hadoop-slave:latest; 39 | fi 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | 205 | Apache Knox Subcomponents (binary distributions): 206 | 207 | Apache Knox includes a number of sub-components with separate copyright 208 | notices and license terms. Your use of these sub-components is subject 209 | to the terms and conditions of the following licenses. 210 | 211 | ------------------------------------------------------------------------------ 212 | From Jetty and Jerico 213 | ------------------------------------------------------------------------------ 214 | Eclipse Public License - v 1.0 215 | 216 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 217 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 218 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 219 | 220 | 1. DEFINITIONS 221 | 222 | "Contribution" means: 223 | 224 | a) in the case of the initial Contributor, the initial code and documentation 225 | distributed under this Agreement, and 226 | 227 | b) in the case of each subsequent Contributor: 228 | 229 | i) changes to the Program, and 230 | 231 | ii) additions to the Program; 232 | 233 | where such changes and/or additions to the Program originate from and are 234 | distributed by that particular Contributor. A Contribution 'originates' from a 235 | Contributor if it was added to the Program by such Contributor itself or anyone 236 | acting on such Contributor's behalf. Contributions do not include additions to 237 | the Program which: (i) are separate modules of software distributed in 238 | conjunction with the Program under their own license agreement, and (ii) are not 239 | derivative works of the Program. 240 | 241 | "Contributor" means any person or entity that distributes the Program. 242 | 243 | "Licensed Patents" mean patent claims licensable by a Contributor which are 244 | necessarily infringed by the use or sale of its Contribution alone or when 245 | combined with the Program. 246 | 247 | "Program" means the Contributions distributed in accordance with this Agreement. 248 | 249 | "Recipient" means anyone who receives the Program under this Agreement, 250 | including all Contributors. 251 | 252 | 2. GRANT OF RIGHTS 253 | 254 | a) Subject to the terms of this Agreement, each Contributor hereby grants 255 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 256 | reproduce, prepare derivative works of, publicly display, publicly perform, 257 | distribute and sublicense the Contribution of such Contributor, if any, and such 258 | derivative works, in source code and object code form. 259 | 260 | b) Subject to the terms of this Agreement, each Contributor hereby grants 261 | Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed 262 | Patents to make, use, sell, offer to sell, import and otherwise transfer the 263 | Contribution of such Contributor, if any, in source code and object code form. 264 | This patent license shall apply to the combination of the Contribution and the 265 | Program if, at the time the Contribution is added by the Contributor, such 266 | addition of the Contribution causes such combination to be covered by the 267 | Licensed Patents. The patent license shall not apply to any other combinations 268 | which include the Contribution. No hardware per se is licensed hereunder. 269 | 270 | c) Recipient understands that although each Contributor grants the licenses to 271 | its Contributions set forth herein, no assurances are provided by any 272 | Contributor that the Program does not infringe the patent or other intellectual 273 | property rights of any other entity. Each Contributor disclaims any liability to 274 | Recipient for claims brought by any other entity based on infringement of 275 | intellectual property rights or otherwise. As a condition to exercising the 276 | rights and licenses granted hereunder, each Recipient hereby assumes sole 277 | responsibility to secure any other intellectual property rights needed, if any. 278 | For example, if a third party patent license is required to allow Recipient to 279 | distribute the Program, it is Recipient's responsibility to acquire that license 280 | before distributing the Program. 281 | 282 | d) Each Contributor represents that to its knowledge it has sufficient copyright 283 | rights in its Contribution, if any, to grant the copyright license set forth in 284 | this Agreement. 285 | 286 | 3. REQUIREMENTS 287 | 288 | A Contributor may choose to distribute the Program in object code form under its 289 | own license agreement, provided that: 290 | 291 | a) it complies with the terms and conditions of this Agreement; and 292 | 293 | b) its license agreement: 294 | 295 | i) effectively disclaims on behalf of all Contributors all warranties and 296 | conditions, express and implied, including warranties or conditions of title and 297 | non-infringement, and implied warranties or conditions of merchantability and 298 | fitness for a particular purpose; 299 | 300 | ii) effectively excludes on behalf of all Contributors all liability for 301 | damages, including direct, indirect, special, incidental and consequential 302 | damages, such as lost profits; 303 | 304 | iii) states that any provisions which differ from this Agreement are offered by 305 | that Contributor alone and not by any other party; and 306 | 307 | iv) states that source code for the Program is available from such Contributor, 308 | and informs licensees how to obtain it in a reasonable manner on or through a 309 | medium customarily used for software exchange. 310 | 311 | When the Program is made available in source code form: 312 | 313 | a) it must be made available under this Agreement; and 314 | 315 | b) a copy of this Agreement must be included with each copy of the Program. 316 | 317 | Contributors may not remove or alter any copyright notices contained within the 318 | Program. 319 | 320 | Each Contributor must identify itself as the originator of its Contribution, if 321 | any, in a manner that reasonably allows subsequent Recipients to identify the 322 | originator of the Contribution. 323 | 324 | 4. COMMERCIAL DISTRIBUTION 325 | 326 | Commercial distributors of software may accept certain responsibilities with 327 | respect to end users, business partners and the like. While this license is 328 | intended to facilitate the commercial use of the Program, the Contributor who 329 | includes the Program in a commercial product offering should do so in a manner 330 | which does not create potential liability for other Contributors. Therefore, if 331 | a Contributor includes the Program in a commercial product offering, such 332 | Contributor ("Commercial Contributor") hereby agrees to defend and indemnify 333 | every other Contributor ("Indemnified Contributor") against any losses, damages 334 | and costs (collectively "Losses") arising from claims, lawsuits and other legal 335 | actions brought by a third party against the Indemnified Contributor to the 336 | extent caused by the acts or omissions of such Commercial Contributor in 337 | connection with its distribution of the Program in a commercial product 338 | offering. The obligations in this section do not apply to any claims or Losses 339 | relating to any actual or alleged intellectual property infringement. In order 340 | to qualify, an Indemnified Contributor must: a) promptly notify the Commercial 341 | Contributor in writing of such claim, and b) allow the Commercial Contributor 342 | to control, and cooperate with the Commercial Contributor in, the defense and 343 | any related settlement negotiations. The Indemnified Contributor may 344 | participate in any such claim at its own expense. 345 | 346 | For example, a Contributor might include the Program in a commercial product 347 | offering, Product X. That Contributor is then a Commercial Contributor. If that 348 | Commercial Contributor then makes performance claims, or offers warranties 349 | related to Product X, those performance claims and warranties are such 350 | Commercial Contributor's responsibility alone. Under this section, the 351 | Commercial Contributor would have to defend claims against the other 352 | Contributors related to those performance claims and warranties, and if a court 353 | requires any other Contributor to pay any damages as a result, the Commercial 354 | Contributor must pay those damages. 355 | 356 | 5. NO WARRANTY 357 | 358 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN 359 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 360 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, 361 | NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each 362 | Recipient is solely responsible for determining the appropriateness of using and 363 | distributing the Program and assumes all risks associated with its exercise of 364 | rights under this Agreement , including but not limited to the risks and costs 365 | of program errors, compliance with applicable laws, damage to or loss of data, 366 | programs or equipment, and unavailability or interruption of operations. 367 | 368 | 6. DISCLAIMER OF LIABILITY 369 | 370 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 371 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 372 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST 373 | PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 374 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 375 | OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS 376 | GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 377 | 378 | 7. GENERAL 379 | 380 | If any provision of this Agreement is invalid or unenforceable under applicable 381 | law, it shall not affect the validity or enforceability of the remainder of the 382 | terms of this Agreement, and without further action by the parties hereto, such 383 | provision shall be reformed to the minimum extent necessary to make such 384 | provision valid and enforceable. 385 | 386 | If Recipient institutes patent litigation against any entity (including a 387 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 388 | (excluding combinations of the Program with other software or hardware) 389 | infringes such Recipient's patent(s), then such Recipient's rights granted under 390 | Section 2(b) shall terminate as of the date such litigation is filed. 391 | 392 | All Recipient's rights under this Agreement shall terminate if it fails to 393 | comply with any of the material terms or conditions of this Agreement and does 394 | not cure such failure in a reasonable period of time after becoming aware of 395 | such noncompliance. If all Recipient's rights under this Agreement terminate, 396 | Recipient agrees to cease use and distribution of the Program as soon as 397 | reasonably practicable. However, Recipient's obligations under this Agreement 398 | and any licenses granted by Recipient relating to the Program shall continue and 399 | survive. 400 | 401 | Everyone is permitted to copy and distribute copies of this Agreement, but in 402 | order to avoid inconsistency the Agreement is copyrighted and may only be 403 | modified in the following manner. The Agreement Steward reserves the right to 404 | publish new versions (including revisions) of this Agreement from time to time. 405 | No one other than the Agreement Steward has the right to modify this Agreement. 406 | The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation 407 | may assign the responsibility to serve as the Agreement Steward to a suitable 408 | separate entity. Each new version of the Agreement will be given a 409 | distinguishing version number. The Program (including Contributions) may always 410 | be distributed subject to the version of the Agreement under which it was 411 | received. In addition, after a new version of the Agreement is published, 412 | Contributor may elect to distribute the Program (including its Contributions) 413 | under the new version. Except as expressly stated in Sections 2(a) and 2(b) 414 | above, Recipient receives no rights or licenses to the intellectual property of 415 | any Contributor under this Agreement, whether expressly, by implication, 416 | estoppel or otherwise. All rights in the Program not expressly granted under 417 | this Agreement are reserved. 418 | 419 | This Agreement is governed by the laws of the State of New York and the 420 | intellectual property laws of the United States of America. No party to this 421 | Agreement will bring a legal action under this Agreement more than one year 422 | after the cause of action arose. Each party waives its rights to a jury trial in 423 | any resulting litigation. 424 | 425 | 426 | For TODO.jar (Jave EE Servlet API) 427 | 428 | COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 429 | 430 | 1. Definitions. 431 | 432 | 1.1. Contributor. means each individual or entity that creates or contributes 433 | to the creation of Modifications. 434 | 435 | 1.2. Contributor Version. means the combination of the Original Software, 436 | prior Modifications used by a Contributor (if any), and the 437 | Modifications made by that particular Contributor. 438 | 439 | 1.3. Covered Software. means (a) the Original Software, or (b) Modifications, 440 | or (c) the combination of files containing Original Software with files 441 | containing Modifications, in each case including portions thereof. 442 | 443 | 1.4. Executable. means the Covered Software in any form other than Source 444 | Code. 445 | 446 | 1.5. Initial Developer. means the individual or entity that first makes 447 | Original Software available under this License. 448 | 449 | 1.6. Larger Work. means a work which combines Covered Software or portions 450 | thereof with code not governed by the terms of this License. 451 | 452 | 1.7. License. means this document. 453 | 454 | 1.8. Licensable. means having the right to grant, to the maximum extent 455 | possible, whether at the time of the initial grant or subsequently 456 | acquired, any and all of the rights conveyed herein. 457 | 458 | 1.9. Modifications. means the Source Code and Executable form of any of the 459 | following: 460 | 461 | A. Any file that results from an addition to, deletion from or 462 | modification of the contents of a file containing Original Software 463 | or previous Modifications; 464 | 465 | B. Any new file that contains any part of the Original Software or 466 | previous Modification; or 467 | 468 | C. Any new file that is contributed or otherwise made available under 469 | the terms of this License. 470 | 471 | 1.10. Original Software. means the Source Code and Executable form of 472 | computer software code that is originally released under this License. 473 | 474 | 1.11. Patent Claims. means any patent claim(s), now owned or hereafter 475 | acquired, including without limitation, method, process, and apparatus 476 | claims, in any patent Licensable by grantor. 477 | 478 | 1.12. Source Code. means (a) the common form of computer software code in 479 | which modifications are made and (b) associated documentation included 480 | in or with such code. 481 | 482 | 1.13. You. (or .Your.) means an individual or a legal entity exercising 483 | rights under, and complying with all of the terms of, this License. For 484 | legal entities, .You. includes any entity which controls, is controlled 485 | by, or is under common control with You. For purposes of this 486 | definition, .control. means (a) the power, direct or indirect, to cause 487 | the direction or management of such entity, whether by contract or 488 | otherwise, or (b) ownership of more than fifty percent (50%) of the 489 | outstanding shares or beneficial ownership of such entity. 490 | 491 | 2. License Grants. 492 | 493 | 2.1. The Initial Developer Grant. 494 | 495 | Conditioned upon Your compliance with Section 3.1 below and subject to 496 | third party intellectual property claims, the Initial Developer hereby 497 | grants You a world-wide, royalty-free, non-exclusive license: 498 | 499 | (a) under intellectual property rights (other than patent or trademark) 500 | Licensable by Initial Developer, to use, reproduce, modify, display, 501 | perform, sublicense and distribute the Original Software (or 502 | portions thereof), with or without Modifications, and/or as part of 503 | a Larger Work; and 504 | 505 | (b) under Patent Claims infringed by the making, using or selling of 506 | Original Software, to make, have made, use, practice, sell, and 507 | offer for sale, and/or otherwise dispose of the Original Software 508 | (or portions thereof). 509 | 510 | (c) The licenses granted in Sections 2.1(a) and (b) are effective on the 511 | date Initial Developer first distributes or otherwise makes the 512 | Original Software available to a third party under the terms of this 513 | License. 514 | 515 | (d) Notwithstanding Section 2.1(b) above, no patent license is granted: 516 | (1) for code that You delete from the Original Software, or (2) for 517 | infringements caused by: (i) the modification of the Original 518 | Software, or (ii) the combination of the Original Software with 519 | other software or devices. 520 | 521 | 2.2. Contributor Grant. 522 | 523 | Conditioned upon Your compliance with Section 3.1 below and subject to third 524 | party intellectual property claims, each Contributor hereby grants You a 525 | world-wide, royalty-free, non-exclusive license: 526 | 527 | (a) under intellectual property rights (other than patent or trademark) 528 | Licensable by Contributor to use, reproduce, modify, display, 529 | perform, sublicense and distribute the Modifications created by such 530 | Contributor (or portions thereof), either on an unmodified basis, 531 | with other Modifications, as Covered Software and/or as part of a 532 | Larger Work; and 533 | 534 | (b) under Patent Claims infringed by the making, using, or selling of 535 | Modifications made by that Contributor either alone and/or in 536 | combination with its Contributor Version (or portions of such 537 | combination), to make, use, sell, offer for sale, have made, and/or 538 | otherwise dispose of: (1) Modifications made by that Contributor (or 539 | portions thereof); and (2) the combination of Modifications made by 540 | that Contributor with its Contributor Version (or portions of such 541 | combination). 542 | 543 | (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on 544 | the date Contributor first distributes or otherwise makes the 545 | Modifications available to a third party. 546 | 547 | (d) Notwithstanding Section 2.2(b) above, no patent license is granted: 548 | (1) for any code that Contributor has deleted from the Contributor 549 | Version; (2) for infringements caused by: (i) third party 550 | modifications of Contributor Version, or (ii) the combination of 551 | Modifications made by that Contributor with other software (except 552 | as part of the Contributor Version) or other devices; or (3) under 553 | Patent Claims infringed by Covered Software in the absence of 554 | Modifications made by that Contributor. 555 | 556 | 3. Distribution Obligations. 557 | 558 | 3.1. Availability of Source Code. 559 | Any Covered Software that You distribute or otherwise make available in 560 | Executable form must also be made available in Source Code form and that 561 | Source Code form must be distributed only under the terms of this License. 562 | You must include a copy of this License with every copy of the Source Code 563 | form of the Covered Software You distribute or otherwise make available. 564 | You must inform recipients of any such Covered Software in Executable form 565 | as to how they can obtain such Covered Software in Source Code form in a 566 | reasonable manner on or through a medium customarily used for software 567 | exchange. 568 | 569 | 3.2. Modifications. 570 | The Modifications that You create or to which You contribute are governed 571 | by the terms of this License. You represent that You believe Your 572 | Modifications are Your original creation(s) and/or You have sufficient 573 | rights to grant the rights conveyed by this License. 574 | 575 | 3.3. Required Notices. 576 | You must include a notice in each of Your Modifications that identifies 577 | You as the Contributor of the Modification. You may not remove or alter 578 | any copyright, patent or trademark notices contained within the Covered 579 | Software, or any notices of licensing or any descriptive text giving 580 | attribution to any Contributor or the Initial Developer. 581 | 582 | 3.4. Application of Additional Terms. 583 | You may not offer or impose any terms on any Covered Software in Source 584 | Code form that alters or restricts the applicable version of this License 585 | or the recipients. rights hereunder. You may choose to offer, and to 586 | charge a fee for, warranty, support, indemnity or liability obligations to 587 | one or more recipients of Covered Software. However, you may do so only on 588 | Your own behalf, and not on behalf of the Initial Developer or any 589 | Contributor. You must make it absolutely clear that any such warranty, 590 | support, indemnity or liability obligation is offered by You alone, and 591 | You hereby agree to indemnify the Initial Developer and every Contributor 592 | for any liability incurred by the Initial Developer or such Contributor as 593 | a result of warranty, support, indemnity or liability terms You offer. 594 | 595 | 3.5. Distribution of Executable Versions. 596 | You may distribute the Executable form of the Covered Software under the 597 | terms of this License or under the terms of a license of Your choice, 598 | which may contain terms different from this License, provided that You are 599 | in compliance with the terms of this License and that the license for the 600 | Executable form does not attempt to limit or alter the recipient.s rights 601 | in the Source Code form from the rights set forth in this License. If You 602 | distribute the Covered Software in Executable form under a different 603 | license, You must make it absolutely clear that any terms which differ 604 | from this License are offered by You alone, not by the Initial Developer 605 | or Contributor. You hereby agree to indemnify the Initial Developer and 606 | every Contributor for any liability incurred by the Initial Developer or 607 | such Contributor as a result of any such terms You offer. 608 | 609 | 3.6. Larger Works. 610 | You may create a Larger Work by combining Covered Software with other code 611 | not governed by the terms of this License and distribute the Larger Work 612 | as a single product. In such a case, You must make sure the requirements 613 | of this License are fulfilled for the Covered Software. 614 | 615 | 4. Versions of the License. 616 | 617 | 4.1. New Versions. 618 | Sun Microsystems, Inc. is the initial license steward and may publish 619 | revised and/or new versions of this License from time to time. Each 620 | version will be given a distinguishing version number. Except as provided 621 | in Section 4.3, no one other than the license steward has the right to 622 | modify this License. 623 | 624 | 4.2. Effect of New Versions. 625 | You may always continue to use, distribute or otherwise make the Covered 626 | Software available under the terms of the version of the License under 627 | which You originally received the Covered Software. If the Initial 628 | Developer includes a notice in the Original Software prohibiting it from 629 | being distributed or otherwise made available under any subsequent version 630 | of the License, You must distribute and make the Covered Software 631 | available under the terms of the version of the License under which You 632 | originally received the Covered Software. Otherwise, You may also choose 633 | to use, distribute or otherwise make the Covered Software available under 634 | the terms of any subsequent version of the License published by the 635 | license steward. 636 | 637 | 4.3. Modified Versions. 638 | When You are an Initial Developer and You want to create a new license for 639 | Your Original Software, You may create and use a modified version of this 640 | License if You: (a) rename the license and remove any references to the 641 | name of the license steward (except to note that the license differs from 642 | this License); and (b) otherwise make it clear that the license contains 643 | terms which differ from this License. 644 | 645 | 5. DISCLAIMER OF WARRANTY. 646 | 647 | COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN .AS IS. BASIS, WITHOUT 648 | WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT 649 | LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, 650 | MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK 651 | AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD 652 | ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL 653 | DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY 654 | SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN 655 | ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED 656 | HEREUNDER EXCEPT UNDER THIS DISCLAIMER. 657 | 658 | 6. TERMINATION. 659 | 660 | 6.1. This License and the rights granted hereunder will terminate 661 | automatically if You fail to comply with terms herein and fail to 662 | cure such breach within 30 days of becoming aware of the breach. 663 | Provisions which, by their nature, must remain in effect beyond the 664 | termination of this License shall survive. 665 | 666 | 6.2. If You assert a patent infringement claim (excluding declaratory 667 | judgment actions) against Initial Developer or a Contributor (the 668 | Initial Developer or Contributor against whom You assert such claim 669 | is referred to as .Participant.) alleging that the Participant 670 | Software (meaning the Contributor Version where the Participant is a 671 | Contributor or the Original Software where the Participant is the 672 | Initial Developer) directly or indirectly infringes any patent, then 673 | any and all rights granted directly or indirectly to You by such 674 | Participant, the Initial Developer (if the Initial Developer is not 675 | the Participant) and all Contributors under Sections 2.1 and/or 2.2 676 | of this License shall, upon 60 days notice from Participant terminate 677 | prospectively and automatically at the expiration of such 60 day 678 | notice period, unless if within such 60 day period You withdraw Your 679 | claim with respect to the Participant Software against such 680 | Participant either unilaterally or pursuant to a written agreement 681 | with Participant. 682 | 683 | 6.3. In the event of termination under Sections 6.1 or 6.2 above, all end 684 | user licenses that have been validly granted by You or any 685 | distributor hereunder prior to termination (excluding licenses 686 | granted to You by any distributor) shall survive termination. 687 | 688 | 7. LIMITATION OF LIABILITY. 689 | 690 | UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING 691 | NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY 692 | OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF 693 | ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, 694 | INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT 695 | LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, 696 | COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR 697 | LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF 698 | SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR 699 | DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY.S NEGLIGENCE TO THE EXTENT 700 | APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE 701 | EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS 702 | EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. 703 | 704 | 8. U.S. GOVERNMENT END USERS. 705 | 706 | The Covered Software is a .commercial item,. as that term is defined in 48 707 | C.F.R. 2.101 (Oct. 1995), consisting of .commercial computer software. (as 708 | that term is defined at 48 C.F.R. ? 252.227-7014(a)(1)) and commercial 709 | computer software documentation. as such terms are used in 48 C.F.R. 12.212 710 | (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 711 | through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered 712 | Software with only those rights set forth herein. This U.S. Government Rights 713 | clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or 714 | provision that addresses Government rights in computer software under this 715 | License. 716 | 717 | 9. MISCELLANEOUS. 718 | 719 | This License represents the complete agreement concerning subject matter 720 | hereof. If any provision of this License is held to be unenforceable, such 721 | provision shall be reformed only to the extent necessary to make it 722 | enforceable. This License shall be governed by the law of the jurisdiction 723 | specified in a notice contained within the Original Software (except to the 724 | extent applicable law, if any, provides otherwise), excluding such 725 | jurisdiction's conflict-of-law provisions. Any litigation relating to this 726 | License shall be subject to the jurisdiction of the courts located in the 727 | jurisdiction and venue specified in a notice contained within the Original 728 | Software, with the losing party responsible for costs, including, without 729 | limitation, court costs and reasonable attorneys. fees and expenses. The 730 | application of the United Nations Convention on Contracts for the 731 | International Sale of Goods is expressly excluded. Any law or regulation 732 | which provides that the language of a contract shall be construed against 733 | the drafter shall not apply to this License. You agree that You alone are 734 | responsible for compliance with the United States export administration 735 | regulations (and the export control laws and regulation of any other 736 | countries) when You use, distribute or otherwise make available any Covered 737 | Software. 738 | 739 | 10. RESPONSIBILITY FOR CLAIMS. 740 | 741 | As between Initial Developer and the Contributors, each party is responsible 742 | for claims and damages arising, directly or indirectly, out of its 743 | utilization of rights under this License and You agree to work with Initial 744 | Developer and Contributors to distribute such responsibility on an equitable 745 | basis. Nothing herein is intended or shall be deemed to constitute any 746 | admission of liability. 747 | 748 | NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION 749 | LICENSE (CDDL) 750 | 751 | The code released under the CDDL shall be governed by the laws of the State 752 | of California (excluding conflict-of-law provisions). Any litigation relating 753 | to this License shall be subject to the jurisdiction of the Federal Courts of 754 | the Northern District of California and the state courts of the State of 755 | California, with venue lying in Santa Clara County, California. 756 | 757 | 758 | ------------------------------------------------------------------------------ 759 | ANTLR 2 License (from ApacheDS, Groovy) 760 | ------------------------------------------------------------------------------ 761 | We reserve no legal rights to the ANTLR--it is fully in the public domain. 762 | An individual or company may do whatever they wish with source code 763 | distributed with ANTLR or the code generated by ANTLR, including the 764 | incorporation of ANTLR, or its output, into commerical software. 765 | We encourage users to develop software with ANTLR. However, we do ask that 766 | credit is given to us for developing ANTLR. By "credit", we mean that if you 767 | use ANTLR or incorporate any source code into one of your programs 768 | (commercial product, research project, or otherwise) that you acknowledge 769 | this fact somewhere in the documentation, research report, etc... If you like 770 | ANTLR and have developed a nice tool with the output, please mention that you 771 | developed it using ANTLR. In addition, we ask that the headers remain intact 772 | in our source code. As long as these guidelines are kept, we expect to 773 | continue enhancing this system and expect to make other tools available as 774 | they are completed. 775 | 776 | ------------------------------------------------------------------------------ 777 | ASM Project License (from CGLib, Groovy) 778 | ------------------------------------------------------------------------------ 779 | Copyright (c) 2000-2011 INRIA, France Telecom 780 | All rights reserved. 781 | 782 | Redistribution and use in source and binary forms, with or without 783 | modification, are permitted provided that the following conditions 784 | are met: 785 | 786 | 1. Redistributions of source code must retain the above copyright 787 | notice, this list of conditions and the following disclaimer. 788 | 789 | 2. Redistributions in binary form must reproduce the above copyright 790 | notice, this list of conditions and the following disclaimer in the 791 | documentation and/or other materials provided with the distribution. 792 | 793 | 3. Neither the name of the copyright holders nor the names of its 794 | contributors may be used to endorse or promote products derived from 795 | this software without specific prior written permission. 796 | 797 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 798 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 799 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 800 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 801 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 802 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 803 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 804 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 805 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 806 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 807 | THE POSSIBILITY OF SUCH DAMAGE. 808 | 809 | 810 | ------------------------------------------------------------------------------ 811 | Bouncy Castle License (from ApacheDS) 812 | ------------------------------------------------------------------------------ 813 | Copyright (c) 2000 - 2012 The Legion Of The Bouncy Castle 814 | (http://www.bouncycastle.org) 815 | 816 | Permission is hereby granted, free of charge, to any person obtaining a copy 817 | of this software and associated documentation files (the "Software"), to deal 818 | in the Software without restriction, including without limitation the rights 819 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 820 | copies of the Software, and to permit persons to whom the Software is furnished 821 | to do so, subject to the following conditions: 822 | 823 | The above copyright notice and this permission notice shall be included in 824 | all copies or substantial portions of the Software. 825 | 826 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 827 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 828 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 829 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 830 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 831 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 832 | SOFTWARE. 833 | 834 | 835 | ------------------------------------------------------------------------------ 836 | Eclipse Public License - v1.0 (from Jetty/Jerico) 837 | ------------------------------------------------------------------------------ 838 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 839 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 840 | CONSTITUTES RECIPIENT’S ACCEPTANCE OF THIS AGREEMENT. 841 | 842 | 1. DEFINITIONS 843 | 844 | "Contribution" means: 845 | 846 | a) in the case of the initial Contributor, the initial code and documentation 847 | distributed under this Agreement, and 848 | b) in the case of each subsequent Contributor: 849 | 850 | i)changes to the Program, and 851 | 852 | ii)additions to the Program; 853 | 854 | where such changes and/or additions to the Program originate from and are 855 | distributed by that particular Contributor. A Contribution 'originates' from 856 | a Contributor if it was added to the Program by such Contributor itself or 857 | anyone acting on such Contributor’s behalf. Contributions do not include 858 | additions to the Program which: (i) are separate modules of software 859 | distributed in conjunction with the Program under their own license agreement, 860 | and (ii) are not derivative works of the Program. 861 | 862 | "Contributor" means any person or entity that distributes the Program. 863 | 864 | "Licensed Patents " mean patent claims licensable by a Contributor which are 865 | necessarily infringed by the use or sale of its Contribution alone or when 866 | combined with the Program. 867 | 868 | "Program" means the Contributions distributed in accordance with this 869 | Agreement. 870 | 871 | "Recipient" means anyone who receives the Program under this Agreement, 872 | including all Contributors. 873 | 874 | 2. GRANT OF RIGHTS 875 | 876 | a) Subject to the terms of this Agreement, each Contributor hereby grants 877 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 878 | reproduce, prepare derivative works of, publicly display, publicly perform, 879 | distribute and sublicense the Contribution of such Contributor, if any, 880 | and such derivative works, in source code and object code form. 881 | 882 | b) Subject to the terms of this Agreement, each Contributor hereby grants 883 | Recipient a non-exclusive, worldwide, royalty-free patent license under 884 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 885 | transfer the Contribution of such Contributor, if any, in source code and 886 | object code form. This patent license shall apply to the combination of the 887 | Contribution and the Program if, at the time the Contribution is added by 888 | the Contributor, such addition of the Contribution causes such combination 889 | to be covered by the Licensed Patents. The patent license shall not apply 890 | to any other combinations which include the Contribution. No hardware per 891 | se is licensed hereunder. 892 | 893 | c) Recipient understands that although each Contributor grants the licenses 894 | to its Contributions set forth herein, no assurances are provided by any 895 | Contributor that the Program does not infringe the patent or other 896 | intellectual property rights of any other entity. Each Contributor 897 | disclaims any liability to Recipient for claims brought by any other 898 | entity based on infringement of intellectual property rights or otherwise. 899 | As a condition to exercising the rights and licenses granted hereunder, 900 | each Recipient hereby assumes sole responsibility to secure any other 901 | intellectual property rights needed, if any. For example, if a third 902 | party patent license is required to allow Recipient to distribute the 903 | Program, it is Recipient’s responsibility to acquire that license before 904 | distributing the Program. 905 | 906 | d) Each Contributor represents that to its knowledge it has sufficient 907 | copyright rights in its Contribution, if any, to grant the copyright 908 | license set forth in this Agreement. 909 | 910 | 3. REQUIREMENTS 911 | 912 | A Contributor may choose to distribute the Program in object code form under 913 | its own license agreement, provided that: 914 | 915 | a) it complies with the terms and conditions of this Agreement; and 916 | 917 | b) its license agreement: 918 | 919 | i) effectively disclaims on behalf of all Contributors all warranties and 920 | conditions, express and implied, including warranties or conditions of 921 | title and non-infringement, and implied warranties or conditions of 922 | merchantability and fitness for a particular purpose; 923 | 924 | ii) effectively excludes on behalf of all Contributors all liability for 925 | damages, including direct, indirect, special, incidental and consequential 926 | damages, such as lost profits; 927 | 928 | iii) states that any provisions which differ from this Agreement are offered by 929 | that Contributor alone and not by any other party; and 930 | 931 | iv) states that source code for the Program is available from such 932 | Contributor, and informs licensees how to obtain it in a reasonable manner 933 | on or through a medium customarily used for software exchange. 934 | 935 | When the Program is made available in source code form: 936 | 937 | a) it must be made available under this Agreement; and 938 | 939 | b) a copy of this Agreement must be included with each copy of the Program. 940 | 941 | Contributors may not remove or alter any copyright notices contained within 942 | the Program. 943 | 944 | Each Contributor must identify itself as the originator of its Contribution, 945 | if any, in a manner that reasonably allows subsequent Recipients to identify 946 | the originator of the Contribution. 947 | 948 | 4. COMMERCIAL DISTRIBUTION 949 | 950 | Commercial distributors of software may accept certain responsibilities with 951 | respect to end users, business partners and the like. While this license is 952 | intended to facilitate the commercial use of the Program, the Contributor who 953 | includes the Program in a commercial product offering should do so in a manner 954 | which does not create potential liability for other Contributors. Therefore, 955 | if a Contributor includes the Program in a commercial product offering, such 956 | Contributor ("Commercial Contributor") hereby agrees to defend and indemnify 957 | every other Contributor ("Indemnified Contributor") against any losses, 958 | damages and costs (collectively "Losses") arising from claims, lawsuits and 959 | other legal actions brought by a third party against the Indemnified 960 | Contributor to the extent caused by the acts or omissions of such Commercial 961 | Contributor in connection with its distribution of the Program in a commercial 962 | product offering. The obligations in this section do not apply to any claims 963 | or Losses relating to any actual or alleged intellectual property infringement. 964 | In order to qualify, an Indemnified Contributor must: a) promptly notify the 965 | Commercial Contributor in writing of such claim, and b) allow the Commercial 966 | Contributor to control, and cooperate with the Commercial Contributor in, the 967 | defense and any related settlement negotiations. The Indemnified Contributor 968 | may participate in any such claim at its own expense. 969 | 970 | For example, a Contributor might include the Program in a commercial product 971 | offering, Product X. That Contributor is then a Commercial Contributor. If 972 | that Commercial Contributor then makes performance claims, or offers 973 | warranties related to Product X, those performance claims and warranties are 974 | such Commercial Contributor’s responsibility alone. Under this section, the 975 | Commercial Contributor would have to defend claims against the other 976 | Contributors related to those performance claims and warranties, and if a 977 | court requires any other Contributor to pay any damages as a result, the 978 | Commercial Contributor must pay those damages. 979 | 980 | 5. NO WARRANTY 981 | 982 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 983 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS 984 | OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF 985 | TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. 986 | Each Recipient is solely responsible for determining the appropriateness of 987 | using and distributing the Program and assumes all risks associated with its 988 | exercise of rights under this Agreement , including but not limited to the 989 | risks and costs of program errors, compliance with applicable laws, damage to 990 | or loss of data, programs or equipment, and unavailability or interruption of 991 | operations. 992 | 993 | 6. DISCLAIMER OF LIABILITY 994 | 995 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 996 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 997 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 998 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 999 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 1000 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 1001 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 1002 | OF SUCH DAMAGES. 1003 | 1004 | 7. GENERAL 1005 | 1006 | If any provision of this Agreement is invalid or unenforceable under 1007 | applicable law, it shall not affect the validity or enforceability of the 1008 | remainder of the terms of this Agreement, and without further action by the 1009 | parties hereto, such provision shall be reformed to the minimum extent 1010 | necessary to make such provision valid and enforceable. 1011 | 1012 | If Recipient institutes patent litigation against any entity (including a 1013 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 1014 | (excluding combinations of the Program with other software or hardware) 1015 | infringes such Recipient’s patent(s), then such Recipient’s rights granted 1016 | under Section 2(b) shall terminate as of the date such litigation is filed. 1017 | 1018 | All Recipient’s rights under this Agreement shall terminate if it fails to 1019 | comply with any of the material terms or conditions of this Agreement and 1020 | does not cure such failure in a reasonable period of time after becoming 1021 | aware of such noncompliance. If all Recipient’s rights under this Agreement 1022 | terminate, Recipient agrees to cease use and distribution of the Program as 1023 | soon as reasonably practicable. However, Recipient’s obligations under this 1024 | Agreement and any licenses granted by Recipient relating to the Program shall 1025 | continue and survive. 1026 | 1027 | Everyone is permitted to copy and distribute copies of this Agreement, but in 1028 | order to avoid inconsistency the Agreement is copyrighted and may only be 1029 | modified in the following manner. The Agreement Steward reserves the right to 1030 | publish new versions (including revisions) of this Agreement from time to 1031 | time. No one other than the Agreement Steward has the right to modify this 1032 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 1033 | Eclipse Foundation may assign the responsibility to serve as the Agreement 1034 | Steward to a suitable separate entity. Each new version of the Agreement 1035 | will be given a distinguishing version number. The Program (including 1036 | Contributions) may always be distributed subject to the version of the 1037 | Agreement under which it was received. In addition, after a new version of 1038 | the Agreement is published, Contributor may elect to distribute the Program 1039 | (including its Contributions) under the new version. Except as expressly 1040 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 1041 | licenses to the intellectual property of any Contributor under this Agreement, 1042 | whether expressly, by implication, estoppel or otherwise. All rights in the 1043 | Program not expressly granted under this Agreement are reserved. 1044 | 1045 | This Agreement is governed by the laws of the State of New York and the 1046 | intellectual property laws of the United States of America. No party to this 1047 | Agreement will bring a legal action under this Agreement more than one year 1048 | after the cause of action arose. Each party waives its rights to a jury trial 1049 | in any resulting litigation. 1050 | 1051 | 1052 | -------------------------------------------------------------------------------------------------- 1053 | JDBM LICENSE v1.00 (from ApacheDS) 1054 | -------------------------------------------------------------------------------------------------- 1055 | /** 1056 | * JDBM LICENSE v1.00 1057 | * 1058 | * Redistribution and use of this software and associated documentation 1059 | * ("Software"), with or without modification, are permitted provided 1060 | * that the following conditions are met: 1061 | * 1062 | * 1. Redistributions of source code must retain copyright 1063 | * statements and notices. Redistributions must also contain a 1064 | * copy of this document. 1065 | * 1066 | * 2. Redistributions in binary form must reproduce the 1067 | * above copyright notice, this list of conditions and the 1068 | * following disclaimer in the documentation and/or other 1069 | * materials provided with the distribution. 1070 | * 1071 | * 3. The name "JDBM" must not be used to endorse or promote 1072 | * products derived from this Software without prior written 1073 | * permission of Cees de Groot. For written permission, 1074 | * please contact cg@cdegroot.com. 1075 | * 1076 | * 4. Products derived from this Software may not be called "JDBM" 1077 | * nor may "JDBM" appear in their names without prior written 1078 | * permission of Cees de Groot. 1079 | * 1080 | * 5. Due credit should be given to the JDBM Project 1081 | * (http://jdbm.sourceforge.net/). 1082 | * 1083 | * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS 1084 | * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT 1085 | * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 1086 | * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 1087 | * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 1088 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 1089 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 1090 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 1091 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 1092 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 1093 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 1094 | * OF THE POSSIBILITY OF SUCH DAMAGE. 1095 | * 1096 | * Copyright 2000 (C) Cees de Groot. All Rights Reserved. 1097 | * Contributions are Copyright (C) 2000 by their associated contributors. 1098 | * 1099 | * $Id: LICENSE.txt,v 1.1 2000/05/05 23:59:52 boisvert Exp $ 1100 | */ 1101 | 1102 | ------------------------------------------------------------------------------ 1103 | JLine License - BSD (from Groovy) 1104 | ------------------------------------------------------------------------------ 1105 | Copyright (c) 2002-2006, Marc Prud'hommeaux 1106 | All rights reserved. 1107 | 1108 | Redistribution and use in source and binary forms, with or 1109 | without modification, are permitted provided that the following 1110 | conditions are met: 1111 | 1112 | Redistributions of source code must retain the above copyright 1113 | notice, this list of conditions and the following disclaimer. 1114 | 1115 | Redistributions in binary form must reproduce the above copyright 1116 | notice, this list of conditions and the following disclaimer 1117 | in the documentation and/or other materials provided with 1118 | the distribution. 1119 | 1120 | Neither the name of JLine nor the names of its contributors 1121 | may be used to endorse or promote products derived from this 1122 | software without specific prior written permission. 1123 | 1124 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1125 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, 1126 | BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 1127 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 1128 | EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 1129 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 1130 | OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 1131 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 1132 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 1133 | AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 1134 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 1135 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 1136 | OF THE POSSIBILITY OF SUCH DAMAGE. 1137 | 1138 | 1139 | ------------------------------------------------------------------------------ 1140 | SL4J License - MIT 1141 | ------------------------------------------------------------------------------ 1142 | Copyright (c) 2004-2013 QOS.ch 1143 | All rights reserved. 1144 | 1145 | Permission is hereby granted, free of charge, to any person obtaining 1146 | a copy of this software and associated documentation files (the 1147 | "Software"), to deal in the Software without restriction, including 1148 | without limitation the rights to use, copy, modify, merge, publish, 1149 | distribute, sublicense, and/or sell copies of the Software, and to 1150 | permit persons to whom the Software is furnished to do so, subject to 1151 | the following conditions: 1152 | 1153 | The above copyright notice and this permission notice shall be 1154 | included in all copies or substantial portions of the Software. 1155 | 1156 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 1157 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 1158 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 1159 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 1160 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 1161 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 1162 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1163 | 1164 | ------------------------------------------------------------------------------ 1165 | Tanuki Software License (from ApacheDS) 1166 | ------------------------------------------------------------------------------ 1167 | Copyright (c) 1999, 2004 Tanuki Software 1168 | 1169 | Permission is hereby granted, free of charge, to any person 1170 | obtaining a copy of the Java Service Wrapper and associated 1171 | documentation files (the "Software"), to deal in the Software 1172 | without restriction, including without limitation the rights 1173 | to use, copy, modify, merge, publish, distribute, sub-license, 1174 | and/or sell copies of the Software, and to permit persons to 1175 | whom the Software is furnished to do so, subject to the 1176 | following conditions: 1177 | 1178 | The above copyright notice and this permission notice shall be 1179 | included in all copies or substantial portions of the Software. 1180 | 1181 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 1182 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 1183 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 1184 | NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 1185 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 1186 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 1187 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 1188 | OTHER DEALINGS IN THE SOFTWARE. 1189 | 1190 | 1191 | ------------------------------------------------------------------------------ 1192 | Silver Egg Technology License (from ApacheDS) 1193 | ------------------------------------------------------------------------------ 1194 | Portions of the Software have been derived from source code 1195 | developed by Silver Egg Technology under the following license: 1196 | 1197 | Copyright (c) 2001 Silver Egg Technology 1198 | 1199 | Permission is hereby granted, free of charge, to any person 1200 | obtaining a copy of this software and associated documentation 1201 | files (the "Software"), to deal in the Software without 1202 | restriction, including without limitation the rights to use, 1203 | copy, modify, merge, publish, distribute, sub-license, and/or 1204 | sell copies of the Software, and to permit persons to whom the 1205 | Software is furnished to do so, subject to the following 1206 | conditions: 1207 | 1208 | The above copyright notice and this permission notice shall be 1209 | included in all copies or substantial portions of the Software. 1210 | 1211 | 1212 | ------------------------------------------------------------------------------ 1213 | Sun Microsystems, Inc. License (from Groovy) 1214 | ------------------------------------------------------------------------------ 1215 | The following notice applies to the files: 1216 | 1217 | src/main/org/codehaus/groovy/jsr223/GroovyCompiledScript.java 1218 | src/main/org/codehaus/groovy/jsr223/GroovyScriptEngineFactory.java 1219 | src/main/org/codehaus/groovy/jsr223/GroovyScriptEngineImpl.java 1220 | 1221 | /* 1222 | * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 1223 | * Use is subject to license terms. 1224 | * 1225 | * Redistribution and use in source and binary forms, with or without modification, are 1226 | * permitted provided that the following conditions are met: Redistributions of source code 1227 | * must retain the above copyright notice, this list of conditions and the following disclaimer. 1228 | * Redistributions in binary form must reproduce the above copyright notice, this list of 1229 | * conditions and the following disclaimer in the documentation and/or other materials 1230 | * provided with the distribution. Neither the name of the Sun Microsystems nor the names of 1231 | * is contributors may be used to endorse or promote products derived from this software 1232 | * without specific prior written permission. 1233 | 1234 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS 1235 | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 1236 | * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 1237 | * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 1238 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 1239 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 1240 | * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 1241 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 1242 | * POSSIBILITY OF SUCH DAMAGE. 1243 | */ 1244 | 1245 | ------------------------------------------------------------------------------ 1246 | European Commission License (from Hadoop) 1247 | ------------------------------------------------------------------------------ 1248 | For the org.apache.hadoop.util.bloom.* classes: 1249 | 1250 | /** 1251 | * 1252 | * Copyright (c) 2005, European Commission project OneLab under contract 1253 | * 034819 (http://www.one-lab.org) 1254 | * All rights reserved. 1255 | * Redistribution and use in source and binary forms, with or 1256 | * without modification, are permitted provided that the following 1257 | * conditions are met: 1258 | * - Redistributions of source code must retain the above copyright 1259 | * notice, this list of conditions and the following disclaimer. 1260 | * - Redistributions in binary form must reproduce the above copyright 1261 | * notice, this list of conditions and the following disclaimer in 1262 | * the documentation and/or other materials provided with the distribution. 1263 | * - Neither the name of the University Catholique de Louvain - UCL 1264 | * nor the names of its contributors may be used to endorse or 1265 | * promote products derived from this software without specific prior 1266 | * written permission. 1267 | * 1268 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1269 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1270 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 1271 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 1272 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 1273 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 1274 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 1275 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 1276 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 1277 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 1278 | * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 1279 | * POSSIBILITY OF SUCH DAMAGE. 1280 | */ 1281 | 1282 | ------------------------------------------------------------------------------ 1283 | zlib/libpng License 1284 | ------------------------------------------------------------------------------ 1285 | This software is provided 'as-is', without any express or implied warranty. In 1286 | no event will the authors be held liable for any damages arising from the use of 1287 | this software. 1288 | 1289 | Permission is granted to anyone to use this software for any purpose, including 1290 | commercial applications, and to alter it and redistribute it freely, subject to 1291 | the following restrictions: 1292 | 1293 | 1. The origin of this software must not be misrepresented; you must not claim 1294 | that you wrote the original software. If you use this software in a 1295 | product, an acknowledgment in the product documentation would be 1296 | appreciated but is not required. 1297 | 2. Altered source versions must be plainly marked as such, and must not be 1298 | misrepresented as being the original software. 1299 | 3. This notice may not be removed or altered from any source distribution. 1300 | 1301 | 1302 | ------------------------------------------------------------------------------ 1303 | bzip2 License 1304 | ------------------------------------------------------------------------------ 1305 | Redistribution and use in source and binary forms, with or without modification, 1306 | are permitted provided that the following conditions are met: 1307 | 1308 | 1. Redistributions of source code must retain the above copyright notice, 1309 | this list of conditions and the following disclaimer. 1310 | 2. The origin of this software must not be misrepresented; you must not claim 1311 | that you wrote the original software. If you use this software in a 1312 | product, an acknowledgment in the product documentation would be 1313 | appreciated but is not required. 1314 | 3. Altered source versions must be plainly marked as such, and must not be 1315 | misrepresented as being the original software. 1316 | 4. The name of the author may not be used to endorse or promote products 1317 | derived from this software without specific prior written permission. 1318 | 1319 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS AND ANY EXPRESS OR IMPLIED 1320 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 1321 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 1322 | SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 1323 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 1324 | OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 1325 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 1326 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 1327 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 1328 | OF SUCH DAMAGE. 1329 | 1330 | Julian Seward, Cambridge, UK. 1331 | jseward@acm.org 1332 | 1333 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | docker build -t lewuathe/hadoop-base:latest hadoop-base 3 | docker build -t lewuathe/hadoop-master:latest hadoop-master 4 | docker build -t lewuathe/hadoop-slave:latest hadoop-slave 5 | docker-compose build 6 | 7 | .PHONY: test clean 8 | 9 | run: 10 | docker-compose up -d 11 | echo "http://localhost:9870 for HDFS" 12 | echo "http://localhost:8088 for YARN" 13 | 14 | down: 15 | docker-compose down 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-hadoop-cluster [![Build Status](https://travis-ci.org/Lewuathe/docker-hadoop-cluster.svg?branch=master)](https://travis-ci.org/Lewuathe/docker-hadoop-cluster) 2 | 3 | Multiple node cluster on Docker for self development. 4 | docker-hadoop-cluster is suitable for testing your patch for Hadoop which has multiple nodes. 5 | 6 | # Build images from your Hadoop source code 7 | 8 | Base image of hadoop service. This image includes JDK, hadoop package configurations etc. This image can include your self-build hadoop package. 9 | In order to bind, tar.gz package assumed be put under `hadoop-base` directory. 10 | 11 | ```bash 12 | $ cd docker-hadoop-cluster 13 | $ cp /path/to/hadoop-3.0.0-alpha3-SNAPSHOT.tar.gz hadoop-base 14 | $ make 15 | ``` 16 | 17 | Once you build `hadoop-base` image, you can launch hadoop cluster by using docker-compose. 18 | 19 | ``` 20 | $ docker-compose up -d 21 | ``` 22 | 23 | or 24 | 25 | ``` 26 | $ make run 27 | ``` 28 | 29 | See http://localhost:9870 for NameNode or http://localhost:8088 for ResourceManager. 30 | 31 | ``` 32 | $ make down 33 | ``` 34 | 35 | shutdowns your cluster. 36 | 37 | # Build images from the latest trunk 38 | 39 | docker-hadoop-cluster also uploads the latest image which refers HEAD of trunk. They are deployed on [Docker Hub](https://hub.docker.com/r/lewuathe/). 40 | If you want to try the trunk (though it can be unstable), `docker-compose.yml` like below is needed. It will launch 3 slave Hadoop cluster. 41 | 42 | ``` 43 | version: '2' 44 | 45 | services: 46 | master: 47 | image: lewuathe/hadoop-master 48 | ports: 49 | - "9870:9870" 50 | - "8088:8088" 51 | - "19888:19888" 52 | - "8188:8188" 53 | container_name: "master" 54 | slave1: 55 | image: lewuathe/hadoop-slave 56 | container_name: "slave1" 57 | depends_on: 58 | - master 59 | ports: 60 | - "9901:9864" 61 | - "8041:8042" 62 | slave2: 63 | image: lewuathe/hadoop-slave 64 | container_name: "slave2" 65 | depends_on: 66 | - master 67 | ports: 68 | - "9902:9864" 69 | - "8042:8042" 70 | slave3: 71 | image: lewuathe/hadoop-slave 72 | container_name: "slave3" 73 | depends_on: 74 | - master 75 | ports: 76 | - "9903:9864" 77 | - "8043:8042" 78 | 79 | ``` 80 | 81 | # Login cluster 82 | 83 | ```bash 84 | $ docker exec -it master bash 85 | bash-4.1# cd /usr/local/hadoop 86 | bash-4.1# bin/hadoop version 87 | Hadoop 3.0.0-SNAPSHOT 88 | Source code repository git://git.apache.org/hadoop.git -r 0c7d3f480548745e9e9ccad1d318371c020c3003 89 | Compiled by lewuathe on 2015-09-13T01:12Z 90 | Compiled with protoc 2.5.0 91 | From source with checksum 9174a352ac823cdfa576f525665e99 92 | This command was run using /usr/local/hadoop-3.0.0-SNAPSHOT/share/hadoop/common/hadoop-common-3.0.0-SNAPSHOT.jar 93 | ``` 94 | 95 | # Deploy on EC2 96 | 97 | We can run docker-hadoop-cluster on EC2 instance with `ec2/` script. 98 | 99 | ``` 100 | $ python ec2/ec2.py -k -s -n launch 101 | ``` 102 | 103 | This script launch EC2 instance and prepare prerequisites to launch docker-hadoop-cluster. 104 | 105 | # Docker Hub 106 | 107 | | Image name | Pulls | Stars | 108 | |:-----|:-----|:-----| 109 | | lewuathe/hadoop-base | [![hadoop-base](https://img.shields.io/docker/pulls/lewuathe/hadoop-base.svg)](https://hub.docker.com/r/lewuathe/hadoop-base/) | ![hadoop-base](https://img.shields.io/docker/stars/lewuathe/hadoop-base.svg) | 110 | | lewuathe/hadoop-master | [![hadoop-master](https://img.shields.io/docker/pulls/lewuathe/hadoop-master.svg)](https://hub.docker.com/r/lewuathe/hadoop-master/) | ![hadoop-master](https://img.shields.io/docker/stars/lewuathe/hadoop-master.svg) | 111 | | lewuathe/hadoop-slave | [![hadoop-slave](https://img.shields.io/docker/pulls/lewuathe/hadoop-slave.svg)](https://hub.docker.com/r/lewuathe/hadoop-slave/) | ![hadoop-slave](https://img.shields.io/docker/stars/lewuathe/hadoop-slave.svg) | 112 | 113 | # License 114 | 115 | [Apache License Version2.0](http://www.apache.org/licenses/LICENSE-2.0) 116 | This images are modified version of [sequenceiq/hadoop-docker](https://github.com/sequenceiq/hadoop-docker). 117 | -------------------------------------------------------------------------------- /bin/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROGNAME=$(basename $0) 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | 6 | $DIR/build_cluster.sh build -------------------------------------------------------------------------------- /bin/build_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROGNAME=$(basename $0) 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | 6 | print_usage() { 7 | echo "Usage: $0 [launch|destroy]" 8 | echo "" 9 | echo " launch : Launch hadoop cluster on docker" 10 | echo " destroy : Remove hadoop cluster on docker" 11 | echo " build : Build docker images with local hadoop binary" 12 | echo "" 13 | echo " Options:" 14 | echo " -h, --help : Print usage" 15 | echo " -s, --slaves : Specify the number of slaves" 16 | echo "" 17 | } 18 | 19 | if [ $# -eq 0 ]; then 20 | print_usage 21 | fi 22 | 23 | DATANODE_NUM=3 24 | CLUSTER_NAME=default_cluster 25 | 26 | for OPT in "$@" 27 | do 28 | case "$OPT" in 29 | '-h'|'--help' ) 30 | print_usage 31 | exit 1 32 | ;; 33 | '-s'|'--slaves' ) 34 | if [[ -z "$2" ]] || [[ "$2" =~ ^-+ ]]; then 35 | echo "$PROGNAME: option requires an argument -- $1" 1>&2 36 | exit 1 37 | fi 38 | DATANODE_NUM="$2" 39 | shift 2 40 | ;; 41 | '-c'|'--cluster' ) 42 | CLUSTER_NAME="$2" 43 | shift 2 44 | ;; 45 | -*) 46 | echo "$PROGNAME: illegal option -- '$(echo $1 | sed 's/^-*//')'" 1>&2 47 | exit 1 48 | ;; 49 | *) 50 | ;; 51 | esac 52 | done 53 | 54 | launch_cluster() { 55 | if ! docker network inspect hadoop-network > /dev/null ; then 56 | echo "Creating hadoop-network" 57 | docker network create --driver bridge hadoop-network 58 | fi 59 | echo "Launching master server" 60 | docker run -d -p 9870:9870 -p 8088:8088 -p 19888:19888 -p 8188:8188 --net hadoop-network --name master -h master lewuathe/hadoop-master:latest 61 | echo "Launching slave servers" 62 | for i in `seq 1 $DATANODE_NUM`; do 63 | docker run -d -p 990${i}:9864 -p 804${i}:8042 --name slave${i} -h slave${i} --net hadoop-network lewuathe/hadoop-slave:latest 64 | done 65 | } 66 | 67 | destroy_cluster() { 68 | docker kill master; docker rm master 69 | for i in `seq 1 $DATANODE_NUM`; do 70 | docker kill slave${i}; docker rm slave${i} 71 | done 72 | docker network rm hadoop-network 73 | } 74 | 75 | build_images() { 76 | cd $DIR/../hadoop-base 77 | docker build -f Dockerfile-local -t lewuathe/hadoop-base:latest . 78 | cd $DIR/../hadoop-master 79 | docker build -t lewuathe/hadoop-master:latest . 80 | cd $DIR/../hadoop-slave 81 | docker build -t lewuathe/hadoop-slave:latest . 82 | } 83 | 84 | case $1 in 85 | launch) launch_cluster 86 | ;; 87 | destroy) destroy_cluster 88 | ;; 89 | build) build_images 90 | ;; 91 | esac 92 | 93 | exit 0 94 | -------------------------------------------------------------------------------- /bin/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROGNAME=$(basename $0) 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | 6 | $DIR/build_cluster.sh -s $1 destroy -------------------------------------------------------------------------------- /bin/launch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROGNAME=$(basename $0) 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | 6 | $DIR/build_cluster.sh -s $1 launch -------------------------------------------------------------------------------- /bin/resolv_host.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import subprocess 5 | import json 6 | 7 | def exec_process(cmd): 8 | p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 9 | stdout_data, stderr_data = p.communicate() 10 | return p.returncode, stdout_data, stderr_data 11 | 12 | def get_ipaddress(host): 13 | (code, out, err) = exec_process('sudo docker inspect {}'.format(host)) 14 | conf = json.loads(out) 15 | return conf[0]['NetworkSettings']['IPAddress'] 16 | 17 | if __name__ == "__main__": 18 | argvs = sys.argv 19 | for h in range(0, int(argvs[1])): 20 | print("{}\tdn{}".format(get_ipaddress('dn' + str(h + 1)), h + 1)) 21 | print("{}\tdn{}.bridge".format(get_ipaddress('dn' + str(h + 1)), h + 1)) 22 | 23 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | master: 5 | build: ./hadoop-master 6 | ports: 7 | - "9870:9870" 8 | - "8088:8088" 9 | - "19888:19888" 10 | - "8188:8188" 11 | container_name: "master" 12 | slave1: 13 | build: ./hadoop-slave 14 | container_name: "slave1" 15 | ports: 16 | - "9901:9864" 17 | - "8041:8042" 18 | slave2: 19 | build: ./hadoop-slave 20 | container_name: "slave2" 21 | ports: 22 | - "9902:9864" 23 | - "8042:8042" 24 | slave3: 25 | build: ./hadoop-slave 26 | container_name: "slave3" 27 | ports: 28 | - "9903:9864" 29 | - "8043:8042" 30 | slave4: 31 | build: ./hadoop-slave 32 | container_name: "slave4" 33 | ports: 34 | - "9904:9864" 35 | - "8044:8042" 36 | slave5: 37 | build: ./hadoop-slave 38 | container_name: "slave5" 39 | ports: 40 | - "9905:9864" 41 | - "8045:8042" 42 | -------------------------------------------------------------------------------- /ec2/ec2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import time 4 | from optparse import OptionParser 5 | import boto3 6 | from fabric.api import local 7 | 8 | def launch(key_name, security_group, subnet_id): 9 | client = boto3.client('ec2') 10 | response = client.run_instances( 11 | ImageId = 'ami-f5f41398', 12 | MinCount = 1, 13 | MaxCount = 1, 14 | KeyName = key_name, 15 | SecurityGroupIds = [security_group], 16 | InstanceType = 'm4.xlarge', 17 | SubnetId = subnet_id, 18 | BlockDeviceMappings = [ 19 | { 20 | 'DeviceName': '/dev/xvda', 21 | 'Ebs': { 22 | 'VolumeSize': 80, 23 | 'DeleteOnTermination': True, 24 | 'VolumeType': 'gp2' 25 | } 26 | } 27 | ] 28 | ) 29 | instance_id = response['Instances'][0]['InstanceId'] 30 | waiter = client.get_waiter('instance_status_ok') 31 | waiter.wait( 32 | InstanceIds = [instance_id] 33 | ) 34 | response = client.describe_instances( 35 | InstanceIds = [ 36 | instance_id 37 | ] 38 | ) 39 | public_ip = response['Reservations'][0]['Instances'][0]['PublicIpAddress'] 40 | local("fab -f ./ec2/fabfile.py -i ~/.ssh/{}.pem -u ec2-user -H {} deploy".format(key_name, public_ip)) 41 | print('Succeed to launch instance') 42 | print(' InstanceId: {}, PublicIp: {}'.format(instance_id, public_ip)) 43 | print('Login') 44 | print(' ssh -i ~/.ssh/{}.pem ec2-user@{}'.format(key_name, public_ip)) 45 | 46 | def terminate(instance_id): 47 | client = boto3.client('ec2') 48 | client.terminate_instances( 49 | InstanceIds = [instance_id] 50 | ) 51 | 52 | if __name__ == "__main__": 53 | parser = OptionParser() 54 | parser.add_option('-k', '--key-name', 55 | dest='key_name', help='Specify key name to login the instance') 56 | parser.add_option('-s', '--security-group', 57 | dest='security_group', help='Specify security group to launch instance') 58 | parser.add_option('-n', '--subnet-id', 59 | dest='subnet_id', help='Specify subnet_id') 60 | parser.add_option('-i', '--instance', 61 | dest='instance_id', help='Specify instance id') 62 | 63 | (options, args) = parser.parse_args() 64 | 65 | if args[0] == 'launch': 66 | launch(key_name=options.key_name, security_group=options.security_group, subnet_id=options.subnet_id) 67 | elif args[0] == 'terminate': 68 | terminate(options.instance_id) 69 | -------------------------------------------------------------------------------- /ec2/fabfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from fabric.api import sudo, run 4 | 5 | def uname(): 6 | run('uname') 7 | 8 | def prepare(): 9 | sudo('yum install git docker -y') 10 | sudo('service docker start') 11 | sudo('gpasswd -a ec2-user docker') 12 | 13 | def install_docker_hadoop_cluster(): 14 | run('git clone https://github.com/Lewuathe/docker-hadoop-cluster.git') 15 | 16 | def deploy(): 17 | prepare() 18 | install_docker_hadoop_cluster() 19 | -------------------------------------------------------------------------------- /hadoop-base/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # This image is modified version of sequenceiq/hadoop-docker 3 | # * sequenceiq/hadoop-docker 4 | # 5 | # The modifications are 6 | # * Use local hadoop package 7 | # * Change template files to indicate docker master node 8 | # * Modify bootstrap script 9 | # 10 | # Author: Kai Sasaki 11 | # Date: 2015 Sep,13 12 | # 13 | # Creates multi node hadoop cluster on Docker 14 | 15 | FROM sequenceiq/pam:ubuntu-14.04 16 | MAINTAINER lewuathe 17 | 18 | USER root 19 | 20 | # install dev tools 21 | RUN apt-get update 22 | RUN apt-get install -y curl tar sudo openssh-server openssh-client rsync 23 | 24 | # passwordless ssh 25 | RUN rm -f /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_rsa_key /root/.ssh/id_rsa 26 | RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key 27 | RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key 28 | RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa 29 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 30 | 31 | # java 32 | RUN mkdir -p /usr/java/default && \ 33 | curl -Ls 'http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz' -H 'Cookie: oraclelicense=accept-securebackup-cookie' | \ 34 | tar --strip-components=1 -xz -C /usr/java/default/ 35 | 36 | # ADD jdk-8u112-linux-x64.tar.gz /usr/java 37 | # RUN sudo ln -s /usr/java/jdk1.8.0_112/ /usr/java/default 38 | 39 | ENV JAVA_HOME /usr/java/default 40 | ENV PATH $PATH:$JAVA_HOME/bin 41 | 42 | # download native support 43 | RUN mkdir -p /tmp/native 44 | RUN curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-2.7.0.tar | tar -x -C /tmp/native 45 | 46 | ENV HADOOP_VERSION=3.2.0-SNAPSHOT 47 | ADD hadoop-${HADOOP_VERSION}.tar.gz /usr/local/ 48 | WORKDIR /usr/local 49 | RUN ln -s /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop 50 | 51 | ENV HADOOP_HOME /usr/local/hadoop 52 | ENV HADOOP_COMMON_HOME /usr/local/hadoop 53 | ENV HADOOP_HDFS_HOME /usr/local/hadoop 54 | ENV HADOOP_MAPRED_HOME /usr/local/hadoop 55 | ENV HADOOP_YARN_HOME /usr/local/hadoop 56 | ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop 57 | ENV YARN_CONF_DIR /usr/local/hadoop/etc/hadoop 58 | ENV HADOOP_LOG_DIR /var/log/hadoop 59 | 60 | RUN mkdir /var/log/hadoop 61 | 62 | RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/java/default\nexport H=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_HOME/etc/hadoop/hadoop-env.sh 63 | RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_HOME/etc/hadoop/hadoop-env.sh 64 | #RUN . $HADOOP_HOME/etc/hadoop/hadoop-env.sh 65 | 66 | RUN mkdir $HADOOP_HOME/input 67 | RUN cp $HADOOP_HOME/etc/hadoop/*.xml $HADOOP_HOME/input 68 | 69 | ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml 70 | ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml 71 | ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml 72 | ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml 73 | ADD log4j.properties $HADOOP_HOME/etc/hadoop/log4j.properties 74 | 75 | RUN $HADOOP_HOME/bin/hdfs namenode -format 76 | 77 | # fixing the libhadoop.so like a boss 78 | RUN rm -rf /usr/local/hadoop/lib/native 79 | RUN mv /tmp/native /usr/local/hadoop/lib 80 | 81 | ADD ssh_config /root/.ssh/config 82 | RUN chmod 600 /root/.ssh/config 83 | RUN chown root:root /root/.ssh/config 84 | 85 | # workingaround docker.io build error 86 | RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh 87 | RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh 88 | RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh 89 | 90 | # fix the 254 error code 91 | RUN sed -i "/^[^#]*UsePAM/ s/.*/#&/" /etc/ssh/sshd_config 92 | RUN echo "UsePAM no" >> /etc/ssh/sshd_config 93 | RUN echo "Port 2122" >> /etc/ssh/sshd_config 94 | 95 | RUN service ssh start 96 | 97 | # Hdfs ports 98 | EXPOSE 9000 50010 50020 50070 50075 50090 99 | # See https://issues.apache.org/jira/browse/HDFS-9427 100 | EXPOSE 9871 9870 9820 9869 9868 9867 9866 9865 9864 101 | # Mapred ports 102 | EXPOSE 19888 103 | #Yarn ports 104 | EXPOSE 8030 8031 8032 8033 8040 8042 8088 8188 105 | #Other ports 106 | EXPOSE 49707 2122 107 | -------------------------------------------------------------------------------- /hadoop-base/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | fs.defaultFS 4 | hdfs://master:9000 5 | 6 | 7 | -------------------------------------------------------------------------------- /hadoop-base/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | dfs.replication 4 | 3 5 | 6 | 7 | dfs.namenode.ec.policies.enabled 8 | RS-3-2-1024k 9 | 10 | 11 | -------------------------------------------------------------------------------- /hadoop-base/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Define some default values that can be overridden by system properties 18 | hadoop.root.logger=INFO,console 19 | hadoop.log.dir=. 20 | hadoop.log.file=hadoop.log 21 | 22 | # Define the root logger to the system property "hadoop.root.logger". 23 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 24 | 25 | # Logging Threshold 26 | log4j.threshold=ALL 27 | 28 | # Null Appender 29 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 30 | 31 | # 32 | # Rolling File Appender - cap space usage at 5gb. 33 | # 34 | hadoop.log.maxfilesize=256MB 35 | hadoop.log.maxbackupindex=20 36 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 37 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 38 | 39 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 40 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 41 | 42 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 43 | 44 | # Pattern format: Date LogLevel LoggerName LogMessage 45 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 46 | # Debugging Pattern format 47 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 48 | 49 | 50 | # 51 | # Daily Rolling File Appender 52 | # 53 | 54 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 55 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 56 | 57 | # Rollover at midnight 58 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 59 | 60 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 61 | 62 | # Pattern format: Date LogLevel LoggerName LogMessage 63 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 64 | # Debugging Pattern format 65 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 66 | 67 | 68 | # 69 | # console 70 | # Add "console" to rootlogger above if you want to use this 71 | # 72 | 73 | log4j.appender.console=org.apache.log4j.ConsoleAppender 74 | log4j.appender.console.target=System.err 75 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 76 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 77 | 78 | # 79 | # TaskLog Appender 80 | # 81 | 82 | #Default values 83 | hadoop.tasklog.taskid=null 84 | hadoop.tasklog.iscleanup=false 85 | hadoop.tasklog.noKeepSplits=4 86 | hadoop.tasklog.totalLogFileSize=100 87 | hadoop.tasklog.purgeLogSplits=true 88 | hadoop.tasklog.logsRetainHours=12 89 | 90 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 91 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 92 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 93 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 94 | 95 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 96 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 97 | 98 | # 99 | # HDFS block state change log from block manager 100 | # 101 | # Uncomment the following to log normal block state change 102 | # messages from BlockManager in NameNode. 103 | #log4j.logger.BlockStateChange=DEBUG 104 | 105 | # 106 | #Security appender 107 | # 108 | hadoop.security.logger=INFO,NullAppender 109 | hadoop.security.log.maxfilesize=256MB 110 | hadoop.security.log.maxbackupindex=20 111 | log4j.category.SecurityLogger=${hadoop.security.logger} 112 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 113 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 114 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 115 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 116 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 117 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 118 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 119 | 120 | # 121 | # Daily Rolling Security appender 122 | # 123 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 124 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 125 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 126 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 127 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 128 | 129 | # 130 | # hadoop configuration logging 131 | # 132 | 133 | # Uncomment the following line to turn off configuration deprecation warnings. 134 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 135 | 136 | # 137 | # hdfs audit logging 138 | # 139 | hdfs.audit.logger=INFO,NullAppender 140 | hdfs.audit.log.maxfilesize=256MB 141 | hdfs.audit.log.maxbackupindex=20 142 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 143 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 144 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 145 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 146 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 147 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 148 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 149 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 150 | 151 | # 152 | # NameNode metrics logging. 153 | # The default is to retain two namenode-metrics.log files up to 64MB each. 154 | # 155 | namenode.metrics.logger=INFO,NullAppender 156 | log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger} 157 | log4j.additivity.NameNodeMetricsLog=false 158 | log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender 159 | log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log 160 | log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout 161 | log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n 162 | log4j.appender.NNMETRICSRFA.MaxBackupIndex=1 163 | log4j.appender.NNMETRICSRFA.MaxFileSize=64MB 164 | 165 | # 166 | # DataNode metrics logging. 167 | # The default is to retain two datanode-metrics.log files up to 64MB each. 168 | # 169 | datanode.metrics.logger=INFO,NullAppender 170 | log4j.logger.DataNodeMetricsLog=${datanode.metrics.logger} 171 | log4j.additivity.DataNodeMetricsLog=false 172 | log4j.appender.DNMETRICSRFA=org.apache.log4j.RollingFileAppender 173 | log4j.appender.DNMETRICSRFA.File=${hadoop.log.dir}/datanode-metrics.log 174 | log4j.appender.DNMETRICSRFA.layout=org.apache.log4j.PatternLayout 175 | log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n 176 | log4j.appender.DNMETRICSRFA.MaxBackupIndex=1 177 | log4j.appender.DNMETRICSRFA.MaxFileSize=64MB 178 | 179 | # 180 | # mapred audit logging 181 | # 182 | mapred.audit.logger=INFO,NullAppender 183 | mapred.audit.log.maxfilesize=256MB 184 | mapred.audit.log.maxbackupindex=20 185 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 186 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 187 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 188 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 189 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 190 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 191 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 192 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 193 | 194 | # Custom Logging levels 195 | 196 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 197 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 198 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 199 | 200 | # Jets3t library 201 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 202 | 203 | # AWS SDK & S3A FileSystem 204 | log4j.logger.com.amazonaws=ERROR 205 | log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR 206 | log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN 207 | 208 | # 209 | # Event Counter Appender 210 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 211 | # 212 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 213 | 214 | # 215 | # Job Summary Appender 216 | # 217 | # Use following logger to send summary to separate file defined by 218 | # hadoop.mapreduce.jobsummary.log.file : 219 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 220 | # 221 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 222 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 223 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 224 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 225 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 226 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 227 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 228 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 229 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 230 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 231 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 232 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 233 | 234 | # 235 | # shuffle connection log from shuffleHandler 236 | # Uncomment the following line to enable logging of shuffle connections 237 | # log4j.logger.org.apache.hadoop.mapred.ShuffleHandler.audit=DEBUG 238 | 239 | # 240 | # Yarn ResourceManager Application Summary Log 241 | # 242 | # Set the ResourceManager summary log filename 243 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 244 | # Set the ResourceManager summary log level and appender 245 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 246 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 247 | 248 | # To enable AppSummaryLogging for the RM, 249 | # set yarn.server.resourcemanager.appsummary.logger to 250 | # ,RMSUMMARY in hadoop-env.sh 251 | 252 | # Appender for ResourceManager Application Summary Log 253 | # Requires the following properties to be set 254 | # - hadoop.log.dir (Hadoop Log directory) 255 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 256 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 257 | 258 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 259 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 260 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 261 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 262 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 263 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 264 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 265 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 266 | 267 | # HS audit log configs 268 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 269 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 270 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 271 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 272 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 273 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 274 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 275 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 276 | 277 | # Http Server Request Logs 278 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 279 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 280 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 281 | #log4j.appender.namenoderequestlog.RetainDays=3 282 | 283 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 284 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 285 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 286 | #log4j.appender.datanoderequestlog.RetainDays=3 287 | 288 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 289 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 290 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 291 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 292 | 293 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 294 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 295 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 296 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 297 | 298 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 299 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 300 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 301 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 302 | 303 | # Appender for viewing information for errors and warnings 304 | yarn.ewma.cleanupInterval=300 305 | yarn.ewma.messageAgeLimitSeconds=86400 306 | yarn.ewma.maxUniqueMessages=250 307 | log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender 308 | log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} 309 | log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} 310 | log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} 311 | 312 | ## NameNode log 313 | log4j.appender.NAMENODE_RFA=org.apache.log4j.RollingFileAppender 314 | log4j.appender.NAMENODE_RFA.File=${hadoop.log.dir}/hadoop-namenode.log 315 | log4j.appender.NAMENODE_RFA.layout=org.apache.log4j.PatternLayout 316 | log4j.appender.NAMENODE_RFA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 317 | log4j.logger.org.apache.hadoop.hdfs.server.namenode=INFO,NAMENODE_RFA 318 | 319 | ## DataNode log 320 | log4j.appender.DATANODE_RFA=org.apache.log4j.RollingFileAppender 321 | log4j.appender.DATANODE_RFA.File=${hadoop.log.dir}/hadoop-datanode.log 322 | log4j.appender.DATANODE_RFA.layout=org.apache.log4j.PatternLayout 323 | log4j.appender.DATANODE_RFA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 324 | log4j.logger.org.apache.hadoop.hdfs.server.datanode=INFO,DATANODE_RFA 325 | 326 | ## ResourceManager log 327 | log4j.appender.RESOURCEMANAGER_RFA=org.apache.log4j.RollingFileAppender 328 | log4j.appender.RESOURCEMANAGER_RFA.File=${hadoop.log.dir}/hadoop-resourcemanager.log 329 | log4j.appender.RESOURCEMANAGER_RFA.layout=org.apache.log4j.PatternLayout 330 | log4j.appender.RESOURCEMANAGER_RFA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 331 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager=INFO,RESOURCEMANAGER_RFA 332 | 333 | ## NodeManager log 334 | log4j.appender.NODEMANAGER_RFA=org.apache.log4j.RollingFileAppender 335 | log4j.appender.NODEMANAGER_RFA.File=${hadoop.log.dir}/hadoop-nodemanager.log 336 | log4j.appender.NODEMANAGER_RFA.layout=org.apache.log4j.PatternLayout 337 | log4j.appender.NODEMANAGER_RFA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 338 | log4j.logger.org.apache.hadoop.yarn.server.nodemanager=INFO,NODEMANAGER_RFA 339 | 340 | ## HistoryServer log 341 | log4j.appender.HISTORYSERVER_RFA=org.apache.log4j.RollingFileAppender 342 | log4j.appender.HISTORYSERVER_RFA.File=${hadoop.log.dir}/hadoop-historyserver.log 343 | log4j.appender.HISTORYSERVER_RFA.layout=org.apache.log4j.PatternLayout 344 | log4j.appender.HISTORYSERVER_RFA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 345 | log4j.logger.org.apache.hadoop.mapreduce.v2.hs=INFO,HISTORYSERVER_RFA 346 | 347 | ## TimelineServer log 348 | log4j.appender.TIMELINESERVER_RFA=org.apache.log4j.RollingFileAppender 349 | log4j.appender.TIMELINESERVER_RFA.File=${hadoop.log.dir}/hadoop-timelineserver.log 350 | log4j.appender.TIMELINESERVER_RFA.layout=org.apache.log4j.PatternLayout 351 | log4j.appender.TIMELINESERVER_RFA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 352 | log4j.logger.org.apache.hadoop.yarn.server.applicationhistoryservice=INFO,TIMELINESERVER_RFA 353 | -------------------------------------------------------------------------------- /hadoop-base/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | mapreduce.framework.name 4 | yarn 5 | 6 | -------------------------------------------------------------------------------- /hadoop-base/ssh_config: -------------------------------------------------------------------------------- 1 | Host * 2 | UserKnownHostsFile /dev/null 3 | StrictHostKeyChecking no 4 | LogLevel quiet 5 | Port 2122 -------------------------------------------------------------------------------- /hadoop-base/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | yarn.nodemanager.aux-services 4 | mapreduce_shuffle 5 | 6 | 7 | 8 | yarn.application.classpath 9 | /usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/* 10 | 11 | 12 | 13 | 14 | Number of seconds after an application finishes before the nodemanager's 15 | DeletionService will delete the application's localized file directory 16 | and log directory. 17 | 18 | To diagnose Yarn application problems, set this property's value large 19 | enough (for example, to 600 = 10 minutes) to permit examination of these 20 | directories. After changing the property's value, you must restart the 21 | nodemanager in order for it to have an effect. 22 | 23 | The roots of Yarn applications' work directories is configurable with 24 | the yarn.nodemanager.local-dirs property (see below), and the roots 25 | of the Yarn applications' log directories is configurable with the 26 | yarn.nodemanager.log-dirs property (see also below). 27 | 28 | yarn.nodemanager.delete.debug-delay-sec 29 | 600 30 | 31 | 32 | 33 | yarn.resourcemanager.address 34 | master:8032 35 | 36 | 37 | yarn.resourcemanager.scheduler.address 38 | master:8030 39 | 40 | 41 | yarn.resourcemanager.resource-tracker.address 42 | master:8031 43 | 44 | 45 | 46 | yarn.log-aggregation-enable 47 | true 48 | 49 | 50 | yarn.timeline-service.enabled 51 | true 52 | 53 | 54 | yarn.timeline-service.hostname 55 | master 56 | 57 | 58 | yarn.timeline-service.generic-application-history.enabled 59 | true 60 | 61 | 62 | yarn.resourcemanager.system-metrics-publisher.enabled 63 | true 64 | 65 | 66 | 67 | yarn.webapp.ui2.enable 68 | true 69 | 70 | 71 | 72 | 82 | 83 | -------------------------------------------------------------------------------- /hadoop-master/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # This image is modified version of sequenceiq/hadoop-docker 3 | # * sequenceiq/hadoop-docker 4 | # 5 | # The modifications are 6 | # * Use local hadoop package 7 | # * Change template files to indicate docker master node 8 | # * Modify bootstrap script 9 | # 10 | # Author: Kai Sasaki 11 | # Date: 2015 Sep,13 12 | # 13 | # Creates multi node hadoop cluster on Docker 14 | 15 | FROM lewuathe/hadoop-base:latest 16 | MAINTAINER lewuathe 17 | 18 | ADD bootstrap.sh /etc/bootstrap.sh 19 | RUN chown root:root /etc/bootstrap.sh 20 | RUN chmod 700 /etc/bootstrap.sh 21 | 22 | ENV BOOTSTRAP /etc/bootstrap.sh 23 | 24 | CMD ["/etc/bootstrap.sh", "-d"] 25 | -------------------------------------------------------------------------------- /hadoop-master/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | : ${HADOOP_PREFIX:=/usr/local/hadoop} 4 | 5 | $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh 6 | 7 | rm /tmp/*.pid 8 | 9 | # installing libraries if any - (resource urls added comma separated to the ACP system variable) 10 | cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd - 11 | 12 | # altering the core-site configuration 13 | #sed s/NAMENODE/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml 14 | #sed s/NAMENODE/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/hdfs-site.xml.template > /usr/local/hadoop/etc/hadoop/hdfs-site.xml 15 | #sed s/RESOURCEMANAGER/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/yarn-site.xml.template.template > /usr/local/hadoop/etc/hadoop/yarn-site.xml.template 16 | 17 | service ssh start 18 | nohup $HADOOP_PREFIX/bin/hdfs namenode & 19 | nohup $HADOOP_PREFIX/bin/yarn resourcemanager & 20 | nohup $HADOOP_PREFIX/bin/yarn timelineserver & 21 | nohup $HADOOP_PREFIX/bin/mapred historyserver & 22 | 23 | if [[ $1 == "-d" ]]; then 24 | while true; do sleep 1000; done 25 | fi 26 | 27 | if [[ $1 == "-bash" ]]; then 28 | /bin/bash 29 | fi 30 | -------------------------------------------------------------------------------- /hadoop-slave/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # This image is modified version of sequenceiq/hadoop-docker 3 | # * sequenceiq/hadoop-docker 4 | # 5 | # The modifications are 6 | # * Use local hadoop package 7 | # * Change template files to indicate docker master node 8 | # * Modify bootstrap script 9 | # 10 | # Author: Kai Sasaki 11 | # Date: 2015 Sep,13 12 | # 13 | # Creates multi node hadoop cluster on Docker 14 | 15 | FROM lewuathe/hadoop-base:latest 16 | MAINTAINER lewuathe 17 | 18 | ADD bootstrap.sh /etc/bootstrap.sh 19 | RUN chown root:root /etc/bootstrap.sh 20 | RUN chmod 700 /etc/bootstrap.sh 21 | 22 | ENV BOOTSTRAP /etc/bootstrap.sh 23 | 24 | CMD ["/etc/bootstrap.sh", "-d"] 25 | -------------------------------------------------------------------------------- /hadoop-slave/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | : ${HADOOP_PREFIX:=/usr/local/hadoop} 4 | 5 | $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh 6 | 7 | rm /tmp/*.pid 8 | 9 | # installing libraries if any - (resource urls added comma separated to the ACP system variable) 10 | cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd - 11 | 12 | # altering the core-site configuration 13 | #sed s/NAMENODE/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml 14 | #sed s/NAMENODE/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/hdfs-site.xml.template > /usr/local/hadoop/etc/hadoop/hdfs-site.xml 15 | #sed s/RESOURCEMANAGER/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/yarn-site.xml.template.template > /usr/local/hadoop/etc/hadoop/yarn-site.xml.template 16 | 17 | service ssh start 18 | nohup $HADOOP_PREFIX/bin/hdfs datanode 2>> /var/log/hadoop/datanode.err >> /var/log/hadoop/datanode.out & 19 | nohup $HADOOP_PREFIX/bin/yarn nodemanager 2>> /var/log/hadoop/nodemanager.err >> /var/log/hadoop/nodemanager.out & 20 | 21 | if [[ $1 == "-d" ]]; then 22 | while true; do sleep 1000; done 23 | fi 24 | 25 | if [[ $1 == "-bash" ]]; then 26 | /bin/bash 27 | fi 28 | --------------------------------------------------------------------------------