├── .gitignore ├── LICENSE ├── README.md ├── batch ├── flink │ ├── Dockerfile-dev │ ├── build.sbt │ ├── json │ │ └── airports.json │ ├── parquet │ │ └── airports.parquet │ ├── project │ │ ├── build.properties │ │ └── plugins.sbt │ └── src │ │ └── main │ │ ├── resources │ │ ├── app.conf │ │ └── log4j.properties │ │ └── scala │ │ └── com │ │ └── mitosis │ │ ├── Main.scala │ │ ├── beans │ │ └── Airport.scala │ │ └── config │ │ ├── ConfigurationFactory.scala │ │ └── objects │ │ ├── BatchConfig.scala │ │ ├── BatchDbConfig.scala │ │ └── Config.scala ├── hadoop │ ├── Dockerfile-dev │ ├── json │ │ └── airports.json │ ├── parquet │ │ └── airports.parquet │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── mitosis │ │ │ ├── Application.java │ │ │ ├── MongoReducer.java │ │ │ └── ParquetMapper.java │ │ └── resources │ │ ├── airport.schema.avsc │ │ ├── app.conf │ │ └── log4j.properties └── spark │ ├── Dockerfile │ ├── Dockerfile-dev │ ├── build.sbt │ ├── json │ └── airports.json │ ├── parquet │ └── airports.parquet │ ├── pom.xml │ ├── project │ ├── build.properties │ └── plugins.sbt │ └── src │ └── main │ ├── java │ └── com │ │ └── mitosis │ │ └── Application.java │ ├── resources │ ├── app.conf │ └── log4j.properties │ └── scala │ └── com │ └── mitosis │ ├── Main.scala │ └── config │ ├── ConfigurationFactory.scala │ └── objects │ ├── BatchConfig.scala │ ├── BatchDbConfig.scala │ └── Config.scala ├── docker ├── dev │ ├── batch-flink.yml │ ├── batch-hadoop.yml │ ├── batch-spark.yml │ ├── ml-spark.yml │ ├── streaming-flink.yml │ ├── streaming-spark.yml │ ├── streaming-storm.yml │ └── webapp.yml ├── docker-compose.yml ├── flink.yml ├── hadoop-hbase.yml ├── kafka.yml ├── kylo.yml ├── mongo.yml ├── webapp.yml └── zookeeper.yml ├── ml └── spark │ ├── Dockerfile-model-dev │ ├── Dockerfile-stream-dev │ ├── flight-info.schema.avsc │ ├── model.py │ └── stream.py ├── streaming ├── flink │ ├── Dockerfile-dev │ ├── build.sbt │ ├── project │ │ ├── build.properties │ │ └── plugins.sbt │ └── src │ │ └── main │ │ ├── resources │ │ ├── app.conf │ │ ├── flight-info.schema.avsc │ │ ├── hbase-site.xml │ │ └── log4j.properties │ │ └── scala │ │ └── com │ │ └── mitosis │ │ ├── Main.scala │ │ ├── beans │ │ ├── CabinClass.scala │ │ ├── FlightInfoBean.scala │ │ └── TripType.scala │ │ ├── config │ │ ├── ConfigurationFactory.scala │ │ └── objects │ │ │ ├── Config.scala │ │ │ ├── ProducerConfig.scala │ │ │ ├── StreamingConfig.scala │ │ │ └── StreamingDbConfig.scala │ │ └── utils │ │ ├── FlightInfoAvroSchema.scala │ │ ├── FlightInfoHBaseOutputFormat.scala │ │ └── JsonUtils.scala ├── spark │ ├── Dockerfile │ ├── Dockerfile-dev │ ├── build.sbt │ ├── hbase-site.xml │ ├── pom.xml │ ├── project │ │ ├── build.properties │ │ └── plugins.sbt │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── mitosis │ │ │ ├── Application.java │ │ │ └── FlightInfoBean.java │ │ ├── resources │ │ ├── app.conf │ │ ├── flight-info.schema.avsc │ │ └── log4j.properties │ │ └── scala │ │ └── com │ │ └── mitosis │ │ ├── Main.scala │ │ ├── beans │ │ ├── CabinClass.scala │ │ ├── FlightInfoBean.scala │ │ └── TripType.scala │ │ ├── config │ │ ├── ConfigurationFactory.scala │ │ └── objects │ │ │ ├── Config.scala │ │ │ ├── ProducerConfig.scala │ │ │ ├── StreamingConfig.scala │ │ │ └── StreamingDbConfig.scala │ │ └── utils │ │ └── JsonUtils.scala └── storm │ ├── Dockerfile-dev │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── com │ │ └── mitosis │ │ ├── Application.java │ │ ├── FlightInfoBean.java │ │ └── spout │ │ ├── KafkaFlightInfoBolt.java │ │ └── KafkaSpoutTopology.java │ └── resources │ ├── app.conf │ ├── flight-info.schema.avsc │ ├── hbase-site.xml │ └── log4j.properties └── webapp ├── Dockerfile ├── Dockerfile-dev ├── client ├── .editorconfig ├── README.md ├── angular.json ├── package.json ├── protractor.conf.js ├── src │ ├── app │ │ ├── app.component.html │ │ ├── app.component.scss │ │ ├── app.component.ts │ │ ├── app.module.ts │ │ ├── app.routes.ts │ │ ├── dashboard │ │ │ ├── dashboard.component.html │ │ │ ├── dashboard.component.scss │ │ │ ├── dashboard.component.ts │ │ │ ├── dashboard.module.ts │ │ │ ├── dashboard.routes.ts │ │ │ └── search-flight │ │ │ │ ├── gql │ │ │ │ ├── dto │ │ │ │ │ ├── airport.dto.ts │ │ │ │ │ └── index.ts │ │ │ │ ├── mutation │ │ │ │ │ └── flight.mutations.ts │ │ │ │ ├── query │ │ │ │ │ └── airports.queries.ts │ │ │ │ ├── service │ │ │ │ │ ├── airports.service.ts │ │ │ │ │ ├── flight.service.ts │ │ │ │ │ ├── index.ts │ │ │ │ │ └── tweet.service.ts │ │ │ │ └── subscription │ │ │ │ │ └── tweet.subscription.ts │ │ │ │ ├── search-flight.component.html │ │ │ │ ├── search-flight.component.scss │ │ │ │ ├── search-flight.component.ts │ │ │ │ └── util │ │ │ │ ├── airport-dto-mapper.ts │ │ │ │ ├── airport-mapper.ts │ │ │ │ └── index.ts │ │ └── shared │ │ │ ├── material.module.ts │ │ │ └── model │ │ │ ├── airport.model.ts │ │ │ ├── flight-info.model.ts │ │ │ └── index.ts │ ├── assets │ │ └── .gitkeep │ ├── browserslist │ ├── environments │ │ ├── environment.prod.ts │ │ └── environment.ts │ ├── favicon.ico │ ├── index.html │ ├── karma.conf.js │ ├── main.ts │ ├── polyfills.ts │ ├── styles.scss │ ├── test.ts │ ├── tsconfig.app.json │ ├── tsconfig.spec.json │ └── tslint.json ├── tsconfig.json ├── tslint.json └── yarn.lock ├── package.json ├── server ├── package.json ├── server.js ├── src │ ├── db │ │ ├── config │ │ │ └── database.ts │ │ ├── index.ts │ │ └── models │ │ │ ├── airport.ts │ │ │ ├── flight-info.ts │ │ │ └── tweet.ts │ ├── index.ts │ ├── kafka │ │ ├── flight-info-avro.mapper.ts │ │ ├── flight-info-avro.ts │ │ ├── flight-info.producer.ts │ │ ├── index.ts │ │ └── tweet.consumer.ts │ └── schema │ │ ├── index.ts │ │ ├── resolvers │ │ └── resolvers.ts │ │ └── types │ │ └── types.ts ├── tsconfig.json ├── tslint.json └── yarn.lock └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | /.idea 3 | ######## webapp ######## 4 | 5 | # compiled output 6 | /webapp/dist 7 | /webapp/tmp 8 | 9 | # dependencies 10 | /webapp/node_modules 11 | /webapp/client/node_modules 12 | /webapp/server/node_modules 13 | 14 | # IDEs and editors 15 | /webapp/client/.idea 16 | /webapp/server/.idea 17 | /webapp/.idea 18 | /webapp/.project 19 | /webapp/.classpath 20 | /webapp/.c9/ 21 | /webapp/*.launch 22 | /webapp/.settings/ 23 | /webapp/*.sublime-workspace 24 | 25 | # IDE - VSCode 26 | .vscode/* 27 | !.vscode/settings.json 28 | !.vscode/tasks.json 29 | !.vscode/launch.json 30 | !.vscode/extensions.json 31 | 32 | # misc 33 | /webapp/.sass-cache 34 | /webapp/connect.lock 35 | /webapp/coverage/* 36 | /webapp/client/public 37 | /webapp/server/public 38 | /webapp/server/dist 39 | /webapp/libpeerconnection.log 40 | /webapp/npm-debug.log 41 | /webapp/testem.log 42 | /webapp/typings 43 | /webapp.*.log 44 | 45 | #System Files 46 | .DS_Store 47 | Thumbs.db 48 | *.log 49 | 50 | ######## batch spark ######## 51 | /batch/spark/.metadata 52 | /batch/spark/bin/ 53 | /batch/spark/tmp/ 54 | /batch/spark/target/ 55 | /batch/spark/.idea/ 56 | 57 | /batch/spark/.classpath 58 | /batch/spark/.project 59 | /batch/spark/*.tmp 60 | /batch/spark/*.bak 61 | /batch/spark/*.swp 62 | /batch/spark/*~.nib 63 | /batch/spark/local.properties 64 | /batch/spark/.settings/ 65 | /batch/spark/.loadpath 66 | /batch/spark/.recommenders 67 | 68 | # External tool builders 69 | /batch/spark/.externalToolBuilders/ 70 | 71 | # Locally stored "Eclipse launch configurations" 72 | /batch/spark/*.launch 73 | 74 | # PyDev specific (Python IDE for Eclipse) 75 | /batch/spark/*.pydevproject 76 | 77 | # CDT-specific (C/C++ Development Tooling) 78 | /batch/spark/.cproject 79 | 80 | # Java annotation processor (APT) 81 | /batch/spark/.factorypath 82 | 83 | # PDT-specific (PHP Development Tools) 84 | /batch/spark/.buildpath 85 | 86 | # sbteclipse plugin 87 | /batch/spark/.target 88 | /batch/spark/project/project 89 | /batch/spark/project/target 90 | 91 | # Tern plugin 92 | /batch/spark/.tern-project 93 | 94 | # TeXlipse plugin 95 | /batch/spark/.texlipse 96 | 97 | # STS (Spring Tool Suite) 98 | /batch/spark/.springBeans 99 | 100 | # Code Recommenders 101 | /batch/spark/.recommenders/ 102 | 103 | # Scala IDE specific (Scala & Java development for Eclipse) 104 | /batch/spark/.cache-main 105 | /batch/spark/.scala_dependencies 106 | /batch/spark/.worksheet 107 | 108 | ######## batch hadoop ######## 109 | /batch/hadoop/.metadata 110 | /batch/hadoop/bin/ 111 | /batch/hadoop/tmp/ 112 | /batch/hadoop/target/ 113 | /batch/hadoop/.idea/ 114 | /batch/hadoop/*.iml 115 | 116 | /batch/hadoop/.classpath 117 | /batch/hadoop/.project 118 | /batch/hadoop/*.tmp 119 | /batch/hadoop/*.bak 120 | /batch/hadoop/*.swp 121 | /batch/hadoop/*~.nib 122 | /batch/hadoop/local.properties 123 | /batch/hadoop/.settings/ 124 | /batch/hadoop/.loadpath 125 | /batch/hadoop/.recommenders 126 | 127 | ######## batch flink ######## 128 | /batch/flink/.metadata 129 | /batch/flink/bin/ 130 | /batch/flink/tmp/ 131 | /batch/flink/target/ 132 | /batch/flink/.idea/ 133 | 134 | /batch/flink/.classpath 135 | /batch/flink/.project 136 | /batch/flink/*.tmp 137 | /batch/flink/*.bak 138 | /batch/flink/*.swp 139 | /batch/flink/*~.nib 140 | /batch/flink/local.properties 141 | /batch/flink/.settings/ 142 | /batch/flink/.loadpath 143 | /batch/flink/.recommenders 144 | 145 | # External tool builders 146 | /batch/flink/.externalToolBuilders/ 147 | 148 | # Locally stored "Eclipse launch configurations" 149 | /batch/flink/*.launch 150 | 151 | # PyDev specific (Python IDE for Eclipse) 152 | /batch/flink/*.pydevproject 153 | 154 | # CDT-specific (C/C++ Development Tooling) 155 | /batch/flink/.cproject 156 | 157 | # Java annotation processor (APT) 158 | /batch/flink/.factorypath 159 | 160 | # PDT-specific (PHP Development Tools) 161 | /batch/flink/.buildpath 162 | 163 | # sbteclipse plugin 164 | /batch/flink/.target 165 | /batch/flink/project/project 166 | /batch/flink/project/target 167 | 168 | # Tern plugin 169 | /batch/flink/.tern-project 170 | 171 | # TeXlipse plugin 172 | /batch/flink/.texlipse 173 | 174 | # STS (Spring Tool Suite) 175 | /batch/flink/.springBeans 176 | 177 | # Code Recommenders 178 | /batch/flink/.recommenders/ 179 | 180 | # Scala IDE specific (Scala & Java development for Eclipse) 181 | /batch/flink/.cache-main 182 | /batch/flink/.scala_dependencies 183 | /batch/flink/.worksheet 184 | 185 | ######## streaming - spark ######## 186 | /streaming/spark/.metadata 187 | /streaming/spark/bin/ 188 | /streaming/spark/tmp/ 189 | /streaming/spark/target/ 190 | /streaming/spark/.idea/ 191 | 192 | /streaming/spark/.classpath 193 | /streaming/spark/.project 194 | /streaming/spark/*.tmp 195 | /streaming/spark/*.bak 196 | /streaming/spark/*.swp 197 | /streaming/spark/*~.nib 198 | /streaming/spark/local.properties 199 | /streaming/spark/.settings/ 200 | /streaming/spark/.loadpath 201 | /streaming/spark/.recommenders 202 | 203 | # External tool builders 204 | /streaming/spark/.externalToolBuilders/ 205 | 206 | # Locally stored "Eclipse launch configurations" 207 | /streaming/spark/*.launch 208 | 209 | # PyDev specific (Python IDE for Eclipse) 210 | /streaming/spark/*.pydevproject 211 | 212 | # CDT-specific (C/C++ Development Tooling) 213 | /streaming/spark/.cproject 214 | 215 | # Java annotation processor (APT) 216 | /streaming/spark/.factorypath 217 | 218 | # PDT-specific (PHP Development Tools) 219 | /streaming/spark/.buildpath 220 | 221 | # sbteclipse plugin 222 | /streaming/spark/.target 223 | /streaming/spark/project/project 224 | /streaming/spark/project/target 225 | 226 | # Tern plugin 227 | /streaming/spark/.tern-project 228 | 229 | # TeXlipse plugin 230 | /streaming/spark/.texlipse 231 | 232 | # STS (Spring Tool Suite) 233 | /streaming/spark/.springBeans 234 | 235 | # Code Recommenders 236 | /streaming/spark/.recommenders/ 237 | 238 | # Scala IDE specific (Scala & Java development for Eclipse) 239 | /streaming/spark/.cache-main 240 | /streaming/spark/.scala_dependencies 241 | /streaming/spark/.worksheet 242 | 243 | ######## streaming - flink ######## 244 | /streaming/flink/.metadata 245 | /streaming/flink/bin/ 246 | /streaming/flink/tmp/ 247 | /streaming/flink/target/ 248 | /streaming/flink/.idea/ 249 | 250 | /streaming/flink/.classpath 251 | /streaming/flink/.project 252 | /streaming/flink/*.tmp 253 | /streaming/flink/*.bak 254 | /streaming/flink/*.swp 255 | /streaming/flink/*~.nib 256 | /streaming/flink/local.properties 257 | /streaming/flink/.settings/ 258 | /streaming/flink/.loadpath 259 | /streaming/flink/.recommenders 260 | 261 | # External tool builders 262 | /streaming/flink/.externalToolBuilders/ 263 | 264 | # Locally stored "Eclipse launch configurations" 265 | /streaming/flink/*.launch 266 | 267 | # PyDev specific (Python IDE for Eclipse) 268 | /streaming/flink/*.pydevproject 269 | 270 | # CDT-specific (C/C++ Development Tooling) 271 | /streaming/flink/.cproject 272 | 273 | # Java annotation processor (APT) 274 | /streaming/flink/.factorypath 275 | 276 | # PDT-specific (PHP Development Tools) 277 | /streaming/flink/.buildpath 278 | 279 | # sbteclipse plugin 280 | /streaming/flink/.target 281 | /streaming/flink/project/project 282 | /streaming/flink/project/target 283 | 284 | # Tern plugin 285 | /streaming/flink/.tern-project 286 | 287 | # TeXlipse plugin 288 | /streaming/flink/.texlipse 289 | 290 | # STS (Spring Tool Suite) 291 | /streaming/flink/.springBeans 292 | 293 | # Code Recommenders 294 | /streaming/flink/.recommenders/ 295 | 296 | # Scala IDE specific (Scala & Java development for Eclipse) 297 | /streaming/flink/.cache-main 298 | /streaming/flink/.scala_dependencies 299 | /streaming/flink/.worksheet 300 | 301 | 302 | ######## streaming - storm ######## 303 | /streaming/storm/.metadata 304 | /streaming/storm/bin/ 305 | /streaming/storm/tmp/ 306 | /streaming/storm/target/ 307 | /streaming/storm/logs/ 308 | /streaming/storm/.idea/ 309 | 310 | /streaming/storm/.classpath 311 | /streaming/storm/.project 312 | /streaming/storm/*.tmp 313 | /streaming/storm/*.bak 314 | /streaming/storm/*.swp 315 | /streaming/storm/*~.nib 316 | /streaming/storm/local.properties 317 | /streaming/storm/.settings/ 318 | /streaming/storm/.loadpath 319 | /streaming/storm/.recommenders 320 | 321 | # External tool builders 322 | /streaming/storm/.externalToolBuilders/ 323 | 324 | # Locally stored "Eclipse launch configurations" 325 | /streaming/storm/*.launch 326 | 327 | # PyDev specific (Python IDE for Eclipse) 328 | /streaming/storm/*.pydevproject 329 | 330 | # CDT-specific (C/C++ Development Tooling) 331 | /streaming/storm/.cproject 332 | 333 | # Java annotation processor (APT) 334 | /streaming/storm/.factorypath 335 | 336 | # PDT-specific (PHP Development Tools) 337 | /streaming/storm/.buildpath 338 | 339 | # sbteclipse plugin 340 | /streaming/storm/.target 341 | /streaming/storm/project/project 342 | /streaming/storm/project/target 343 | 344 | # Tern plugin 345 | /streaming/storm/.tern-project 346 | 347 | # TeXlipse plugin 348 | /streaming/storm/.texlipse 349 | 350 | # STS (Spring Tool Suite) 351 | /streaming/storm/.springBeans 352 | 353 | # Code Recommenders 354 | /streaming/storm/.recommenders/ 355 | 356 | # Scala IDE specific (Scala & Java development for Eclipse) 357 | /streaming/storm/.cache-main 358 | /streaming/storm/.scala_dependencies 359 | /streaming/storm/.worksheet 360 | 361 | ######## ml ######## 362 | /ml/spark/.idea/ 363 | /ml/spark/venv/ 364 | /ml/spark/tweet_traveling_partners_model 365 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bigdata Playground 2 | 3 | The aim is to create a Batch/Streaming/ML/WebApp stack where you can test your jobs locally or to submit them to the Yarn resource manager. We are using Docker to build the environment and Docker-Compose to provision it with the required components (Next step using Kubernetes). Along with the infrastructure, We are check that it works with 4 projects that just probes everything is working as expected. The boilerplate is based on a sample search flight Web application. 4 | 5 | ## Installation 6 | If you are on mac then, you can use package manager like `brew` to install `sbt` on your machine: 7 | 8 | ```bash 9 | $ brew install sbt 10 | ``` 11 | 12 | For other systems, you can refer to manual instructions from `sbt` website http://www.scala-sbt.org/0.13/tutorial/Manual-Installation.html. 13 | 14 | If you are on mac then, you can use package manager like `brew` to install `maven` on your machine: 15 | ```bash 16 | $ brew install maven 17 | ``` 18 | For other systems, you can refer to manual instructions from `maven` website https://maven.apache.org/install.html. 19 | 20 | Install Docker by following the instructions for mac, linux, or windows. 21 | 22 | ``` 23 | docker network create vnet 24 | npm install yarn -g 25 | cd webapp && yarn && cd client && yarn && cd ../server && yarn && cd ../ && npm run build:dev && cd ../ 26 | cd batch/spark && sbt clean package assembly && cd ../.. 27 | 28 | cd batch/hadoop && mvn clean package && cd ../.. 29 | cd streaming/spark && sbt clean assembly && cd ../.. 30 | cd streaming/flink && sbt clean assembly && cd ../.. 31 | cd streaming/storm && mvn clean package && cd ../.. 32 | cd docker 33 | docker-compose -f mongo.yml -f zookeeper.yml -f kafka.yml -f hadoop-hbase.yml -f flink.yml up -d 34 | docker-compose -f dev/webapp.yml up -d 35 | docker-compose -f dev/batch-spark.yml up -d 36 | docker-compose -f dev/batch-hadoop.yml up -d 37 | docker-compose -f dev/streaming-spark.yml up -d 38 | docker-compose -f dev/streaming-flink.yml up -d 39 | docker-compose -f dev/streaming-storm.yml up -d 40 | ``` 41 | Create your Twitter app on https://apps.twitter.com 42 | ``` 43 | export TWITTER_CONSUMER_KEY= 44 | export TWITTER_CONSUMER_SECRET= 45 | export TWITTER_CONSUMER_ACCESS_TOKEN= 46 | export TWITTER_CONSUMER_ACCESS_TOKEN_SECRET= 47 | docker-compose -f dev/ml-spark.yml up -d 48 | ``` 49 | 50 | ## Interactions / OnGoing 51 | 52 | 53 | ## Contributing 54 | `Pull requests` are welcome. 55 | 56 | ## Support 57 | Please raise tickets for issues and improvements at https://github.com/Chabane/bigdata-playground/issues 58 | 59 | ## License 60 | This example is released under version 2.0 of the [Apache License](LICENSE). 61 | 62 | -------------------------------------------------------------------------------- /batch/flink/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM flink:1.6.2-hadoop28-alpine 2 | VOLUME /tmp 3 | 4 | ADD target/scala-2.11/search-flight-flink-batch-assembly-0.1.0.jar $FLINK_HOME/libs/search-flight-batch.jar 5 | RUN sh -c 'touch $FLINK_HOME/libs/search-flight-batch.jar' 6 | 7 | WORKDIR $FLINK_HOME 8 | 9 | CMD ["./bin/flink", "run", "-m", "jobmanager:6123", "-c", "com.mitosis.Main", "./libs/search-flight-batch.jar", "--port", "9000"] 10 | -------------------------------------------------------------------------------- /batch/flink/build.sbt: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | resolvers in ThisBuild ++= Seq("Apache Development Snapshot Repository" at "https://repository.apache.org/content/repositories/snapshots/", 5 | Resolver.mavenLocal) 6 | 7 | name := "search-flight-flink-batch" 8 | organization := "com.mitosis" 9 | version := "0.1.0" 10 | 11 | mainClass in assembly := Some("com.mitosis.Main") 12 | 13 | scalaVersion := "2.11.11" 14 | 15 | val flinkVersion = "1.4.1" 16 | val jacksonVersion = "2.9.8" 17 | val typesafeVersion = "1.3.0" 18 | val log4jVersion = "1.2.14" 19 | 20 | val projectDependencies = Seq( 21 | "log4j" % "log4j" % log4jVersion, 22 | "org.apache.flink" %% "flink-scala" % flinkVersion % "provided", 23 | 24 | "com.typesafe" % "config" % typesafeVersion, 25 | "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, 26 | "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion 27 | ) 28 | 29 | lazy val root = (project in file(".")). 30 | settings( 31 | libraryDependencies ++= projectDependencies 32 | ) 33 | 34 | // make run command include the provided dependencies 35 | run in Compile := Defaults.runTask(fullClasspath in Compile, 36 | mainClass in (Compile, run), 37 | runner in (Compile,run) 38 | ).evaluated 39 | 40 | // exclude Scala library from assembly 41 | assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) 42 | -------------------------------------------------------------------------------- /batch/flink/parquet/airports.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Chabane/bigdata-playground/6838b5e33fefe81b7e1514c057dad5ebc48f4132/batch/flink/parquet/airports.parquet -------------------------------------------------------------------------------- /batch/flink/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.2 2 | -------------------------------------------------------------------------------- /batch/flink/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // https://github.com/sbt/sbt/wiki/sbt-1.x-plugin-migration 2 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 3 | // addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.1") 4 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.2-RC2") -------------------------------------------------------------------------------- /batch/flink/src/main/resources/app.conf: -------------------------------------------------------------------------------- 1 | batch { 2 | db { 3 | host: "mongo" 4 | port: 27017 5 | database: "mitosis" 6 | collection: "airports" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /batch/flink/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=DEBUG, CA 2 | 3 | log4j.appender.CA=org.apache.log4j.ConsoleAppender 4 | 5 | log4j.appender.CA.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -------------------------------------------------------------------------------- /batch/flink/src/main/scala/com/mitosis/Main.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis 2 | 3 | import com.mitosis.config.ConfigurationFactory 4 | import org.apache.log4j.Logger 5 | 6 | import org.apache.flink.api.scala._ 7 | 8 | object Main { 9 | 10 | private[this] lazy val logger = Logger.getLogger(getClass) 11 | 12 | private[this] val config: com.mitosis.config.objects.Config = ConfigurationFactory.load() 13 | 14 | def main(args: Array[String]): Unit = { 15 | val env = ExecutionEnvironment.getExecutionEnvironment 16 | // ParquetInputFormat not implemented yet - FLINK-7243 - https://issues.apache.org/jira/browse/FLINK-7243 17 | // MongoDB Connector not implemented yet - FLINK-6573 - https://issues.apache.org/jira/browse/FLINK-6573 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /batch/flink/src/main/scala/com/mitosis/beans/Airport.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class Airport { 9 | 10 | @BeanProperty 11 | var code: String = _ 12 | 13 | @BeanProperty 14 | var city: String = _ 15 | 16 | @BeanProperty 17 | var airport: String = _ 18 | } 19 | -------------------------------------------------------------------------------- /batch/flink/src/main/scala/com/mitosis/config/ConfigurationFactory.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config 2 | 3 | import com.mitosis.config.objects.Config 4 | import com.typesafe.config.ConfigBeanFactory 5 | import com.typesafe.config.ConfigFactory 6 | 7 | import java.io.File 8 | import java.io.InputStreamReader 9 | 10 | object ConfigurationFactory { 11 | 12 | /** 13 | * Loads configuration object by file 14 | * 15 | * @return Returns configuration objects 16 | */ 17 | def load(): Config = { 18 | val is = new InputStreamReader(getClass.getResourceAsStream(s"/app.conf")) 19 | val config: com.typesafe.config.Config = ConfigFactory.parseReader(is).resolve() 20 | ConfigBeanFactory.create(config, classOf[Config]) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /batch/flink/src/main/scala/com/mitosis/config/objects/BatchConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class BatchConfig { 9 | 10 | @BeanProperty 11 | var db: BatchDbConfig = _ 12 | } 13 | -------------------------------------------------------------------------------- /batch/flink/src/main/scala/com/mitosis/config/objects/BatchDbConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class BatchDbConfig { 9 | 10 | @BeanProperty 11 | var host: String = _ 12 | 13 | @BeanProperty 14 | var port: Int = _ 15 | 16 | @BeanProperty 17 | var database: String = _ 18 | 19 | @BeanProperty 20 | var collection: String = _ 21 | } 22 | -------------------------------------------------------------------------------- /batch/flink/src/main/scala/com/mitosis/config/objects/Config.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class Config { 9 | 10 | @BeanProperty 11 | var batch: BatchConfig = _ 12 | } 13 | -------------------------------------------------------------------------------- /batch/hadoop/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre 2 | 3 | ENV HADOOP_HOME /usr/local/hadoop 4 | 5 | ADD target/search-flight-hadoop-batch-0.1.0-jar-with-dependencies.jar $HADOOP_HOME/libs/search-flight-batch.jar 6 | ADD parquet/ $HADOOP_HOME/parquet/ 7 | 8 | WORKDIR $HADOOP_HOME 9 | CMD ["java", "-jar", "./libs/search-flight-batch.jar" ] 10 | -------------------------------------------------------------------------------- /batch/hadoop/parquet/airports.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Chabane/bigdata-playground/6838b5e33fefe81b7e1514c057dad5ebc48f4132/batch/hadoop/parquet/airports.parquet -------------------------------------------------------------------------------- /batch/hadoop/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.mitosis 7 | search-flight-hadoop-batch 8 | 0.1.0 9 | 10 | 11 | 2.11.11 12 | 13 | 2.4.0 14 | 2.2.1 15 | 1.3.0 16 | 1.2.14 17 | 1.16.20 18 | 20180813 19 | 1.10.0 20 | 1.5.1 21 | 1.8.2 22 | 23 | 24 | 25 | 26 | 27 | org.apache.maven.plugins 28 | maven-compiler-plugin 29 | 30 | 1.8 31 | 1.8 32 | 33 | 34 | 35 | 36 | org.apache.maven.plugins 37 | maven-assembly-plugin 38 | 39 | 40 | package 41 | 42 | single 43 | 44 | 45 | 46 | 47 | 48 | com.mitosis.Application 49 | 50 | 51 | 52 | 53 | jar-with-dependencies 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | com.typesafe 65 | config 66 | ${typesafe.version} 67 | 68 | 69 | 70 | log4j 71 | log4j 72 | ${log4j.version} 73 | 74 | 75 | 76 | org.projectlombok 77 | lombok 78 | ${lombok.version} 79 | 80 | 81 | org.json 82 | json 83 | ${json.version} 84 | 85 | 86 | 87 | 88 | 89 | org.apache.parquet 90 | parquet-common 91 | ${parquet.version} 92 | 93 | 94 | org.apache.parquet 95 | parquet-encoding 96 | ${parquet.version} 97 | 98 | 99 | org.apache.parquet 100 | parquet-column 101 | ${parquet.version} 102 | 103 | 104 | org.apache.parquet 105 | parquet-hadoop 106 | ${parquet.version} 107 | 108 | 109 | org.apache.parquet 110 | parquet-avro 111 | ${parquet.version} 112 | 113 | 114 | org.apache.avro 115 | avro-mapred 116 | ${avro.version} 117 | 118 | 119 | 120 | 121 | 122 | org.mongodb.mongo-hadoop 123 | mongo-hadoop-core 124 | ${mongodb.version} 125 | 126 | 127 | -------------------------------------------------------------------------------- /batch/hadoop/src/main/java/com/mitosis/Application.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import java.text.MessageFormat; 4 | 5 | import com.mongodb.hadoop.MongoOutputFormat; 6 | import com.mongodb.hadoop.io.BSONWritable; 7 | import com.typesafe.config.Config; 8 | import com.typesafe.config.ConfigFactory; 9 | import org.apache.avro.Schema; 10 | 11 | import org.apache.avro.mapred.AvroValue; 12 | import org.apache.avro.mapreduce.AvroJob; 13 | import org.apache.hadoop.conf.Configuration; 14 | import org.apache.hadoop.conf.Configured; 15 | import org.apache.hadoop.fs.Path; 16 | import org.apache.hadoop.io.NullWritable; 17 | import org.apache.hadoop.io.Text; 18 | import org.apache.hadoop.mapred.JobConf; 19 | import org.apache.hadoop.mapreduce.Job; 20 | import org.apache.hadoop.util.Tool; 21 | import org.apache.hadoop.util.ToolRunner; 22 | import org.apache.parquet.avro.AvroParquetInputFormat; 23 | 24 | public class Application extends Configured implements Tool { 25 | 26 | public static void main(String[] args) throws Exception { 27 | 28 | 29 | Config config = ConfigFactory.parseResources("app.conf"); 30 | String mongodbUri = MessageFormat.format("mongodb://{0}:{1}/{2}.{3}", 31 | config.getString("batch.db.host"), 32 | config.getString("batch.db.port"), 33 | config.getString("batch.db.database"), 34 | config.getString("batch.db.collection")); 35 | Configuration conf = new Configuration(); 36 | conf.set("mongo.output.uri", mongodbUri); 37 | 38 | int res = ToolRunner.run(conf, new Application(), args); 39 | System.exit(res); 40 | } 41 | 42 | /** 43 | * @param args the arguments, which consist of the input file or directory and the output directory 44 | * @return the job return status, 0 for success, 1 for failure 45 | * @throws Exception 46 | */ 47 | @Override 48 | public int run(String[] args) throws Exception { 49 | JobConf conf = new JobConf(getConf()); 50 | Job job = Job.getInstance(conf); 51 | job.setJarByClass(Application.class); 52 | 53 | Schema avroSchema = Schema.parse(ParquetMapper.class.getResourceAsStream("/airport.schema.avsc")); 54 | 55 | // Mapper 56 | job.setMapperClass(ParquetMapper.class); 57 | // Input 58 | job.setInputFormatClass(AvroParquetInputFormat.class); 59 | AvroParquetInputFormat.addInputPath(job, new Path("parquet/airports.parquet")); 60 | AvroParquetInputFormat.setAvroReadSchema(job, avroSchema); 61 | 62 | // Intermediate Output 63 | job.setMapOutputKeyClass(Text.class); 64 | job.setMapOutputValueClass(AvroValue.class); 65 | AvroJob.setMapOutputValueSchema(job, avroSchema); 66 | 67 | // Reducer 68 | job.setReducerClass(MongoReducer.class); 69 | // Output 70 | job.setOutputFormatClass(MongoOutputFormat.class); 71 | job.setOutputKeyClass(NullWritable.class); 72 | job.setOutputValueClass(BSONWritable.class); 73 | 74 | // Execute job and return status 75 | return job.waitForCompletion(true) ? 0 : 1; 76 | } 77 | 78 | } -------------------------------------------------------------------------------- /batch/hadoop/src/main/java/com/mitosis/MongoReducer.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import com.mongodb.hadoop.io.BSONWritable; 4 | import org.apache.avro.generic.GenericRecord; 5 | import org.apache.avro.mapred.AvroValue; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Reducer; 9 | import org.bson.BSONObject; 10 | import org.bson.BasicBSONObject; 11 | 12 | import java.io.IOException; 13 | 14 | public class MongoReducer extends Reducer, NullWritable, BSONWritable> { 15 | 16 | private final BSONWritable bsonWritable; 17 | 18 | public MongoReducer() { 19 | super(); 20 | bsonWritable = new BSONWritable(); 21 | } 22 | 23 | @Override 24 | protected void reduce(Text key, Iterable> values, Context context) throws IOException, InterruptedException { 25 | for (AvroValue record : values) { 26 | BSONObject result = new BasicBSONObject(); 27 | result.put("AirportID", record.datum().get("AirportID")); 28 | result.put("City", record.datum().get("City")); 29 | result.put("Country", record.datum().get("Country")); 30 | result.put("DST", record.datum().get("DST")); 31 | result.put("DBTZ", record.datum().get("DBTZ")); 32 | result.put("IATAFAA", record.datum().get("IATAFAA")); 33 | result.put("ICAO", record.datum().get("ICAO")); 34 | result.put("Latitude", record.datum().get("Latitude")); 35 | result.put("Longitude", record.datum().get("Longitude")); 36 | result.put("Name", record.datum().get("Name")); 37 | result.put("Timezone", record.datum().get("Timezone")); 38 | result.put("destinations", record.datum().get("destinations")); 39 | bsonWritable.setDoc(result); 40 | context.write(null, bsonWritable); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /batch/hadoop/src/main/java/com/mitosis/ParquetMapper.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import org.apache.avro.generic.GenericData; 4 | import org.apache.avro.generic.GenericRecord; 5 | import org.apache.avro.mapred.AvroValue; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Mapper; 9 | 10 | import org.json.JSONArray; 11 | import org.json.JSONObject; 12 | 13 | import java.io.IOException; 14 | 15 | public class ParquetMapper extends Mapper> { 16 | 17 | // Reuse output objects. 18 | private final Text outputKey = new Text(); 19 | private final AvroValue outputValue = new AvroValue<>(); 20 | /** 21 | * Just creates a simple key using two fields and passes the rest of the value directly through. 22 | * Do some more processing here. 23 | * 24 | * @param key the mapper's key -- not used 25 | * @param value the Avro representation of this Parquet record 26 | * @param context the mapper's context 27 | */ 28 | @Override 29 | protected void map(LongWritable key, GenericRecord value, Context context) throws IOException, InterruptedException { 30 | outputKey.set(value.get("AirportID").toString()); 31 | JSONObject jsonObject = new JSONObject(value.toString()); 32 | 33 | GenericRecord datum = new GenericData.Record(value.getSchema()); 34 | for (String keyJson : jsonObject.keySet()) { 35 | if("destinations".equals(keyJson)) { 36 | JSONArray jsonArray = (JSONArray) jsonObject.get(keyJson); 37 | datum.put(keyJson, jsonArray.toList()); 38 | } else { 39 | datum.put(keyJson, jsonObject.get(keyJson)); 40 | } 41 | } 42 | outputValue.datum(datum); 43 | context.write(outputKey, outputValue); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /batch/hadoop/src/main/resources/airport.schema.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "airport", 4 | "fields": [ 5 | { "name": "AirportID", "type": "string" }, 6 | { "name": "City", "type": "string" }, 7 | { "name": "Country", "type": "string" }, 8 | { "name": "DST", "type": "string" }, 9 | { "name": "DBTZ", "type": "string" }, 10 | { "name": "IATAFAA", "type": "string" }, 11 | { "name": "ICAO", "type": "string" }, 12 | { "name": "Latitude", "type": "string" }, 13 | { "name": "Longitude", "type": "string" }, 14 | { "name": "Name", "type": "string" }, 15 | { "name": "Timezone", "type": "string" }, 16 | { "name": "destinations", "type": {"type": "array", "items": "string"}, "default": []} 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /batch/hadoop/src/main/resources/app.conf: -------------------------------------------------------------------------------- 1 | batch { 2 | db { 3 | host: "mongo" 4 | port: 27017 5 | database: "mitosis" 6 | collection: "airports" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /batch/hadoop/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=DEBUG, CA 2 | 3 | log4j.appender.CA=org.apache.log4j.ConsoleAppender 4 | 5 | log4j.appender.CA.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -------------------------------------------------------------------------------- /batch/spark/Dockerfile: -------------------------------------------------------------------------------- 1 | # FROM uhopper/hadoop:2.7.2 2 | FROM hseeberger/scala-sbt:8u151-2.12.4-1.0.4 3 | VOLUME /tmp 4 | 5 | ENV SPARK_VERSION spark-2.4.0-bin-hadoop2.7 6 | ENV SPARK_HOME /usr/local/spark 7 | 8 | RUN curl https://archive.apache.org/dist/spark/spark-2.4.0/$SPARK_VERSION.tgz -o $SPARK_VERSION.tgz; \ 9 | tar xzf $SPARK_VERSION.tgz -C /usr/local/; 10 | RUN cd /usr/local && ln -s $SPARK_VERSION spark 11 | 12 | ENV PROJECT_HOME /usr/local/project 13 | 14 | ADD src/main/scala $PROJECT_HOME/src/main/scala 15 | ADD src/main/resources $PROJECT_HOME/src/main/resources 16 | ADD parquet $PROJECT_HOME/parquet 17 | ADD build.sbt $PROJECT_HOME/ 18 | ADD project/build.properties $PROJECT_HOME/project/ 19 | ADD project/plugins.sbt $PROJECT_HOME/project/ 20 | 21 | WORKDIR $PROJECT_HOME 22 | 23 | RUN sbt package assembly 24 | 25 | WORKDIR $SPARK_HOME 26 | 27 | CMD ["bin/spark-submit", "--class", "com.mitosis.Main", "--master", "local[2]", "/usr/local/project/target/scala-2.11/search-flight-spark-batch-assembly-0.1.0.jar", "/usr/local/project/parquet" ] 28 | -------------------------------------------------------------------------------- /batch/spark/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre 2 | VOLUME /tmp 3 | 4 | ENV SPARK_VERSION spark-2.4.0-bin-hadoop2.7 5 | ENV SPARK_HOME /usr/local/spark 6 | ENV PARQUET_HOME $SPARK_HOME/parquet 7 | 8 | RUN curl https://archive.apache.org/dist/spark/spark-2.4.0/$SPARK_VERSION.tgz -o $SPARK_VERSION.tgz; \ 9 | tar xzf $SPARK_VERSION.tgz -C /usr/local/; 10 | 11 | RUN cd /usr/local && ln -s $SPARK_VERSION spark 12 | 13 | ADD target/scala-2.11/search-flight-spark-batch-assembly-0.1.0.jar $SPARK_HOME/libs/search-flight-batch.jar 14 | ADD parquet/ $PARQUET_HOME 15 | RUN sh -c 'touch $SPARK_HOME/libs/search-flight-batch.jar' 16 | 17 | WORKDIR $SPARK_HOME 18 | CMD ["bin/spark-submit", "--class", "com.mitosis.Main", "--master", "local[2]", "./libs/search-flight-batch.jar", "./parquet" ] 19 | -------------------------------------------------------------------------------- /batch/spark/build.sbt: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | name := "search-flight-spark-batch" 5 | organization := "com.mitosis" 6 | version := "0.1.0" 7 | 8 | scalaVersion := "2.11.11" 9 | 10 | val sparkVersion = "2.4.0" 11 | val mongodbVersion = "2.2.1" 12 | val typesafeVersion = "1.3.0" 13 | val log4jVersion = "1.2.14" 14 | 15 | libraryDependencies ++= Seq( 16 | "log4j" % "log4j" % log4jVersion, 17 | 18 | "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", 19 | ("org.apache.spark" %% "spark-core" % sparkVersion % "provided"). 20 | exclude("org.apache.spark", "spark-network-common_2.12"). 21 | exclude("org.apache.spark", "spark-network-shuffle_2.12"), 22 | 23 | // avoid an ivy bug 24 | "org.apache.spark" %% "spark-network-common" % sparkVersion % "provided", 25 | "org.apache.spark" %% "spark-network-shuffle" % sparkVersion % "provided", 26 | 27 | "com.typesafe" % "config" % typesafeVersion, 28 | "org.mongodb.spark" %% "mongo-spark-connector" % mongodbVersion 29 | 30 | ) 31 | 32 | unmanagedSourceDirectories in Compile := (scalaSource in Compile).value :: Nil 33 | 34 | assemblyMergeStrategy in assembly := { 35 | case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") => MergeStrategy.first 36 | case x => (assemblyMergeStrategy in assembly).value(x) 37 | } -------------------------------------------------------------------------------- /batch/spark/parquet/airports.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Chabane/bigdata-playground/6838b5e33fefe81b7e1514c057dad5ebc48f4132/batch/spark/parquet/airports.parquet -------------------------------------------------------------------------------- /batch/spark/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.mitosis 7 | search-flight-spark-batch 8 | 0.1.0 9 | 10 | 11 | 2.11.11 12 | 13 | 2.4.0 14 | 2.2.1 15 | 1.3.0 16 | 1.2.14 17 | 1.16.20 18 | 19 | 20 | 21 | 22 | 23 | org.apache.maven.plugins 24 | maven-compiler-plugin 25 | 26 | 1.8 27 | 1.8 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | org.apache.spark 36 | spark-core_2.12 37 | ${spark.version} 38 | 39 | 40 | 41 | org.apache.spark 42 | spark-sql_2.12 43 | ${spark.version} 44 | 45 | 46 | 47 | org.mongodb.spark 48 | mongo-spark-connector_2.11 49 | ${mongodb.version} 50 | 51 | 52 | 53 | com.typesafe 54 | config 55 | ${typesafe.version} 56 | 57 | 58 | 59 | log4j 60 | log4j 61 | ${log4j.version} 62 | 63 | 64 | 65 | org.projectlombok 66 | lombok 67 | ${lombok.version} 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /batch/spark/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.2 2 | -------------------------------------------------------------------------------- /batch/spark/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // https://github.com/sbt/sbt/wiki/sbt-1.x-plugin-migration 2 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 3 | // addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.1") 4 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.2-RC2") -------------------------------------------------------------------------------- /batch/spark/src/main/java/com/mitosis/Application.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import java.text.MessageFormat; 4 | 5 | import org.apache.spark.sql.Dataset; 6 | import org.apache.spark.sql.Row; 7 | import org.apache.spark.sql.SparkSession; 8 | 9 | import com.mongodb.spark.MongoSpark; 10 | import com.typesafe.config.Config; 11 | import com.typesafe.config.ConfigFactory; 12 | 13 | public class Application { 14 | 15 | public static void main(String[] args) { 16 | 17 | Config config = ConfigFactory.parseResources("app.conf"); 18 | String mongodbUri = MessageFormat.format("mongodb://{0}:{1}/{2}.{3}", 19 | config.getString("batch.db.host"), 20 | config.getString("batch.db.port"), 21 | config.getString("batch.db.database"), 22 | config.getString("batch.db.collection")); 23 | 24 | SparkSession sparkSession = SparkSession 25 | .builder() 26 | .master("local[*]") 27 | .appName("search-flight-batch-spark") 28 | .config("spark.mongodb.output.uri", mongodbUri) 29 | .getOrCreate(); 30 | 31 | // read parquet 32 | Dataset airportsDF = sparkSession.read().parquet(args[0]); 33 | // save airports 34 | MongoSpark.save(airportsDF.write().option("collection", "airports").mode("overwrite")); 35 | 36 | sparkSession.stop(); 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /batch/spark/src/main/resources/app.conf: -------------------------------------------------------------------------------- 1 | batch { 2 | db { 3 | host: "mongo" 4 | port: 27017 5 | database: "mitosis" 6 | collection: "airports" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /batch/spark/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.logger.org.apache.spark.storage.BlockManager=OFF 2 | 3 | log4j.rootLogger=DEBUG, CA 4 | 5 | log4j.appender.CA=org.apache.log4j.ConsoleAppender 6 | 7 | log4j.appender.CA.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -------------------------------------------------------------------------------- /batch/spark/src/main/scala/com/mitosis/Main.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis 2 | 3 | import com.mitosis.config.ConfigurationFactory 4 | import org.apache.log4j.Logger 5 | import org.apache.spark.sql.SparkSession 6 | import com.mongodb.spark._ 7 | 8 | object Main { 9 | 10 | private[this] lazy val logger = Logger.getLogger(getClass) 11 | 12 | private[this] val config: com.mitosis.config.objects.Config = ConfigurationFactory.load() 13 | 14 | def main(args: Array[String]): Unit = { 15 | 16 | val sparkSession = SparkSession.builder 17 | .appName("search-flight-batch-spark") 18 | .config("spark.mongodb.output.uri", "mongodb://%s/%s.%s".format( 19 | config.batch.db.host, 20 | config.batch.db.database, 21 | config.batch.db.collection)) 22 | .getOrCreate() 23 | 24 | val airportDF = sparkSession.read.parquet(args(0)) 25 | MongoSpark.save(airportDF.write.option("collection", "airports").mode("overwrite")) 26 | 27 | sparkSession.stop() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /batch/spark/src/main/scala/com/mitosis/config/ConfigurationFactory.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config 2 | 3 | import com.mitosis.config.objects.Config 4 | import com.typesafe.config.ConfigBeanFactory 5 | import com.typesafe.config.ConfigFactory 6 | 7 | import java.io.File 8 | import java.io.InputStreamReader 9 | 10 | object ConfigurationFactory { 11 | 12 | /** 13 | * Loads configuration object by file 14 | * 15 | * @return Returns configuration objects 16 | */ 17 | def load(): Config = { 18 | val is = new InputStreamReader(getClass.getResourceAsStream(s"/app.conf")) 19 | val config: com.typesafe.config.Config = ConfigFactory.parseReader(is).resolve() 20 | ConfigBeanFactory.create(config, classOf[Config]) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /batch/spark/src/main/scala/com/mitosis/config/objects/BatchConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class BatchConfig { 9 | 10 | @BeanProperty 11 | var db: BatchDbConfig = _ 12 | } 13 | -------------------------------------------------------------------------------- /batch/spark/src/main/scala/com/mitosis/config/objects/BatchDbConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class BatchDbConfig { 9 | 10 | @BeanProperty 11 | var host: String = _ 12 | 13 | @BeanProperty 14 | var port: Int = _ 15 | 16 | @BeanProperty 17 | var database: String = _ 18 | 19 | @BeanProperty 20 | var collection: String = _ 21 | } 22 | -------------------------------------------------------------------------------- /batch/spark/src/main/scala/com/mitosis/config/objects/Config.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class Config { 9 | 10 | @BeanProperty 11 | var batch: BatchConfig = _ 12 | } 13 | -------------------------------------------------------------------------------- /docker/dev/batch-flink.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | batch-flink-dev: 5 | build: 6 | context: ../../batch/flink 7 | dockerfile: Dockerfile-dev 8 | container_name: batch_flink_dev 9 | networks: ["vnet"] 10 | hostname: batch-flink-dev 11 | 12 | networks: 13 | vnet: 14 | external: 15 | name: vnet 16 | -------------------------------------------------------------------------------- /docker/dev/batch-hadoop.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | batch-hadoop-dev: 5 | build: 6 | context: ../../batch/hadoop 7 | dockerfile: Dockerfile-dev 8 | container_name: batch_hadoop_dev 9 | networks: ["vnet"] 10 | hostname: batch-hadoop-dev 11 | 12 | networks: 13 | vnet: 14 | external: 15 | name: vnet 16 | -------------------------------------------------------------------------------- /docker/dev/batch-spark.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | batch-spark-dev: 5 | build: 6 | context: ../../batch/spark 7 | dockerfile: Dockerfile-dev 8 | container_name: batch_spark_dev 9 | networks: ["vnet"] 10 | hostname: batch-spark-dev 11 | 12 | networks: 13 | vnet: 14 | external: 15 | name: vnet 16 | -------------------------------------------------------------------------------- /docker/dev/ml-spark.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | ml-spark-model-dev: 5 | build: 6 | context: ../../ml/spark 7 | dockerfile: Dockerfile-model-dev 8 | container_name: ml-spark-model-dev 9 | networks: ["vnet"] 10 | hostname: ml-spark-model-dev 11 | environment: 12 | - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY} 13 | - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} 14 | - TWITTER_CONSUMER_ACCESS_TOKEN=${TWITTER_CONSUMER_ACCESS_TOKEN} 15 | - TWITTER_CONSUMER_ACCESS_TOKEN_SECRET=${TWITTER_CONSUMER_ACCESS_TOKEN_SECRET} 16 | deploy: 17 | resources: 18 | limits: 19 | cpus: '0.75' 20 | memory: 2G 21 | reservations: 22 | cpus: '0.50' 23 | memory: 1G 24 | volumes: 25 | - model:/usr/local/spark/libs/tweet_traveling_partners_model 26 | 27 | ml-spark-stream-dev: 28 | build: 29 | context: ../../ml/spark 30 | dockerfile: Dockerfile-stream-dev 31 | container_name: ml-spark-stream-dev 32 | networks: ["vnet"] 33 | hostname: ml-spark-stream-dev 34 | environment: 35 | - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY} 36 | - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} 37 | - TWITTER_CONSUMER_ACCESS_TOKEN=${TWITTER_CONSUMER_ACCESS_TOKEN} 38 | - TWITTER_CONSUMER_ACCESS_TOKEN_SECRET=${TWITTER_CONSUMER_ACCESS_TOKEN_SECRET} 39 | deploy: 40 | resources: 41 | limits: 42 | cpus: '0.75' 43 | memory: 2G 44 | reservations: 45 | cpus: '0.50' 46 | memory: 1G 47 | volumes: 48 | - model:/usr/local/spark/libs/tweet_traveling_partners_model 49 | 50 | networks: 51 | vnet: 52 | external: 53 | name: vnet 54 | volumes: 55 | model: -------------------------------------------------------------------------------- /docker/dev/streaming-flink.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | streaming-flink-dev: 5 | build: 6 | context: ../../streaming/flink 7 | dockerfile: Dockerfile-dev 8 | container_name: streaming_flink_dev 9 | networks: ["vnet"] 10 | hostname: streaming-flink-dev 11 | 12 | networks: 13 | vnet: 14 | external: 15 | name: vnet 16 | -------------------------------------------------------------------------------- /docker/dev/streaming-spark.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | streaming-spark-dev: 5 | build: 6 | context: ../../streaming/spark 7 | dockerfile: Dockerfile-dev 8 | container_name: streaming_spark_dev 9 | networks: ["vnet"] 10 | hostname: streaming-spark-dev 11 | 12 | networks: 13 | vnet: 14 | external: 15 | name: vnet 16 | -------------------------------------------------------------------------------- /docker/dev/streaming-storm.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | streaming-storm-dev: 5 | build: 6 | context: ../../streaming/storm 7 | dockerfile: Dockerfile-dev 8 | container_name: streaming_storm_dev 9 | networks: ["vnet"] 10 | hostname: streaming-storm-dev 11 | 12 | networks: 13 | vnet: 14 | external: 15 | name: vnet 16 | -------------------------------------------------------------------------------- /docker/dev/webapp.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | webapp-dev: 5 | build: 6 | context: ../../webapp 7 | dockerfile: Dockerfile-dev 8 | container_name: webapp-dev 9 | networks: ["vnet"] 10 | hostname: webapp-dev 11 | ports: 12 | - 4000:4000 13 | - 5000:5000 14 | 15 | networks: 16 | vnet: 17 | external: 18 | name: vnet 19 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | webapp: 5 | build: ../webapp 6 | container_name: webapp 7 | ports: 8 | - 3000:3000 9 | - 5000:5000 10 | networks: ["vnet"] 11 | hostname: webapp 12 | 13 | batch-spark: 14 | build: ../batch/spark 15 | container_name: batch_spark 16 | networks: ["vnet"] 17 | hostname: batch-spark 18 | 19 | streaming-spark: 20 | build: ../streaming/spark 21 | container_name: streaming_spark 22 | networks: ["vnet"] 23 | hostname: streaming-spark 24 | 25 | networks: 26 | vnet: 27 | external: 28 | name: vnet 29 | -------------------------------------------------------------------------------- /docker/flink.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | jobmanager: 5 | image: flink:1.4.1-hadoop28-alpine 6 | expose: 7 | - "6123" 8 | networks: ["vnet"] 9 | hostname: streaming-flink-jobmanager 10 | ports: 11 | - "8081:8081" 12 | command: jobmanager 13 | environment: 14 | - JOB_MANAGER_RPC_ADDRESS=jobmanager 15 | 16 | taskmanager: 17 | image: flink:1.4.1-hadoop28-alpine 18 | expose: 19 | - "6121" 20 | - "6122" 21 | networks: ["vnet"] 22 | hostname: streaming-flink-taskmanager 23 | depends_on: 24 | - jobmanager 25 | command: taskmanager 26 | links: 27 | - "jobmanager:jobmanager" 28 | environment: 29 | - JOB_MANAGER_RPC_ADDRESS=jobmanager 30 | 31 | networks: 32 | vnet: 33 | external: 34 | name: vnet 35 | -------------------------------------------------------------------------------- /docker/hadoop-hbase.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | services: 3 | 4 | namenode-1: 5 | container_name: namenode-1 6 | networks: ["vnet"] 7 | hostname: namenode-1 8 | image: smizy/hadoop-base:2.8.3-alpine 9 | expose: ["8020"] 10 | ports: ["50070"] 11 | environment: 12 | - SERVICE_8020_NAME=namenode 13 | - SERVICE_50070_IGNORE=true 14 | - HADOOP_ZOOKEEPER_QUORUM=zoo1:2181 15 | - HADOOP_HEAPSIZE=1000 16 | - HADOOP_NAMENODE_HA= 17 | volumes: 18 | - namenode:/usr/local/hadoop-2.8.3 19 | entrypoint: entrypoint.sh 20 | command: namenode-1 21 | 22 | datanode-1: 23 | container_name: datanode-1 24 | networks: ["vnet"] 25 | hostname: datanode-1 26 | image: smizy/hadoop-base:2.8.3-alpine 27 | expose: ["50010", "50020", "50075"] 28 | environment: 29 | - SERVICE_50010_NAME=datanode 30 | - SERVICE_50020_IGNORE=true 31 | - SERVICE_50075_IGNORE=true 32 | - HADOOP_ZOOKEEPER_QUORUM=zoo1:2181 33 | - HADOOP_HEAPSIZE=1000 34 | - HADOOP_NAMENODE_HA= 35 | 36 | entrypoint: entrypoint.sh 37 | command: datanode 38 | 39 | hmaster-1: 40 | container_name: hmaster-1 41 | networks: ["vnet"] 42 | hostname: hmaster-1 43 | image: smizy/hbase:1.4.0-alpine 44 | expose: [16000] 45 | ports: [16010] 46 | environment: 47 | - SERVICE_16000_NAME=hmaster 48 | - SERVICE_16010_IGNORE=true 49 | - HBASE_ZOOKEEPER_QUORUM=zoo1:2181 50 | command: hmaster-1 51 | volumes: 52 | - namenode:/usr/local/hadoop-2.8.3 53 | 54 | regionserver-1: 55 | container_name: regionserver-1 56 | networks: ["vnet"] 57 | hostname: regionserver-1 58 | image: smizy/hbase:1.4.0-alpine 59 | expose: [16020, 16030] 60 | environment: 61 | - SERVICE_16020_NAME=regionserver 62 | - SERVICE_16030_IGNORE=true 63 | - HBASE_ZOOKEEPER_QUORUM=zoo1:2181 64 | command: regionserver 65 | 66 | networks: 67 | vnet: 68 | external: 69 | name: vnet 70 | 71 | volumes: 72 | namenode: -------------------------------------------------------------------------------- /docker/kafka.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | 3 | services: 4 | kafka: 5 | image: wurstmeister/kafka:2.11-2.0.1 6 | hostname: kafka 7 | networks: ["vnet"] 8 | container_name: kafka 9 | ports: 10 | - "9092:9092" 11 | environment: 12 | KAFKA_ADVERTISED_HOST_NAME: kafka 13 | KAFKA_ADVERTISED_PORT: 9092 14 | KAFKA_CREATE_TOPICS: flightInfoTopic:1:1,tweetsTopic:1:1 15 | KAFKA_ZOOKEEPER_CONNECT: zoo1:2181 16 | KAFKA_LOG4J_ROOT_LOGLEVEL: WARN 17 | KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR 18 | volumes: 19 | - /var/run/docker.sock:/var/run/docker.sock 20 | 21 | networks: 22 | vnet: 23 | external: 24 | name: vnet -------------------------------------------------------------------------------- /docker/kylo.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | kylo: 5 | networks: ["vnet"] 6 | hostname: kylo 7 | image: keven4ever/kylo_docker 8 | 9 | networks: 10 | vnet: 11 | external: 12 | name: vnet 13 | -------------------------------------------------------------------------------- /docker/mongo.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | mongo: 5 | image: mongo:3.4 6 | networks: ["vnet"] 7 | hostname: mongo 8 | ports: 9 | - "27017:27017" 10 | 11 | networks: 12 | vnet: 13 | external: 14 | name: vnet 15 | -------------------------------------------------------------------------------- /docker/webapp.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | webapp: 5 | build: ../webapp 6 | container_name: webapp 7 | networks: ["vnet"] 8 | hostname: webapp 9 | ports: 10 | - 3000:3000 11 | - 5000:5000 12 | 13 | networks: 14 | vnet: 15 | external: 16 | name: vnet 17 | -------------------------------------------------------------------------------- /docker/zookeeper.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | 3 | services: 4 | zoo1: 5 | image: zookeeper:3.5 6 | restart: always 7 | networks: ["vnet"] 8 | hostname: zoo1 9 | ports: 10 | - 2181:2181 11 | environment: 12 | ZOO_MY_ID: 1 13 | ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 14 | 15 | zoo2: 16 | image: zookeeper:3.5 17 | restart: always 18 | networks: ["vnet"] 19 | hostname: zoo2 20 | ports: 21 | - 2182:2181 22 | environment: 23 | ZOO_MY_ID: 2 24 | ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zoo3:2888:3888 25 | 26 | zoo3: 27 | image: zookeeper:3.5 28 | restart: always 29 | networks: ["vnet"] 30 | hostname: zoo3 31 | ports: 32 | - 2183:2181 33 | environment: 34 | ZOO_MY_ID: 3 35 | ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=0.0.0.0:2888:3888 36 | 37 | networks: 38 | vnet: 39 | external: 40 | name: vnet -------------------------------------------------------------------------------- /ml/spark/Dockerfile-model-dev: -------------------------------------------------------------------------------- 1 | FROM python:3.6.4-alpine 2 | 3 | RUN apk add --no-cache curl bash alpine-sdk openjdk8 4 | RUN pip install tweepy pyspark numpy 5 | 6 | ENV JAVA8_HOME /usr/lib/jvm/default-jvm 7 | ENV JAVA_HOME $JAVA8_HOME 8 | 9 | ENV SPARK_VERSION spark-2.4.0-bin-hadoop2.7 10 | ENV SPARK_HOME /usr/local/spark 11 | 12 | RUN curl https://archive.apache.org/dist/spark/spark-2.4.0/$SPARK_VERSION.tgz -o $SPARK_VERSION.tgz; \ 13 | tar xzf $SPARK_VERSION.tgz -C /usr/local/; 14 | RUN cd /usr/local && ln -s $SPARK_VERSION spark 15 | 16 | ADD model.py $SPARK_HOME/libs/mlApp.py 17 | 18 | WORKDIR $SPARK_HOME 19 | RUN python3 -m py_compile libs/mlApp.py 20 | CMD ["bin/spark-submit", "--master", "local[*]", "libs/mlApp.py"] -------------------------------------------------------------------------------- /ml/spark/Dockerfile-stream-dev: -------------------------------------------------------------------------------- 1 | FROM python:3.6.4-alpine 2 | 3 | RUN apk add --no-cache curl bash alpine-sdk openjdk8 4 | RUN pip install tweepy pyspark avro-python3 5 | 6 | ENV JAVA8_HOME /usr/lib/jvm/default-jvm 7 | ENV JAVA_HOME $JAVA8_HOME 8 | 9 | ENV SPARK_VERSION spark-2.4.0-bin-hadoop2.7 10 | ENV SPARK_HOME /usr/local/spark 11 | 12 | RUN curl https://archive.apache.org/dist/spark/spark-2.4.0/$SPARK_VERSION.tgz -o $SPARK_VERSION.tgz; \ 13 | tar xzf $SPARK_VERSION.tgz -C /usr/local/; 14 | RUN cd /usr/local && ln -s $SPARK_VERSION spark 15 | 16 | ADD stream.py $SPARK_HOME/libs/mlApp.py 17 | ADD flight-info.schema.avsc $SPARK_HOME/libs/flight-info.schema.avsc 18 | 19 | WORKDIR $SPARK_HOME 20 | RUN python3 -m py_compile libs/mlApp.py 21 | CMD ["bin/spark-submit", "--master", "local[*]", "--packages", "org.apache.spark:spark-sql-kafka-0-10_2.12:2.4.0", "libs/mlApp.py"] 22 | -------------------------------------------------------------------------------- /ml/spark/flight-info.schema.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "flightInfo", 4 | "fields": [ 5 | { "name": "departingId", "type": "string" }, 6 | { "name": "arrivingId", "type": "string" }, 7 | { "name": "tripType", "type": { "type": "enum", "name": "TripType", "symbols": ["ONE_WAY", "ROUND_TRIP"] } }, 8 | { "name": "departureDate", "type": "string" }, 9 | { "name": "arrivalDate", "type": "string" }, 10 | { "name": "passengerNumber", "type": "int" }, 11 | { "name": "cabinClass", "type": { "type": "enum", "name": "CabinClass", "symbols": ["ECONOMY", "PRENIUM", "BUSINESS"] } } 12 | ] 13 | } -------------------------------------------------------------------------------- /ml/spark/model.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tweepy 4 | import os 5 | 6 | from pyspark.sql import * 7 | from pyspark.sql.types import * 8 | from pyspark.sql.functions import col 9 | 10 | from pyspark.ml import Pipeline, PipelineModel 11 | from pyspark.ml.classification import LogisticRegression 12 | from pyspark.ml.feature import HashingTF, Tokenizer, StringIndexer, NGram, IDF 13 | 14 | MAX_TWEETS = 50 15 | 16 | ACCESS_TOKEN = os.environ['TWITTER_CONSUMER_ACCESS_TOKEN'] 17 | ACCESS_SECRET = os.environ['TWITTER_CONSUMER_ACCESS_TOKEN_SECRET'] 18 | CONSUMER_KEY = os.environ['TWITTER_CONSUMER_KEY'] 19 | CONSUMER_SECRET = os.environ['TWITTER_CONSUMER_SECRET'] 20 | 21 | def initialize(): 22 | 23 | spark = SparkSession \ 24 | .builder \ 25 | .appName("search-flight-spark-ml-model") \ 26 | .getOrCreate() 27 | sc = spark.sparkContext 28 | 29 | auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) 30 | auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET) 31 | api = tweepy.API(auth) 32 | important_fields = ['id', 'text', 'user'] 33 | 34 | schema = StructType([ 35 | StructField('id', LongType(), False), 36 | StructField('text', StringType(), False), 37 | StructField('username', StringType(), False) 38 | ]) 39 | 40 | tweetsDf = spark.createDataFrame(sc.emptyRDD(), schema) 41 | 42 | for tweet in tweepy.Cursor(api.search, q='barajas', rpp=100, lang='en').items(MAX_TWEETS): 43 | json_tweet = {k: tweet._json[k] for k in important_fields} 44 | json_tweet['text'] = json_tweet['text'].replace("'", "").replace("\"", "").replace("\n", "") 45 | tweetDf = spark.createDataFrame([ 46 | (json_tweet['id'], json_tweet['text'], json_tweet['user']['name']) 47 | ], schema) 48 | tweetsDf = tweetsDf.union(tweetDf) 49 | 50 | tweets_df_splitted = tweetsDf.randomSplit([0.75, 0.25], MAX_TWEETS) 51 | training_set = tweets_df_splitted[0] 52 | test_set = tweets_df_splitted[1] 53 | 54 | username_indexed = StringIndexer(inputCol="username", outputCol="username_indexed") 55 | tokenizer = Tokenizer(inputCol="text", outputCol="token_raw") 56 | ngram = NGram(inputCol="token_raw", outputCol="ngram", n=2) 57 | hashing_tf = HashingTF(inputCol="ngram", outputCol="tf", numFeatures=20) 58 | idf = IDF(inputCol="tf", outputCol="idf", minDocFreq=2) 59 | lr = LogisticRegression(featuresCol="idf", labelCol="username_indexed") 60 | pipeline = Pipeline(stages=[username_indexed, tokenizer, ngram, hashing_tf, idf, lr]) 61 | 62 | pipeline_model = pipeline.fit(training_set) 63 | pipeline_model.write().overwrite().save("tweet_traveling_partners_model") 64 | 65 | tweet_traveling_partners_prediction = pipeline_model.transform(test_set) 66 | 67 | selected = tweet_traveling_partners_prediction.select("username", "text", "probability", "prediction") 68 | for row in selected.collect(): 69 | print(row) 70 | 71 | spark.stop() 72 | 73 | if __name__ == "__main__": 74 | initialize() 75 | -------------------------------------------------------------------------------- /ml/spark/stream.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import avro.schema 4 | import tweepy 5 | import os 6 | 7 | from io import BytesIO 8 | from avro.datafile import DataFileReader, DataFileWriter 9 | from avro.io import DatumReader, DatumWriter, BinaryEncoder, BinaryDecoder 10 | from avro.schema import Parse 11 | 12 | from pyspark.sql import * 13 | from pyspark.sql.types import * 14 | from pyspark.sql.functions import col, explode 15 | 16 | import json 17 | 18 | os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages org.apache.spark:spark-sql-kafka-0-10_2.12:2.4.0' 19 | dir_path = os.path.dirname(os.path.realpath(__file__)) 20 | 21 | def deserialize(flight_info_bytes) : 22 | if flight_info_bytes is not None: 23 | bytes_reader = BytesIO(flight_info_bytes) 24 | decoder = BinaryDecoder(bytes_reader) 25 | schema_flight_info = Parse(open(dir_path + "/flight-info.schema.avsc", "rb").read()) 26 | reader = DatumReader(schema_flight_info) 27 | flight_info = reader.read(decoder) 28 | 29 | return json.dumps([{"id": 907955534287978496}]) 30 | else: 31 | return None 32 | 33 | def initialize() : 34 | 35 | spark = SparkSession \ 36 | .builder \ 37 | .appName("search-flight-spark-ml-stream") \ 38 | .getOrCreate() 39 | 40 | search_flight_df = spark \ 41 | .readStream \ 42 | .format("kafka") \ 43 | .option("kafka.bootstrap.servers", "kafka:9092") \ 44 | .option("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") \ 45 | .option("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer") \ 46 | .option("subscribe", "flightInfoTopic") \ 47 | .option("auto.offset.reset", "latest") \ 48 | .option("group.id", "mitosis") \ 49 | .option("enable.auto.commit", False) \ 50 | .load() 51 | 52 | spark.udf.register("deserialize", deserialize) 53 | 54 | search_flight_ds = search_flight_df\ 55 | .selectExpr("key", "deserialize(value) as value") \ 56 | .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)") 57 | 58 | search_flight_ds \ 59 | .writeStream \ 60 | .format("kafka") \ 61 | .option("kafka.bootstrap.servers", "kafka:9092") \ 62 | .option("topic", "tweetsTopic") \ 63 | .option("group.id", "mitosis") \ 64 | .option("checkpointLocation", "/tmp/checkpoint") \ 65 | .option("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") \ 66 | .option("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") \ 67 | .start() \ 68 | .awaitTermination() 69 | 70 | spark.stop() 71 | 72 | if __name__ == "__main__": 73 | initialize() 74 | -------------------------------------------------------------------------------- /streaming/flink/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM flink:1.6.2-hadoop28-alpine 2 | VOLUME /tmp 3 | 4 | ADD target/scala-2.11/search-flight-flink-streaming-assembly-0.1.0.jar $FLINK_HOME/libs/search-flight-streaming.jar 5 | RUN sh -c 'touch $FLINK_HOME/libs/search-flight-streaming.jar' 6 | 7 | WORKDIR $FLINK_HOME 8 | 9 | CMD ["./bin/flink", "run", "-m", "jobmanager:6123", "-c", "com.mitosis.Main", "./libs/search-flight-streaming.jar", "--port", "9000"] 10 | -------------------------------------------------------------------------------- /streaming/flink/build.sbt: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | resolvers in ThisBuild ++= Seq("Apache Development Snapshot Repository" at "https://repository.apache.org/content/repositories/snapshots/", 5 | Resolver.mavenLocal) 6 | 7 | name := "search-flight-flink-streaming" 8 | organization := "com.mitosis" 9 | version := "0.1.0" 10 | 11 | mainClass in assembly := Some("com.mitosis.Main") 12 | 13 | scalaVersion := "2.11.11" 14 | 15 | val flinkVersion = "1.4.1" 16 | val jacksonVersion = "2.9.8" 17 | val typesafeVersion = "1.3.0" 18 | val log4jVersion = "1.2.14" 19 | val hbaseVersion = "1.3.1" 20 | val hadoopVersion = "2.5.1" 21 | 22 | val projectDependencies = Seq( 23 | "log4j" % "log4j" % log4jVersion, 24 | "org.apache.flink" %% "flink-streaming-scala" % flinkVersion % "provided", 25 | "org.apache.flink" %% "flink-hbase" % flinkVersion, 26 | "org.apache.flink" % "flink-avro" % flinkVersion, 27 | "org.apache.flink" %% "flink-connector-kafka-0.11" % flinkVersion, 28 | "org.apache.hbase" % "hbase-common" % hbaseVersion, 29 | ("org.apache.hadoop" % "hadoop-common" % hadoopVersion) 30 | .exclude("commons-beanutils", "commons-beanutils-core") 31 | .exclude("commons-beanutils", "commons-beanutils"), 32 | 33 | "com.typesafe" % "config" % typesafeVersion, 34 | "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, 35 | "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion 36 | ) 37 | 38 | 39 | lazy val root = (project in file(".")). 40 | settings( 41 | libraryDependencies ++= projectDependencies 42 | ) 43 | 44 | // make run command include the provided dependencies 45 | run in Compile := Defaults.runTask(fullClasspath in Compile, 46 | mainClass in (Compile, run), 47 | runner in (Compile,run) 48 | ).evaluated 49 | 50 | // exclude Scala library from assembly 51 | assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) 52 | -------------------------------------------------------------------------------- /streaming/flink/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.2 2 | -------------------------------------------------------------------------------- /streaming/flink/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // https://github.com/sbt/sbt/wiki/sbt-1.x-plugin-migration 2 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 3 | // addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.1") 4 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.2-RC2") -------------------------------------------------------------------------------- /streaming/flink/src/main/resources/app.conf: -------------------------------------------------------------------------------- 1 | producer { 2 | batchSize: 16384 3 | topic: "flightInfoTopic" 4 | 5 | hosts: [ 6 | "kafka:9092" 7 | ] 8 | } 9 | 10 | streaming { 11 | db { 12 | host: "regionserver-1" 13 | port: "16000" 14 | table: "flightInfo" 15 | columnFamily: "flightInfoCF" 16 | } 17 | 18 | window: 1 19 | } 20 | -------------------------------------------------------------------------------- /streaming/flink/src/main/resources/flight-info.schema.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "flightInfo", 4 | "fields": [ 5 | { "name": "departingId", "type": "string" }, 6 | { "name": "arrivingId", "type": "string" }, 7 | { "name": "tripType", "type": { "type": "enum", "name": "TripType", "symbols": ["ONE_WAY", "ROUND_TRIP"] } }, 8 | { "name": "departureDate", "type": "string" }, 9 | { "name": "arrivalDate", "type": "string" }, 10 | { "name": "passengerNumber", "type": "int" }, 11 | { "name": "cabinClass", "type": { "type": "enum", "name": "CabinClass", "symbols": ["ECONOMY", "PRENIUM", "BUSINESS"] } } 12 | ] 13 | } -------------------------------------------------------------------------------- /streaming/flink/src/main/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.zookeeper.quorum 5 | zoo1 6 | 7 | 8 | hbase.rootdir 9 | hdfs://namenode-1:8020/hbase 10 | 11 | 12 | hbase.master.port 13 | 16020 14 | 15 | 16 | zookeeper.znode.parent 17 | /hbase 18 | 19 | 20 | hbase.zookeeper.property.clientPort 21 | 2181 22 | 23 | 24 | hbase.cluster.distributed 25 | true 26 | 27 | 28 | hbase.rest.ssl.enabled 29 | false 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /streaming/flink/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=DEBUG, CA 2 | 3 | log4j.appender.CA=org.apache.log4j.ConsoleAppender 4 | 5 | log4j.appender.CA.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/Main.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis 2 | 3 | import java.util.Properties 4 | 5 | import com.mitosis.config.ConfigurationFactory 6 | import com.mitosis.utils.{FlightInfoAvroSchema, FlightInfoHBaseOutputFormat} 7 | import org.apache.flink.api.common.restartstrategy.RestartStrategies 8 | import org.apache.flink.streaming.api.TimeCharacteristic 9 | import org.apache.flink.streaming.api.functions.AscendingTimestampExtractor 10 | import org.apache.flink.streaming.api.scala._ 11 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011 12 | import org.apache.log4j.Logger 13 | 14 | object Main { 15 | 16 | private[this] lazy val logger = Logger.getLogger(getClass) 17 | 18 | private[this] val config: com.mitosis.config.objects.Config = ConfigurationFactory.load() 19 | 20 | def main(args: Array[String]): Unit = { 21 | 22 | val env = StreamExecutionEnvironment.getExecutionEnvironment 23 | env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)) 24 | // create a checkpoint every 5 seconds 25 | env.enableCheckpointing(1000) 26 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) 27 | env.getConfig.setAutoWatermarkInterval(1000) 28 | env.setParallelism(1) 29 | 30 | val servers = config.producer.hosts.toArray.mkString(",") 31 | val topic = config.producer.topic 32 | 33 | val properties = new Properties(); 34 | properties.setProperty("bootstrap.servers", servers) 35 | properties.setProperty("group.id", "mitosis") 36 | 37 | val kafkaConsumer = new FlinkKafkaConsumer011(topic, FlightInfoAvroSchema, properties) 38 | kafkaConsumer.setCommitOffsetsOnCheckpoints(true); 39 | kafkaConsumer.setStartFromLatest() 40 | 41 | val messageStream = env.addSource(kafkaConsumer) 42 | messageStream.writeUsingOutputFormat(new FlightInfoHBaseOutputFormat) 43 | 44 | env.execute("search-flight-flink-streaming") 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/beans/CabinClass.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | object CabinClass extends Enumeration { 4 | val ECONOMY, PRENIUM, BUSINESS = Value 5 | type CabinClass = Value 6 | } 7 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/beans/FlightInfoBean.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | import scala.beans.{ BeanProperty } 4 | 5 | class FlightInfoBean { 6 | 7 | @BeanProperty 8 | var departingId: String = _ 9 | 10 | @BeanProperty 11 | var arrivingId: String = _ 12 | 13 | @BeanProperty 14 | var tripType: String = _ 15 | 16 | @BeanProperty 17 | var departureDate: String = _ 18 | 19 | @BeanProperty 20 | var arrivalDate: String = _ 21 | 22 | @BeanProperty 23 | var passengerNumber: Int = _ 24 | 25 | @BeanProperty 26 | var cabinClass: String = _ 27 | } 28 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/beans/TripType.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | object TripType extends Enumeration { 4 | val ONE_WAY, ROUND_TRIP = Value 5 | type TripType = Value 6 | } 7 | 8 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/config/ConfigurationFactory.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config 2 | 3 | import com.mitosis.config.objects.Config 4 | import com.typesafe.config.ConfigBeanFactory 5 | import com.typesafe.config.ConfigFactory 6 | 7 | import java.io.File 8 | import java.io.InputStreamReader 9 | 10 | object ConfigurationFactory { 11 | 12 | /** 13 | * Loads configuration object by file 14 | * 15 | * @return Returns configuration objects 16 | */ 17 | def load(): Config = { 18 | val is = new InputStreamReader(getClass.getResourceAsStream(s"/app.conf")) 19 | val config: com.typesafe.config.Config = ConfigFactory.parseReader(is).resolve() 20 | ConfigBeanFactory.create(config, classOf[Config]) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/config/objects/Config.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class Config { 9 | 10 | @BeanProperty 11 | var producer: ProducerConfig = _ 12 | 13 | @BeanProperty 14 | var streaming: StreamingConfig = _ 15 | } 16 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/config/objects/ProducerConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import java.util.List 4 | 5 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 6 | 7 | //remove if not needed 8 | import scala.collection.JavaConversions._ 9 | 10 | class ProducerConfig { 11 | 12 | @BeanProperty 13 | var hosts: List[String] = _ 14 | 15 | @BeanProperty 16 | var batchSize: Int = _ 17 | 18 | @BeanProperty 19 | var topic: String = _ 20 | } 21 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/config/objects/StreamingConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class StreamingConfig { 9 | 10 | @BeanProperty 11 | var db: StreamingDbConfig = _ 12 | 13 | @BeanProperty 14 | var window: Int = _ 15 | } 16 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/config/objects/StreamingDbConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class StreamingDbConfig { 9 | 10 | @BeanProperty 11 | var host: String = _ 12 | 13 | @BeanProperty 14 | var port: Int = _ 15 | 16 | @BeanProperty 17 | var table: String = _ 18 | 19 | @BeanProperty 20 | var columnFamily: String = _ 21 | } 22 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/utils/FlightInfoAvroSchema.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.utils 2 | 3 | import scala.io.Source 4 | 5 | import org.apache.avro.Schema 6 | import org.apache.avro.Schema.Parser 7 | 8 | import org.apache.avro.io.DatumReader 9 | import org.apache.avro.io.Decoder 10 | import org.apache.avro.specific.SpecificDatumReader 11 | import org.apache.avro.generic.GenericRecord 12 | import org.apache.avro.io.DecoderFactory 13 | import org.apache.avro.SchemaBuilder 14 | 15 | import org.apache.flink.api.common.typeinfo.TypeInformation 16 | import org.apache.flink.streaming.util.serialization.DeserializationSchema 17 | 18 | object FlightInfoAvroSchema extends DeserializationSchema[String] { 19 | 20 | import org.apache.flink.api.common.typeinfo.TypeInformation 21 | import org.apache.flink.api.java.typeutils.TypeExtractor 22 | 23 | override def isEndOfStream(t: String): Boolean = false 24 | 25 | override def deserialize(message: Array[Byte]): String = { 26 | val flightInfoAvroSchema: Schema = new Parser().parse(Source.fromURL(getClass.getResource("/flight-info.schema.avsc")).mkString) 27 | 28 | val reader: DatumReader[GenericRecord] = new SpecificDatumReader[GenericRecord](flightInfoAvroSchema) 29 | val decoder: Decoder = DecoderFactory.get().binaryDecoder(message, null) 30 | 31 | val flightInfoJson: GenericRecord = reader.read(null, decoder) 32 | flightInfoJson.toString 33 | } 34 | override def getProducedType: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) 35 | } 36 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/utils/FlightInfoHBaseOutputFormat.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.utils 2 | 3 | import org.apache.hadoop.hbase.HBaseConfiguration 4 | 5 | import org.apache.flink.api.common.io.OutputFormat 6 | import org.apache.flink.configuration.Configuration 7 | 8 | import org.apache.hadoop.hbase.{ HColumnDescriptor, HTableDescriptor, HBaseConfiguration } 9 | import org.apache.hadoop.hbase.client.HBaseAdmin 10 | import org.apache.hadoop.hbase.client.HTable 11 | import org.apache.hadoop.hbase.client.Put 12 | import org.apache.hadoop.hbase.client.Get 13 | import org.apache.hadoop.hbase.util.Bytes 14 | import com.mitosis.beans.FlightInfoBean 15 | import java.io.IOException 16 | import org.apache.log4j.Logger 17 | 18 | /** 19 | * This class implements an OutputFormat for HBase. 20 | */ 21 | class FlightInfoHBaseOutputFormat extends OutputFormat[String] { 22 | 23 | private[this] lazy val logger = Logger.getLogger(getClass) 24 | 25 | def jsonDecode(text: String): FlightInfoBean = { 26 | try { 27 | JsonUtils.deserialize(text, classOf[FlightInfoBean]) 28 | } catch { 29 | case e: 30 | IOException => 31 | logger.error(e.getMessage, e) 32 | null 33 | } 34 | } 35 | 36 | private var conf: org.apache.hadoop.conf.Configuration = null 37 | private var flightInfoTable: HTable = null 38 | 39 | override def configure(parameters: Configuration): Unit = { 40 | conf = HBaseConfiguration.create() 41 | conf.addResource("hbase-site.xml") 42 | 43 | val admin = new HBaseAdmin(conf) 44 | 45 | if (!admin.isTableAvailable("flightInfo")) { 46 | val flightInfoTableDesc = new HTableDescriptor(Bytes.toBytes("flightInfo")) 47 | val searchFlightInfoColumnFamilyDesc = new HColumnDescriptor(Bytes.toBytes("searchFlightInfo")) 48 | flightInfoTableDesc.addFamily(searchFlightInfoColumnFamilyDesc) 49 | admin.createTable(flightInfoTableDesc) 50 | } 51 | } 52 | 53 | override def open(taskNumber: Int, numTasks: Int): Unit = { 54 | flightInfoTable = new HTable(conf, "flightInfo") 55 | } 56 | 57 | override def writeRecord(record: String): Unit = { 58 | val flightInfo = jsonDecode(record) 59 | val random = scala.util.Random 60 | val rowKey = random.nextLong 61 | val flightInfoRow = new Put(Bytes.toBytes(rowKey)) 62 | 63 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("departingId"), Bytes.toBytes(flightInfo.departingId)) 64 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("arrivingId"), Bytes.toBytes(flightInfo.arrivingId)) 65 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("tripType"), Bytes.toBytes(flightInfo.tripType)) 66 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("departureDate"), Bytes.toBytes(flightInfo.departureDate)) 67 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("arrivalDate"), Bytes.toBytes(flightInfo.arrivalDate)) 68 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("passengerNumber"), Bytes.toBytes(flightInfo.passengerNumber)) 69 | flightInfoRow.add(Bytes.toBytes("searchFlightInfo"), Bytes.toBytes("cabinClass"), Bytes.toBytes(flightInfo.cabinClass)) 70 | 71 | flightInfoTable.put(flightInfoRow) 72 | } 73 | 74 | override def close(): Unit = { 75 | flightInfoTable.flushCommits() 76 | flightInfoTable.close() 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /streaming/flink/src/main/scala/com/mitosis/utils/JsonUtils.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.utils 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | 5 | object JsonUtils { 6 | 7 | private val OBJECT_MAPPER: ObjectMapper = new ObjectMapper() 8 | 9 | /** 10 | * Returns object as JSON encoded string 11 | * 12 | * @param input the bean instance 13 | * @return Returns single line JSON string 14 | */ 15 | def serialize(input: AnyRef): String = 16 | OBJECT_MAPPER.writeValueAsString(input) 17 | 18 | /** 19 | * Returns JSON encoded string as real object 20 | * 21 | * @param input the encoded JSON string 22 | * @return Returns PoJo 23 | */ 24 | def deserialize[T](input: String, tClass: Class[T]): T = 25 | OBJECT_MAPPER.readValue(input, tClass) 26 | } 27 | -------------------------------------------------------------------------------- /streaming/spark/Dockerfile: -------------------------------------------------------------------------------- 1 | # FROM uhopper/hadoop:2.7.2 2 | FROM hseeberger/scala-sbt:8u151-2.12.4-1.0.4 3 | VOLUME /tmp 4 | 5 | ENV SPARK_VERSION spark-2.4.0-bin-hadoop2.7 6 | ENV SPARK_HOME /usr/local/spark 7 | 8 | RUN curl https://archive.apache.org/dist/spark/spark-2.4.0/$SPARK_VERSION.tgz -o $SPARK_VERSION.tgz; \ 9 | tar xzf $SPARK_VERSION.tgz -C /usr/local/; 10 | RUN cd /usr/local && ln -s $SPARK_VERSION spark 11 | 12 | ADD hbase-site.xml $SPARK_HOME/conf 13 | 14 | ENV PROJECT_HOME /usr/local/project 15 | 16 | ADD src/main/scala $PROJECT_HOME/src/main/scala 17 | ADD src/main/resources $PROJECT_HOME/src/main/resources 18 | ADD build.sbt $PROJECT_HOME/ 19 | ADD project/build.properties $PROJECT_HOME/project/ 20 | ADD project/plugins.sbt $PROJECT_HOME/project/ 21 | 22 | WORKDIR $PROJECT_HOME 23 | 24 | RUN sbt package assembly 25 | 26 | WORKDIR $SPARK_HOME 27 | 28 | CMD ["bin/spark-submit", "--class", "com.mitosis.Main", "--master", "yarn", "--deploy-mode", "cluster", "/usr/local/project/target/scala-2.11/search-flight-streaming-spark-assembly-0.1.0.jar" ] 29 | -------------------------------------------------------------------------------- /streaming/spark/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre 2 | VOLUME /tmp 3 | 4 | ENV SPARK_VERSION spark-2.4.0-bin-hadoop2.7 5 | ENV SPARK_HOME /usr/local/spark 6 | 7 | RUN curl https://archive.apache.org/dist/spark/spark-2.4.0/$SPARK_VERSION.tgz -o $SPARK_VERSION.tgz; \ 8 | tar xzf $SPARK_VERSION.tgz -C /usr/local/; 9 | 10 | RUN cd /usr/local && ln -s $SPARK_VERSION spark 11 | 12 | ADD hbase-site.xml $SPARK_HOME/conf 13 | ADD target/scala-2.11/search-flight-spark-streaming-assembly-0.1.0.jar $SPARK_HOME/libs/search-flight-streaming.jar 14 | RUN sh -c 'touch $SPARK_HOME/libs/search-flight-streaming.jar' 15 | 16 | WORKDIR $SPARK_HOME 17 | 18 | CMD ["bin/spark-submit", "--class", "com.mitosis.Main", "--master", "local[*]", "--files", "/usr/local/spark/conf/hbase-site.xml", "./libs/search-flight-streaming.jar"] 19 | -------------------------------------------------------------------------------- /streaming/spark/build.sbt: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | resolvers ++= Seq( 5 | "apache-snapshots" at "http://repository.apache.org/snapshots/", 6 | "Spark Packages Repo" at "https://dl.bintray.com/spark-packages/maven", 7 | "confluent" at "http://packages.confluent.io/maven/", 8 | "hortonworks" at "http://repo.hortonworks.com/content/repositories/releases/", 9 | Resolver.sonatypeRepo("public") 10 | ) 11 | 12 | name := "search-flight-spark-streaming" 13 | organization := "com.mitosis" 14 | version := "0.1.0" 15 | 16 | scalaVersion := "2.11.11" 17 | 18 | val sparkVersion = "2.4.0" 19 | val jacksonVersion = "2.9.8" 20 | val typesafeVersion = "1.3.0" 21 | val log4jVersion = "1.2.14" 22 | val avroVersion = "4.0.0" 23 | val hbaseConnectorVersion = "1.1.1-2.1-s_2.11" 24 | 25 | libraryDependencies ++= Seq( 26 | "log4j" % "log4j" % log4jVersion, 27 | 28 | "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", 29 | ("org.apache.spark" %% "spark-core" % sparkVersion % "provided"). 30 | exclude("org.apache.spark", "spark-network-common_2.12"). 31 | exclude("org.apache.spark", "spark-network-shuffle_2.12"), 32 | 33 | // avoid an ivy bug 34 | "org.apache.spark" %% "spark-network-common" % sparkVersion % "provided", 35 | "org.apache.spark" %% "spark-network-shuffle" % sparkVersion % "provided", 36 | ("org.apache.spark" %% "spark-streaming" % sparkVersion % "provided"). 37 | exclude("org.apache.spark", "spark-core_2.12"), 38 | ("org.apache.spark" %% "spark-streaming-kafka-0-10" % sparkVersion). 39 | exclude("org.apache.spark", "spark-core_2.12"), 40 | 41 | "com.typesafe" % "config" % typesafeVersion, 42 | "com.databricks" %% "spark-avro" % avroVersion, 43 | "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, 44 | "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, 45 | "com.hortonworks" % "shc-core" % hbaseConnectorVersion, 46 | ) 47 | 48 | unmanagedSourceDirectories in Compile := (scalaSource in Compile).value :: Nil 49 | 50 | assemblyMergeStrategy in assembly := { 51 | case PathList("META-INF", xs @ _*) => MergeStrategy.discard 52 | case x => MergeStrategy.first 53 | } 54 | -------------------------------------------------------------------------------- /streaming/spark/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.zookeeper.quorum 5 | zoo1 6 | 7 | 8 | hbase.rootdir 9 | hdfs://namenode-1:8020/hbase 10 | 11 | 12 | hbase.master.port 13 | 16020 14 | 15 | 16 | zookeeper.znode.parent 17 | /hbase 18 | 19 | 20 | hbase.zookeeper.property.clientPort 21 | 2181 22 | 23 | 24 | hbase.cluster.distributed 25 | true 26 | 27 | 28 | hbase.rest.ssl.enabled 29 | false 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /streaming/spark/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.mitosis 8 | search-flight-spark-streaming 9 | 0.1.0 10 | 11 | 12 | 2.11.11 13 | 14 | 2.4.0 15 | 1.1.1-2.1-s_2.11 16 | 4.0.0 17 | 2.9.8 18 | 1.3.0 19 | 1.2.14 20 | 1.16.20 21 | 22 | 23 | 24 | 25 | hortonworks 26 | hortonworks 27 | http://repo.hortonworks.com/content/repositories/releases/ 28 | 29 | 30 | 31 | 32 | 33 | org.apache.maven.plugins 34 | maven-compiler-plugin 35 | 2.3.1 36 | 37 | 1.8 38 | 1.8 39 | 40 | 41 | 42 | maven-resources-plugin 43 | 2.6 44 | 45 | 46 | copy-resources 47 | validate 48 | 49 | copy-resources 50 | 51 | 52 | ${basedir}/target 53 | 54 | 55 | resources 56 | true 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-dependency-plugin 66 | 67 | 68 | copy-dependencies 69 | prepare-package 70 | 71 | copy-dependencies 72 | 73 | 74 | ${project.build.directory}/libs 75 | false 76 | false 77 | true 78 | 79 | 80 | 81 | 82 | 83 | 84 | org.apache.maven.plugins 85 | maven-jar-plugin 86 | 3.0.2 87 | 88 | 89 | 90 | true 91 | libs/ 92 | com.mitosis.Application 93 | 94 | 95 | . 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | org.apache.spark 106 | spark-core_2.12 107 | ${spark.version} 108 | 109 | 110 | 111 | org.apache.spark 112 | spark-sql_2.12 113 | ${spark.version} 114 | 115 | 116 | 117 | org.apache.spark 118 | spark-streaming_2.12 119 | ${spark.version} 120 | 121 | 122 | 123 | org.apache.spark 124 | spark-streaming-kafka-0-10_2.12 125 | ${spark.version} 126 | 127 | 128 | 129 | com.databricks 130 | spark-avro_2.11 131 | ${avro.version} 132 | 133 | 134 | 135 | com.hortonworks 136 | shc-core 137 | ${hbase-connector.version} 138 | 139 | 140 | 141 | 142 | com.fasterxml.jackson.core 143 | jackson-databind 144 | ${jackson.version} 145 | 146 | 147 | 148 | 149 | com.fasterxml.jackson.core 150 | jackson-core 151 | ${jackson.version} 152 | 153 | 154 | 155 | com.typesafe 156 | config 157 | ${typesafe.version} 158 | 159 | 160 | 161 | log4j 162 | log4j 163 | ${log4j.version} 164 | 165 | 166 | 167 | org.projectlombok 168 | lombok 169 | ${lombok.version} 170 | 171 | 172 | 173 | 174 | -------------------------------------------------------------------------------- /streaming/spark/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.2 2 | -------------------------------------------------------------------------------- /streaming/spark/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // https://github.com/sbt/sbt/wiki/sbt-1.x-plugin-migration 2 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 3 | // addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.1") 4 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.2-RC2") -------------------------------------------------------------------------------- /streaming/spark/src/main/java/com/mitosis/Application.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | import java.util.Collection; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | 12 | import org.apache.avro.Schema; 13 | import org.apache.avro.Schema.Parser; 14 | import org.apache.avro.generic.GenericRecord; 15 | import org.apache.avro.io.DatumReader; 16 | import org.apache.avro.io.Decoder; 17 | import org.apache.avro.io.DecoderFactory; 18 | import org.apache.avro.specific.SpecificDatumReader; 19 | import org.apache.kafka.clients.consumer.ConsumerRecord; 20 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 21 | import org.apache.kafka.common.serialization.StringDeserializer; 22 | import org.apache.spark.api.java.JavaRDD; 23 | import org.apache.spark.sql.Dataset; 24 | import org.apache.spark.sql.Row; 25 | import org.apache.spark.sql.RowFactory; 26 | import org.apache.spark.sql.SparkSession; 27 | import org.apache.spark.sql.types.DataTypes; 28 | import org.apache.spark.sql.types.StructField; 29 | import org.apache.spark.sql.types.StructType; 30 | import org.apache.spark.streaming.Duration; 31 | import org.apache.spark.streaming.StreamingContext; 32 | import org.apache.spark.streaming.api.java.JavaInputDStream; 33 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 34 | import org.apache.spark.streaming.kafka010.ConsumerStrategies; 35 | import org.apache.spark.streaming.kafka010.KafkaUtils; 36 | import org.apache.spark.streaming.kafka010.LocationStrategies; 37 | 38 | import com.fasterxml.jackson.databind.ObjectMapper; 39 | import com.typesafe.config.Config; 40 | import com.typesafe.config.ConfigFactory; 41 | 42 | public class Application { 43 | 44 | public static void main(String[] args) throws InterruptedException { 45 | 46 | Config config = ConfigFactory.parseResources("app.conf"); 47 | 48 | SparkSession sparkSession = SparkSession 49 | .builder() 50 | .master("local[*]") 51 | .appName("search-flight-spark-streaming") 52 | .config("spark.driver.allowMultipleContexts", "true") // to remove 53 | .getOrCreate(); 54 | 55 | String flightInfoHbaseSchema = "{" 56 | + "\"table\":{\"namespace\":\"default\", \"name\":\"flightInfo\", \"tableCoder\":\"PrimitiveType\"}" 57 | + "\"rowkey\":\"key\"" + "\"columns\":{" 58 | + "\"key\":{\"cf\":\"rowkey\", \"col\":\"key\", \"type\":\"string\"}" 59 | + "\"departingId\":{\"cf\":\"searchFlightInfo\", \"col\":\"departingId\", \"type\":\"string\"}" 60 | + "\"arrivingId\":{\"cf\":\"searchFlightInfo\", \"col\":\"arrivingId\", \"type\":\"string\"}" 61 | + "\"tripType\":{\"cf\":\"searchFlightInfo\", \"col\":\"tripType\", \"type\":\"string\"}" 62 | + "\"departureDate\":{\"cf\":\"searchFlightInfo\", \"col\":\"departureDate\", \"type\":\"string\"}" 63 | + "\"arrivalDate\":{\"cf\":\"searchFlightInfo\", \"col\":\"arrivalDate\", \"type\":\"string\"}" 64 | + "\"passengerNumber\":{\"cf\":\"searchFlightInfo\", \"col\":\"passengerNumber\", \"type\":\"integer\"}" 65 | + "\"cabinClass\":{\"cf\":\"searchFlightInfo\", \"col\":\"cabinClass\", \"type\":\"string\"}" + "\"}" 66 | + "\"}"; 67 | 68 | JavaStreamingContext streamingContext = new JavaStreamingContext(new StreamingContext(sparkSession.sparkContext(), new Duration(1000))); 69 | 70 | String server = config.getStringList("producer.hosts").get(0); 71 | 72 | Map kafkaParams = new HashMap<>(); 73 | kafkaParams.put("bootstrap.servers", server); 74 | kafkaParams.put("key.deserializer", StringDeserializer.class); 75 | kafkaParams.put("value.deserializer", ByteArrayDeserializer.class); 76 | kafkaParams.put("auto.offset.reset", "latest"); 77 | kafkaParams.put("group.id", "mitosis"); 78 | kafkaParams.put("enable.auto.commit", false); 79 | 80 | // topic names which will be read 81 | Collection topics = Arrays.asList(config.getString("producer.topic")); 82 | 83 | JavaInputDStream> stream = KafkaUtils.createDirectStream(streamingContext, 84 | LocationStrategies.PreferConsistent(), 85 | ConsumerStrategies.Subscribe(topics, kafkaParams)); 86 | 87 | stream.foreachRDD(rdd -> { 88 | JavaRDD flightInfoRdd = rdd.map(record -> { 89 | 90 | Schema flightInfoAvroSchema = new Parser() 91 | .parse(new File(Application.class.getResource("/flight-info.schema.avsc").toURI())); 92 | DatumReader reader = new SpecificDatumReader(flightInfoAvroSchema); 93 | Decoder decoder = DecoderFactory.get().binaryDecoder(record.value(), null); 94 | GenericRecord flightInfoJson = reader.read(null, decoder); 95 | ObjectMapper mapper = new ObjectMapper(); 96 | 97 | FlightInfoBean flightInfoBean = mapper.readValue(flightInfoJson.toString(), FlightInfoBean.class); 98 | Random random = new java.util.Random(); 99 | flightInfoBean.setRowKey(random.nextLong()); 100 | return flightInfoBean; 101 | }); 102 | 103 | Dataset flightInfoDF = sparkSession.createDataFrame(flightInfoRdd, FlightInfoBean.class); 104 | 105 | flightInfoDF.write() 106 | .option("HBaseTableCatalog.tableCatalog", flightInfoHbaseSchema) 107 | .option("HBaseTableCatalog.newTable", 8) 108 | .format("org.apache.spark.sql.execution.datasources.hbase") 109 | .save(); 110 | }); 111 | 112 | // create streaming context and submit streaming jobs 113 | streamingContext.start(); 114 | 115 | // wait to killing signals etc. 116 | streamingContext.awaitTermination(); 117 | } 118 | 119 | } -------------------------------------------------------------------------------- /streaming/spark/src/main/java/com/mitosis/FlightInfoBean.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import lombok.ToString; 6 | 7 | @Setter 8 | @Getter 9 | @ToString 10 | public class FlightInfoBean { 11 | 12 | private Long rowKey; 13 | private String departingId; 14 | private String arrivingId; 15 | private String tripType; 16 | private Long departureDate; 17 | private Long arrivalDate; 18 | private Integer passengerNumber; 19 | private String cabinClass; 20 | } 21 | -------------------------------------------------------------------------------- /streaming/spark/src/main/resources/app.conf: -------------------------------------------------------------------------------- 1 | producer { 2 | batchSize: 16384 3 | topic: "flightInfoTopic" 4 | 5 | hosts: [ 6 | "kafka:9092" 7 | ] 8 | } 9 | 10 | streaming { 11 | db { 12 | host: "regionserver-1" 13 | port: "16000" 14 | table: "flightInfo" 15 | columnFamily: "flightInfoCF" 16 | } 17 | 18 | window: 1 19 | } 20 | -------------------------------------------------------------------------------- /streaming/spark/src/main/resources/flight-info.schema.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "flightInfo", 4 | "fields": [ 5 | { "name": "departingId", "type": "string" }, 6 | { "name": "arrivingId", "type": "string" }, 7 | { "name": "tripType", "type": { "type": "enum", "name": "TripType", "symbols": ["ONE_WAY", "ROUND_TRIP"] } }, 8 | { "name": "departureDate", "type": "string" }, 9 | { "name": "arrivalDate", "type": "string" }, 10 | { "name": "passengerNumber", "type": "int" }, 11 | { "name": "cabinClass", "type": { "type": "enum", "name": "CabinClass", "symbols": ["ECONOMY", "PRENIUM", "BUSINESS"] } } 12 | ] 13 | } -------------------------------------------------------------------------------- /streaming/spark/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.logger.org.apache.spark.storage.BlockManager=OFF 2 | 3 | log4j.rootLogger=DEBUG, CA 4 | 5 | log4j.appender.CA=org.apache.log4j.ConsoleAppender 6 | 7 | log4j.appender.CA.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/Main.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis 2 | 3 | import java.io.IOException 4 | import scala.io.Source 5 | 6 | import org.apache.spark.sql.{ DataFrame, SparkSession, Row } 7 | import org.apache.spark.sql.types.{ StructField, StructType, IntegerType, LongType, StringType } 8 | import com.mitosis.beans.FlightInfoBean 9 | import com.mitosis.utils.JsonUtils 10 | import com.mitosis.config.ConfigurationFactory 11 | import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe 12 | import org.apache.spark.streaming.kafka010.KafkaUtils 13 | import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent 14 | import org.apache.spark.sql.SparkSession 15 | import org.apache.kafka.common.serialization.{ ByteArrayDeserializer, StringDeserializer } 16 | import org.apache.avro.io.DatumReader 17 | import org.apache.avro.io.Decoder 18 | import org.apache.avro.specific.SpecificDatumReader 19 | import org.apache.avro.generic.GenericRecord 20 | import org.apache.avro.io.DecoderFactory 21 | import org.apache.log4j.Logger 22 | import org.apache.spark.streaming.{ Seconds, StreamingContext } 23 | 24 | import org.apache.avro.Schema 25 | import org.apache.avro.Schema.Parser 26 | import org.apache.spark.sql.execution.datasources.hbase.HBaseTableCatalog 27 | 28 | object Main { 29 | 30 | private[this] lazy val logger = Logger.getLogger(getClass) 31 | 32 | private[this] val config = ConfigurationFactory.load() 33 | 34 | /** 35 | * Json decode UDF function 36 | * 37 | * @param text the encoded JSON string 38 | * @return Returns record bean 39 | */ 40 | def jsonDecode(text: String): FlightInfoBean = { 41 | try { 42 | JsonUtils.deserialize(text, classOf[FlightInfoBean]) 43 | } catch { 44 | case e: IOException => 45 | logger.error(e.getMessage, e) 46 | null 47 | } 48 | } 49 | 50 | val flightInfoHbaseSchema = s"""{ 51 | |"table":{"namespace":"default", "name":"flightInfo", "tableCoder":"PrimitiveType"}, 52 | |"rowkey":"key", 53 | |"columns":{ 54 | |"key":{"cf":"rowkey", "col":"key", "type":"string"}, 55 | |"departingId":{"cf":"searchFlightInfo", "col":"departingId", "type":"string"}, 56 | |"arrivingId":{"cf":"searchFlightInfo", "col":"arrivingId", "type":"string"}, 57 | |"tripType":{"cf":"searchFlightInfo", "col":"tripType", "type":"string"}, 58 | |"departureDate":{"cf":"searchFlightInfo", "col":"departureDate", "type":"string"}, 59 | |"arrivalDate":{"cf":"searchFlightInfo", "col":"arrivalDate", "type":"string"}, 60 | |"passengerNumber":{"cf":"searchFlightInfo", "col":"passengerNumber", "type":"integer"}, 61 | |"cabinClass":{"cf":"searchFlightInfo", "col":"cabinClass", "type":"string"} 62 | |} 63 | |}""".stripMargin 64 | 65 | val flightInfoDfSchema = new StructType() 66 | .add(StructField("key", StringType, true)) 67 | .add(StructField("departingId", StringType, true)) 68 | .add(StructField("arrivingId", StringType, true)) 69 | .add(StructField("tripType", StringType, true)) 70 | .add(StructField("departureDate", StringType, true)) 71 | .add(StructField("arrivalDate", StringType, true)) 72 | .add(StructField("passengerNumber", IntegerType, true)) 73 | .add(StructField("cabinClass", StringType, true)) 74 | 75 | def main(args: Array[String]): Unit = { 76 | 77 | val sparkSession = SparkSession.builder 78 | .appName("search-flight-spark-streaming") 79 | .config("spark.hbase.host", config.streaming.db.host) 80 | .getOrCreate() 81 | 82 | val streamingContext = new StreamingContext(sparkSession.sparkContext, Seconds(config.streaming.window)) 83 | 84 | val servers = config.producer.hosts.toArray.mkString(",") 85 | 86 | val kafkaParams = Map[String, Object]( 87 | "bootstrap.servers" -> servers, 88 | "key.deserializer" -> classOf[StringDeserializer], 89 | "value.deserializer" -> classOf[ByteArrayDeserializer], 90 | "auto.offset.reset" -> "latest", 91 | "group.id" -> "mitosis", 92 | "enable.auto.commit" -> (false: java.lang.Boolean)) 93 | 94 | // topic names which will be read 95 | val topics = Array(config.producer.topic) 96 | 97 | val stream = KafkaUtils.createDirectStream( 98 | streamingContext, 99 | PreferConsistent, 100 | Subscribe[String, Array[Byte]](topics, kafkaParams)) 101 | 102 | stream.foreachRDD(rdd => { 103 | val flightInfoRdd = rdd.map(record => { 104 | 105 | val flightInfoAvroSchema: Schema = new Parser().parse(Source.fromURL(getClass.getResource("/flight-info.schema.avsc")).mkString) 106 | val reader: DatumReader[GenericRecord] = new SpecificDatumReader[GenericRecord](flightInfoAvroSchema) 107 | val decoder: Decoder = DecoderFactory.get().binaryDecoder(record.value, null) 108 | val flightInfoJson: GenericRecord = reader.read(null, decoder) 109 | val flightInfo = jsonDecode(flightInfoJson.toString) 110 | val random = scala.util.Random 111 | Row( 112 | s"${random.nextLong()}", 113 | flightInfo.departingId, 114 | flightInfo.arrivingId, 115 | flightInfo.tripType, 116 | flightInfo.departureDate, 117 | flightInfo.arrivalDate, 118 | flightInfo.passengerNumber, 119 | flightInfo.cabinClass) 120 | }) 121 | 122 | val flightInfoDF = sparkSession.createDataFrame(flightInfoRdd, flightInfoDfSchema) 123 | 124 | val sc = sparkSession.sparkContext 125 | val sqlContext = sparkSession.sqlContext 126 | 127 | import sqlContext.implicits._ 128 | 129 | flightInfoDF.write.options( 130 | Map(HBaseTableCatalog.tableCatalog -> flightInfoHbaseSchema, HBaseTableCatalog.newTable -> "5")) 131 | .format("org.apache.spark.sql.execution.datasources.hbase") 132 | .save() 133 | }) 134 | 135 | // create streaming context and submit streaming jobs 136 | streamingContext.start() 137 | 138 | // wait to killing signals etc. 139 | streamingContext.awaitTermination() 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/beans/CabinClass.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | object CabinClass extends Enumeration { 4 | val ECONOMY, PRENIUM, BUSINESS = Value 5 | type CabinClass = Value 6 | } 7 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/beans/FlightInfoBean.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | import scala.beans.{ BeanProperty } 4 | 5 | class FlightInfoBean { 6 | 7 | @BeanProperty 8 | var departingId: String = _ 9 | 10 | @BeanProperty 11 | var arrivingId: String = _ 12 | 13 | @BeanProperty 14 | var tripType: String = _ 15 | 16 | @BeanProperty 17 | var departureDate: Long = _ 18 | 19 | @BeanProperty 20 | var arrivalDate: Long = _ 21 | 22 | @BeanProperty 23 | var passengerNumber: Int = _ 24 | 25 | @BeanProperty 26 | var cabinClass: String = _ 27 | } 28 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/beans/TripType.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.beans 2 | 3 | object TripType extends Enumeration { 4 | val ONE_WAY, ROUND_TRIP = Value 5 | type TripType = Value 6 | } 7 | 8 | 9 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/config/ConfigurationFactory.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config 2 | 3 | import com.mitosis.config.objects.Config 4 | import com.typesafe.config.ConfigBeanFactory 5 | import com.typesafe.config.ConfigFactory 6 | 7 | import java.io.File 8 | import java.io.InputStreamReader 9 | 10 | object ConfigurationFactory { 11 | 12 | /** 13 | * Loads configuration object by file 14 | * 15 | * @return Returns configuration objects 16 | */ 17 | def load(): Config = { 18 | val is = new InputStreamReader(getClass.getResourceAsStream(s"/app.conf")) 19 | val config: com.typesafe.config.Config = ConfigFactory.parseReader(is).resolve() 20 | ConfigBeanFactory.create(config, classOf[Config]) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/config/objects/Config.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class Config { 9 | 10 | @BeanProperty 11 | var producer: ProducerConfig = _ 12 | 13 | @BeanProperty 14 | var streaming: StreamingConfig = _ 15 | } 16 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/config/objects/ProducerConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import java.util.List 4 | 5 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 6 | 7 | //remove if not needed 8 | import scala.collection.JavaConversions._ 9 | 10 | class ProducerConfig { 11 | 12 | @BeanProperty 13 | var hosts: List[String] = _ 14 | 15 | @BeanProperty 16 | var batchSize: Int = _ 17 | 18 | @BeanProperty 19 | var topic: String = _ 20 | } 21 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/config/objects/StreamingConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class StreamingConfig { 9 | 10 | @BeanProperty 11 | var db: StreamingDbConfig = _ 12 | 13 | @BeanProperty 14 | var window: Int = _ 15 | } 16 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/config/objects/StreamingDbConfig.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.config.objects 2 | 3 | import scala.beans.{ BeanProperty, BooleanBeanProperty } 4 | 5 | //remove if not needed 6 | import scala.collection.JavaConversions._ 7 | 8 | class StreamingDbConfig { 9 | 10 | @BeanProperty 11 | var host: String = _ 12 | 13 | @BeanProperty 14 | var port: Int = _ 15 | 16 | @BeanProperty 17 | var table: String = _ 18 | 19 | @BeanProperty 20 | var columnFamily: String = _ 21 | } 22 | -------------------------------------------------------------------------------- /streaming/spark/src/main/scala/com/mitosis/utils/JsonUtils.scala: -------------------------------------------------------------------------------- 1 | package com.mitosis.utils 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | 5 | object JsonUtils { 6 | 7 | private val OBJECT_MAPPER: ObjectMapper = new ObjectMapper() 8 | 9 | /** 10 | * Returns object as JSON encoded string 11 | * 12 | * @param input the bean instance 13 | * @return Returns single line JSON string 14 | */ 15 | def serialize(input: AnyRef): String = 16 | OBJECT_MAPPER.writeValueAsString(input) 17 | 18 | /** 19 | * Returns JSON encoded string as real object 20 | * 21 | * @param input the encoded JSON string 22 | * @return Returns PoJo 23 | */ 24 | def deserialize[T](input: String, tClass: Class[T]): T = 25 | OBJECT_MAPPER.readValue(input, tClass) 26 | } 27 | -------------------------------------------------------------------------------- /streaming/storm/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre 2 | VOLUME /tmp 3 | 4 | ENV STORM_HOME /usr/local/storm 5 | 6 | ADD target/search-flight-storm-streaming-0.1.0-jar-with-dependencies.jar $STORM_HOME/libs/search-flight-streaming.jar 7 | 8 | WORKDIR $STORM_HOME 9 | CMD ["java", "-jar", "./libs/search-flight-streaming.jar" ] 10 | -------------------------------------------------------------------------------- /streaming/storm/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.mitosis 8 | search-flight-storm-streaming 9 | 0.1.0 10 | 11 | 12 | 1.2.2 13 | 0.11.0.0 14 | 1.3.0 15 | 1.2.14 16 | 1.16.20 17 | 18 | 19 | 20 | 21 | 22 | org.apache.storm 23 | storm-core 24 | ${storm.version} 25 | 26 | 27 | org.apache.storm 28 | storm-kafka-client 29 | ${storm.version} 30 | 31 | 32 | org.apache.kafka 33 | kafka-clients 34 | ${storm.kafka.client.version} 35 | 36 | 37 | org.apache.storm 38 | storm-hbase 39 | ${storm.version} 40 | 41 | 42 | 43 | com.typesafe 44 | config 45 | ${typesafe.version} 46 | 47 | 48 | 49 | log4j 50 | log4j 51 | ${log4j.version} 52 | 53 | 54 | 55 | org.projectlombok 56 | lombok 57 | ${lombok.version} 58 | 59 | 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-compiler-plugin 66 | 67 | 1.8 68 | 1.8 69 | 70 | 71 | 72 | 73 | org.apache.maven.plugins 74 | maven-assembly-plugin 75 | 76 | 77 | package 78 | 79 | single 80 | 81 | 82 | 83 | 84 | 85 | com.mitosis.Application 86 | 87 | 88 | 89 | 90 | jar-with-dependencies 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /streaming/storm/src/main/java/com/mitosis/Application.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import com.mitosis.spout.KafkaSpoutTopology; 4 | import org.apache.hadoop.hbase.HBaseConfiguration; 5 | import org.apache.hadoop.hbase.HColumnDescriptor; 6 | import org.apache.hadoop.hbase.HTableDescriptor; 7 | import org.apache.hadoop.hbase.client.HBaseAdmin; 8 | import org.apache.hadoop.hbase.util.Bytes; 9 | import org.apache.storm.LocalCluster; 10 | 11 | import com.typesafe.config.Config; 12 | import com.typesafe.config.ConfigFactory; 13 | 14 | import java.io.IOException; 15 | 16 | public class Application { 17 | 18 | public static void main(String[] args) throws IOException { 19 | new Application().run(); 20 | } 21 | 22 | protected void run() throws IOException { 23 | 24 | org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create(); 25 | conf.addResource("hbase-site.xml"); 26 | 27 | HBaseAdmin admin = new HBaseAdmin(conf); 28 | 29 | if (!admin.isTableAvailable("flightInfo")) { 30 | HTableDescriptor flightInfoTableDesc = new HTableDescriptor(Bytes.toBytes("flightInfo")); 31 | HColumnDescriptor searchFlightInfoColumnFamilyDesc = new HColumnDescriptor(Bytes.toBytes("searchFlightInfo")); 32 | flightInfoTableDesc.addFamily(searchFlightInfoColumnFamilyDesc); 33 | admin.createTable(flightInfoTableDesc); 34 | } 35 | 36 | Config config = ConfigFactory.parseResources("app.conf"); 37 | 38 | String brokerUrl = config.getStringList("producer.hosts").get(0); 39 | 40 | KafkaSpoutTopology topology = getTopology(); 41 | org.apache.storm.Config tpConf = topology.getConfig(); 42 | 43 | LocalCluster localCluster = new LocalCluster(); 44 | 45 | // Consumer. Sets up a topology that reads the given Kafka spouts and logs the received messages 46 | localCluster.submitTopology("search-flight-storm-streaming", tpConf, topology.getTopology(topology.getSpoutConfig(brokerUrl))); 47 | 48 | } 49 | 50 | protected KafkaSpoutTopology getTopology() { 51 | return new KafkaSpoutTopology(); 52 | } 53 | } -------------------------------------------------------------------------------- /streaming/storm/src/main/java/com/mitosis/FlightInfoBean.java: -------------------------------------------------------------------------------- 1 | package com.mitosis; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import lombok.ToString; 6 | 7 | @Setter 8 | @Getter 9 | @ToString 10 | public class FlightInfoBean { 11 | 12 | private Long rowKey; 13 | private String departingId; 14 | private String arrivingId; 15 | private String tripType; 16 | private Long departureDate; 17 | private Long arrivalDate; 18 | private Integer passengerNumber; 19 | private String cabinClass; 20 | } 21 | -------------------------------------------------------------------------------- /streaming/storm/src/main/java/com/mitosis/spout/KafkaFlightInfoBolt.java: -------------------------------------------------------------------------------- 1 | package com.mitosis.spout; 2 | 3 | import java.io.*; 4 | import java.util.*; 5 | 6 | import org.apache.avro.Schema; 7 | import org.apache.avro.generic.GenericDatumReader; 8 | import org.apache.avro.generic.GenericRecord; 9 | import org.apache.avro.io.DatumReader; 10 | import org.apache.avro.io.Decoder; 11 | import org.apache.avro.io.DecoderFactory; 12 | import org.apache.storm.shade.org.json.simple.JSONObject; 13 | import org.apache.storm.shade.org.json.simple.parser.JSONParser; 14 | import org.apache.storm.shade.org.json.simple.parser.ParseException; 15 | import org.apache.storm.task.TopologyContext; 16 | import org.apache.storm.topology.BasicOutputCollector; 17 | import org.apache.storm.topology.IBasicBolt; 18 | import org.apache.storm.topology.OutputFieldsDeclarer; 19 | import org.apache.storm.tuple.Fields; 20 | import org.apache.storm.tuple.Tuple; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import static org.apache.storm.utils.Utils.tuple; 25 | 26 | public class KafkaFlightInfoBolt implements IBasicBolt { 27 | 28 | protected static final Logger LOG = LoggerFactory.getLogger(KafkaFlightInfoBolt.class); 29 | 30 | public void prepare(Map topoConf, TopologyContext context) { 31 | } 32 | 33 | public void execute(Tuple tuple, BasicOutputCollector collector) { 34 | LOG.info("input tuple > "+tuple.getValues()); 35 | 36 | byte[] avroJson = (byte[]) tuple.getValues().get(4); 37 | 38 | try { 39 | Schema schema = Schema.parse(KafkaFlightInfoBolt.class.getResourceAsStream("/flight-info.schema.avsc")); 40 | DatumReader reader = new GenericDatumReader<>(schema); 41 | 42 | Decoder decoder = DecoderFactory.get().binaryDecoder(avroJson, null); 43 | GenericRecord payload = reader.read(null, decoder); 44 | 45 | LOG.info("payload = "+payload); 46 | 47 | JSONParser parser = new JSONParser(); 48 | JSONObject jsonObject = (JSONObject) parser.parse(payload.toString()); 49 | String departingId = jsonObject.get("departingId").toString(); 50 | String arrivingId = jsonObject.get("arrivingId").toString(); 51 | String tripType = jsonObject.get("tripType").toString(); 52 | String departureDate = jsonObject.get("departureDate").toString(); 53 | String arrivalDate = jsonObject.get("arrivalDate").toString(); 54 | Integer passengerNumber = Integer.valueOf(jsonObject.get("passengerNumber").toString()); 55 | String cabinClass = jsonObject.get("cabinClass").toString(); 56 | 57 | Random random = new java.util.Random(); 58 | collector.emit(tuple(random.nextLong(), departingId, arrivingId, tripType, departureDate, arrivalDate, passengerNumber, cabinClass)); 59 | } catch(ParseException e) { 60 | LOG.error("KafkaFlightInfoBolt - ParseException = "+e); 61 | } catch(IOException e) { 62 | LOG.error("KafkaFlightInfoBolt - IOException = "+e); 63 | } 64 | } 65 | 66 | public void cleanup() { 67 | 68 | } 69 | 70 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 71 | declarer.declare(new Fields("rowKey", "departingId", "arrivingId", "tripType", "departureDate", "arrivalDate", "passengerNumber", "cabinClass")); 72 | } 73 | 74 | @Override 75 | public Map getComponentConfiguration() { 76 | return null; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /streaming/storm/src/main/java/com/mitosis/spout/KafkaSpoutTopology.java: -------------------------------------------------------------------------------- 1 | package com.mitosis.spout; 2 | 3 | import com.typesafe.config.ConfigFactory; 4 | import org.apache.storm.hbase.bolt.HBaseBolt; 5 | import org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper; 6 | import org.apache.storm.kafka.spout.*; 7 | 8 | import org.apache.storm.Config; 9 | import org.apache.storm.generated.StormTopology; 10 | import org.apache.storm.kafka.spout.KafkaSpoutRetryExponentialBackoff.TimeInterval; 11 | import org.apache.storm.topology.TopologyBuilder; 12 | import org.apache.storm.tuple.Fields; 13 | 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | public class KafkaSpoutTopology { 18 | 19 | public Config getConfig() { 20 | Config config = new Config(); 21 | config.setDebug(true); 22 | Map hbConf = new HashMap<>(); 23 | hbConf.put("hbase.rootdir", "hdfs://namenode-1:8020/hbase"); 24 | hbConf.put("hbase.zookeeper.quorum","zoo1"); 25 | hbConf.put("hbase.master.port","16020"); 26 | hbConf.put("zookeeper.znode.parent","/hbase"); 27 | hbConf.put("hbase.zookeeper.property.clientPort","2181"); 28 | hbConf.put("hbase.cluster.distributed","true"); 29 | hbConf.put("hbase.rest.ssl.enabled","false"); 30 | 31 | config.put("hbase.conf", hbConf); 32 | return config; 33 | } 34 | 35 | public StormTopology getTopology(KafkaSpoutConfig spoutConfig) { 36 | 37 | final TopologyBuilder tp = new TopologyBuilder(); 38 | tp.setSpout("kafka_spout", new KafkaSpout<>(spoutConfig), 1); 39 | KafkaFlightInfoBolt kafkaBolt = new KafkaFlightInfoBolt(); 40 | 41 | SimpleHBaseMapper mapper = new SimpleHBaseMapper() 42 | .withRowKeyField("rowKey") 43 | .withColumnFields(new Fields("rowKey", "departingId", "arrivingId", "tripType", "departureDate", "arrivalDate", "passengerNumber", "cabinClass")) 44 | .withColumnFamily("searchFlightInfo"); 45 | 46 | HBaseBolt hbaseBolt = new HBaseBolt("flightInfo", mapper) 47 | .withConfigKey("hbase.conf"); 48 | 49 | tp.setBolt("kafka_bolt", kafkaBolt) 50 | .shuffleGrouping("kafka_spout"); 51 | 52 | tp.setBolt("hbase_bolt", hbaseBolt, 1) 53 | .fieldsGrouping("kafka_bolt", new Fields("rowKey", "departingId", "arrivingId", "tripType", "departureDate", "arrivalDate", "passengerNumber", "cabinClass")); 54 | 55 | return tp.createTopology(); 56 | } 57 | 58 | public KafkaSpoutConfig getSpoutConfig(String bootstrapServers) { 59 | // topic names which will be read 60 | com.typesafe.config.Config config = ConfigFactory.parseResources("app.conf"); 61 | String topic = config.getString("producer.topic"); 62 | 63 | return KafkaSpoutConfig.builder(bootstrapServers, topic) 64 | .setRetry(getRetryService()) 65 | .setOffsetCommitPeriodMs(10_000) 66 | .setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.LATEST) 67 | .setMaxUncommittedOffsets(250) 68 | .setProp("key.deserializer", 69 | "org.apache.kafka.common.serialization.StringDeserializer") 70 | .setProp("value.deserializer", 71 | "org.apache.kafka.common.serialization.ByteArrayDeserializer") 72 | .build(); 73 | } 74 | 75 | public KafkaSpoutRetryService getRetryService() { 76 | return new KafkaSpoutRetryExponentialBackoff(TimeInterval.microSeconds(500), 77 | TimeInterval.milliSeconds(2), Integer.MAX_VALUE, TimeInterval.seconds(10)); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /streaming/storm/src/main/resources/app.conf: -------------------------------------------------------------------------------- 1 | producer { 2 | batchSize: 16384 3 | topic: "flightInfoTopic" 4 | 5 | hosts: [ 6 | "kafka:9092" 7 | ] 8 | } 9 | 10 | streaming { 11 | db { 12 | host: "regionserver-1" 13 | port: "16000" 14 | table: "flightInfo" 15 | columnFamily: "flightInfoCF" 16 | } 17 | 18 | window: 1 19 | } 20 | -------------------------------------------------------------------------------- /streaming/storm/src/main/resources/flight-info.schema.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "flightInfo", 4 | "fields": [ 5 | { "name": "departingId", "type": "string" }, 6 | { "name": "arrivingId", "type": "string" }, 7 | { "name": "tripType", "type": { "type": "enum", "name": "TripType", "symbols": ["ONE_WAY", "ROUND_TRIP"] } }, 8 | { "name": "departureDate", "type": "string" }, 9 | { "name": "arrivalDate", "type": "string" }, 10 | { "name": "passengerNumber", "type": "int" }, 11 | { "name": "cabinClass", "type": { "type": "enum", "name": "CabinClass", "symbols": ["ECONOMY", "PRENIUM", "BUSINESS"] } } 12 | ] 13 | } -------------------------------------------------------------------------------- /streaming/storm/src/main/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hbase.zookeeper.quorum 5 | zoo1 6 | 7 | 8 | hbase.rootdir 9 | hdfs://namenode-1:8020/hbase 10 | 11 | 12 | hbase.master.port 13 | 16020 14 | 15 | 16 | zookeeper.znode.parent 17 | /hbase 18 | 19 | 20 | hbase.zookeeper.property.clientPort 21 | 2181 22 | 23 | 24 | hbase.cluster.distributed 25 | true 26 | 27 | 28 | hbase.rest.ssl.enabled 29 | false 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /streaming/storm/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Direct log messages to stdout 2 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 3 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 5 | 6 | log4j.rootLogger=DEBUG, CA 7 | 8 | log4j.appender.CA=org.apache.log4j.ConsoleAppender 9 | 10 | log4j.appender.CA.layout=org.apache.log4j.PatternLayout 11 | log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n 12 | -------------------------------------------------------------------------------- /webapp/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:boron 2 | 3 | # Create app directory 4 | RUN mkdir -p /usr/src/app/client 5 | RUN mkdir -p /usr/src/app/server 6 | 7 | WORKDIR /usr/src/app/client 8 | # Bundle client source 9 | ADD client . 10 | # Install client dependencies 11 | RUN yarn 12 | 13 | WORKDIR /usr/src/app/server 14 | # Bundle server source 15 | ADD server . 16 | # Install server dependencies 17 | RUN yarn 18 | 19 | WORKDIR /usr/src/app 20 | ADD package.json package.json 21 | 22 | RUN yarn 23 | 24 | RUN npm run build:prod 25 | 26 | EXPOSE 3000 5000 27 | CMD [ "npm", "run", "start:prod" ] -------------------------------------------------------------------------------- /webapp/Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM node:boron 2 | 3 | 4 | # Create app directory 5 | RUN mkdir -p /usr/src/app/server 6 | 7 | WORKDIR /usr/src/app/server 8 | # Bundle server source 9 | ADD server . 10 | 11 | WORKDIR /usr/src/app 12 | ADD package.json package.json 13 | 14 | RUN yarn 15 | 16 | EXPOSE 4000 5000 17 | CMD [ "npm", "run", "start:prod" ] -------------------------------------------------------------------------------- /webapp/client/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | max_line_length = off 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /webapp/client/README.md: -------------------------------------------------------------------------------- 1 | # search-flight 2 | 3 | This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 1.5.4. 4 | 5 | ## Development server 6 | 7 | Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files. 8 | 9 | ## Code scaffolding 10 | 11 | Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. 12 | 13 | ## Build 14 | 15 | Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `-prod` flag for a production build. 16 | 17 | ## Running unit tests 18 | 19 | Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io). 20 | 21 | ## Running end-to-end tests 22 | 23 | Run `ng e2e` to execute the end-to-end tests via [Protractor](http://www.protractortest.org/). 24 | 25 | ## Further help 26 | 27 | To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI README](https://github.com/angular/angular-cli/blob/master/README.md). 28 | -------------------------------------------------------------------------------- /webapp/client/angular.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@angular/cli/lib/config/schema.json", 3 | "version": 1, 4 | "newProjectRoot": "projects", 5 | "projects": { 6 | "client": { 7 | "root": "", 8 | "sourceRoot": "src", 9 | "projectType": "application", 10 | "prefix": "app", 11 | "schematics": { 12 | "@schematics/angular:component": { 13 | "styleext": "scss" 14 | } 15 | }, 16 | "architect": { 17 | "build": { 18 | "builder": "@angular-devkit/build-angular:browser", 19 | "options": { 20 | "outputPath": "public", 21 | "index": "src/index.html", 22 | "main": "src/main.ts", 23 | "polyfills": "src/polyfills.ts", 24 | "tsConfig": "src/tsconfig.app.json", 25 | "assets": [ 26 | "src/favicon.ico", 27 | "src/assets" 28 | ], 29 | "styles": [ 30 | "src/styles.scss" 31 | ], 32 | "scripts": [] 33 | }, 34 | "configurations": { 35 | "production": { 36 | "fileReplacements": [ 37 | { 38 | "replace": "src/environments/environment.ts", 39 | "with": "src/environments/environment.prod.ts" 40 | } 41 | ], 42 | "optimization": true, 43 | "outputHashing": "all", 44 | "sourceMap": false, 45 | "extractCss": true, 46 | "namedChunks": false, 47 | "aot": true, 48 | "extractLicenses": true, 49 | "vendorChunk": false, 50 | "buildOptimizer": true, 51 | "budgets": [ 52 | { 53 | "type": "initial", 54 | "maximumWarning": "2mb", 55 | "maximumError": "5mb" 56 | } 57 | ] 58 | } 59 | } 60 | }, 61 | "serve": { 62 | "builder": "@angular-devkit/build-angular:dev-server", 63 | "options": { 64 | "browserTarget": "client:build" 65 | }, 66 | "configurations": { 67 | "production": { 68 | "browserTarget": "client:build:production" 69 | } 70 | } 71 | }, 72 | "extract-i18n": { 73 | "builder": "@angular-devkit/build-angular:extract-i18n", 74 | "options": { 75 | "browserTarget": "client:build" 76 | } 77 | }, 78 | "test": { 79 | "builder": "@angular-devkit/build-angular:karma", 80 | "options": { 81 | "main": "src/test.ts", 82 | "polyfills": "src/polyfills.ts", 83 | "tsConfig": "src/tsconfig.spec.json", 84 | "karmaConfig": "src/karma.conf.js", 85 | "styles": [ 86 | "src/styles.scss" 87 | ], 88 | "scripts": [], 89 | "assets": [ 90 | "src/favicon.ico", 91 | "src/assets" 92 | ] 93 | } 94 | }, 95 | "lint": { 96 | "builder": "@angular-devkit/build-angular:tslint", 97 | "options": { 98 | "tsConfig": [ 99 | "src/tsconfig.app.json", 100 | "src/tsconfig.spec.json" 101 | ], 102 | "exclude": [ 103 | "**/node_modules/**" 104 | ] 105 | } 106 | } 107 | } 108 | } 109 | }, 110 | "defaultProject": "client" 111 | } 112 | -------------------------------------------------------------------------------- /webapp/client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "search-flight-webapp-client", 3 | "version": "0.1.0", 4 | "license": "ISC", 5 | "scripts": { 6 | "ng": "ng", 7 | "start": "ng lint && ng serve", 8 | "build": "ng build", 9 | "build:prod": "ng build --prod", 10 | "test": "ng test", 11 | "lint": "ng lint", 12 | "e2e": "ng e2e" 13 | }, 14 | "private": true, 15 | "dependencies": { 16 | "@angular/animations": "7.2.3", 17 | "@angular/cdk": "7.3.0", 18 | "@angular/common": "7.2.3", 19 | "@angular/compiler": "7.2.3", 20 | "@angular/core": "7.2.3", 21 | "@angular/flex-layout": "7.0.0-beta.23", 22 | "@angular/forms": "7.2.3", 23 | "@angular/http": "7.2.3", 24 | "@angular/material": "7.3.0", 25 | "@angular/material-moment-adapter": "7.3.0", 26 | "@angular/platform-browser": "7.2.3", 27 | "@angular/platform-browser-dynamic": "7.2.3", 28 | "@angular/router": "7.2.3", 29 | "apollo-angular": "1.5.0", 30 | "apollo-angular-link-http": "1.4.0", 31 | "apollo-cache-inmemory": "1.4.2", 32 | "apollo-client": "2.4.12", 33 | "apollo-link-http": "1.5.9", 34 | "apollo-link-ws": "1.0.12", 35 | "core-js": "2.6.3", 36 | "graphql": "14.1.1", 37 | "graphql-tag": "2.10.1", 38 | "moment": "2.24.0", 39 | "rxjs": "6.4.0", 40 | "rxjs-compat": "6.4.0", 41 | "subscriptions-transport-ws": "0.9.15", 42 | "zone.js": "0.8.29" 43 | }, 44 | "devDependencies": { 45 | "@angular-devkit/build-angular": "0.13.0", 46 | "@angular/cli": "7.3.0", 47 | "@angular/compiler-cli": "7.2.3", 48 | "@angular/language-service": "7.2.3", 49 | "@types/jasmine": "3.3.8", 50 | "@types/jasminewd2": "2.0.6", 51 | "@types/node": "10.12.20", 52 | "codelyzer": "4.5.0", 53 | "jasmine-core": "3.3.0", 54 | "jasmine-spec-reporter": "4.2.1", 55 | "karma": "4.0.0", 56 | "karma-chrome-launcher": "2.2.0", 57 | "karma-cli": "2.0.0", 58 | "karma-coverage-istanbul-reporter": "2.0.4", 59 | "karma-jasmine": "2.0.1", 60 | "karma-jasmine-html-reporter": "1.4.0", 61 | "karma-phantomjs-launcher": "1.0.4", 62 | "phantomjs-prebuilt": "2.1.16", 63 | "protractor": "5.4.2", 64 | "ts-node": "8.0.2", 65 | "tslint": "5.12.1", 66 | "typescript": ">=3.1.1 <3.3" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /webapp/client/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // Protractor configuration file, see link for more information 2 | // https://github.com/angular/protractor/blob/master/lib/config.ts 3 | 4 | const { SpecReporter } = require('jasmine-spec-reporter'); 5 | 6 | exports.config = { 7 | allScriptsTimeout: 11000, 8 | specs: [ 9 | './e2e/**/*.e2e-spec.ts' 10 | ], 11 | capabilities: { 12 | 'browserName': 'chrome' 13 | }, 14 | directConnect: true, 15 | baseUrl: 'http://localhost:4200/', 16 | framework: 'jasmine', 17 | jasmineNodeOpts: { 18 | showColors: true, 19 | defaultTimeoutInterval: 30000, 20 | print: function() {} 21 | }, 22 | onPrepare() { 23 | require('ts-node').register({ 24 | project: 'e2e/tsconfig.e2e.json' 25 | }); 26 | jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /webapp/client/src/app/app.component.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /webapp/client/src/app/app.component.scss: -------------------------------------------------------------------------------- 1 | body { 2 | margin: auto; 3 | padding: auto; 4 | } -------------------------------------------------------------------------------- /webapp/client/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import { Component } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-root', 5 | templateUrl: './app.component.html', 6 | styleUrls: ['./app.component.scss'] 7 | }) 8 | export class AppComponent { 9 | } 10 | -------------------------------------------------------------------------------- /webapp/client/src/app/app.module.ts: -------------------------------------------------------------------------------- 1 | import { BrowserModule } from '@angular/platform-browser'; 2 | import { NgModule } from '@angular/core'; 3 | import { RouterModule } from '@angular/router'; 4 | import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; 5 | import { HttpClientModule } from '@angular/common/http'; 6 | 7 | import { InMemoryCache } from 'apollo-cache-inmemory'; 8 | import { Apollo, ApolloModule } from 'apollo-angular'; 9 | import { split } from 'apollo-link'; 10 | import { HttpLink, HttpLinkModule } from 'apollo-angular-link-http'; 11 | import { WebSocketLink } from 'apollo-link-ws'; 12 | import { getMainDefinition } from 'apollo-utilities'; 13 | 14 | import { AppComponent } from './app.component'; 15 | import { appRoutes } from './app.routes'; 16 | 17 | @NgModule({ 18 | declarations: [ 19 | AppComponent 20 | ], 21 | imports: [ 22 | BrowserAnimationsModule, 23 | BrowserModule, 24 | RouterModule.forRoot(appRoutes), 25 | HttpLinkModule, 26 | HttpClientModule, 27 | ApolloModule, 28 | ], 29 | providers: [], 30 | bootstrap: [AppComponent] 31 | }) 32 | export class AppModule { 33 | 34 | constructor( 35 | apollo: Apollo, 36 | httpLink: HttpLink 37 | ) { 38 | const hLink = httpLink.create({ 39 | uri: '/graphql'}); 40 | 41 | const subscriptionLink = new WebSocketLink({ 42 | uri: `ws://localhost:5000/gql-ws`, 43 | options: { 44 | reconnect: true 45 | } 46 | }); 47 | 48 | const link = split( 49 | ({ query }) => { 50 | const { kind, operation } = getMainDefinition(query); 51 | return kind === 'OperationDefinition' && operation === 'subscription'; 52 | }, 53 | subscriptionLink, 54 | hLink 55 | ); 56 | 57 | apollo.create({ 58 | link: link, 59 | cache: new InMemoryCache() 60 | }); 61 | } 62 | } 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /webapp/client/src/app/app.routes.ts: -------------------------------------------------------------------------------- 1 | import { AppComponent } from './app.component'; 2 | 3 | export const appRoutes = [ 4 | { path: 'dashboard', loadChildren: './dashboard/dashboard.module#DashboardModule' }, 5 | { 6 | path: '**', redirectTo: 'dashboard', pathMatch: 'full' 7 | } 8 | ]; 9 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/dashboard.component.html: -------------------------------------------------------------------------------- 1 | 2 |
Search Flight
3 | 4 | 5 |

Why travel alone when you can discover new things with new people? Find your traveling partners ...

6 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/dashboard.component.scss: -------------------------------------------------------------------------------- 1 | .dashboard { 2 | &-title { 3 | width: 50%; 4 | } 5 | } -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/dashboard.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'sf-dashboard', 5 | templateUrl: './dashboard.component.html', 6 | styleUrls: ['./dashboard.component.scss'] 7 | }) 8 | export class DashboardComponent implements OnInit { 9 | 10 | constructor() { } 11 | 12 | ngOnInit() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/dashboard.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | import { CommonModule } from '@angular/common'; 3 | import { RouterModule } from '@angular/router'; 4 | import { FormsModule, ReactiveFormsModule } from '@angular/forms'; 5 | 6 | import { SearchFlightComponent } from './search-flight/search-flight.component'; 7 | import { DashboardComponent } from './dashboard.component'; 8 | import { dashboardRoutes } from './dashboard.routes'; 9 | import { AirportsService, FlightService, TweetService } from './search-flight/gql/service'; 10 | 11 | import { MaterialModule } from '../shared/material.module'; 12 | 13 | @NgModule({ 14 | imports: [ 15 | CommonModule, 16 | FormsModule, 17 | ReactiveFormsModule, 18 | RouterModule.forChild(dashboardRoutes), 19 | MaterialModule 20 | ], 21 | declarations: [SearchFlightComponent, DashboardComponent], 22 | providers: [FlightService, AirportsService, TweetService] 23 | }) 24 | export class DashboardModule { } 25 | 26 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/dashboard.routes.ts: -------------------------------------------------------------------------------- 1 | import { DashboardComponent } from './dashboard.component'; 2 | 3 | export const dashboardRoutes = [ 4 | { path: '', component: DashboardComponent } 5 | ]; 6 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/dto/airport.dto.ts: -------------------------------------------------------------------------------- 1 | export class AirportDto { 2 | AirportID: string; 3 | City: string; 4 | Country: string; 5 | Name: string; 6 | destinations: Array; 7 | } 8 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/dto/index.ts: -------------------------------------------------------------------------------- 1 | export * from './airport.dto'; 2 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/mutation/flight.mutations.ts: -------------------------------------------------------------------------------- 1 | export const FLIGHT_INFO_MUTATION = 2 | ` 3 | mutation flightInfo($flightInfo:FlightInfoInput) { 4 | sendFlightInfo(flightInfo:$flightInfo){ 5 | departingId 6 | arrivingId 7 | tripType 8 | departureDate 9 | arrivalDate 10 | passengerNumber 11 | cabinClass 12 | } 13 | } 14 | `; 15 | 16 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/query/airports.queries.ts: -------------------------------------------------------------------------------- 1 | export const AIRPORTS_QUERY = 2 | `query airports($airportToSearch:String, $airportId:String) { 3 | fetchAirports(airportToSearch:$airportToSearch, airportId:$airportId) { 4 | AirportID 5 | City 6 | Country 7 | Name 8 | destinations 9 | } 10 | } 11 | `; 12 | 13 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/service/airports.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import { Apollo } from 'apollo-angular'; 3 | import gql from 'graphql-tag'; 4 | 5 | import { AIRPORTS_QUERY } from '../query/airports.queries'; 6 | 7 | @Injectable() 8 | export class AirportsService { 9 | constructor(private apollo: Apollo) { 10 | } 11 | 12 | getAirports(airportToSearch: string, airportId?: string) { 13 | return this.apollo.query({ 14 | query: gql`${AIRPORTS_QUERY}`, 15 | variables: { 16 | airportId: airportId, 17 | airportToSearch: airportToSearch 18 | }, 19 | fetchPolicy: 'network-only' 20 | }); 21 | } 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/service/flight.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import { Apollo } from 'apollo-angular'; 3 | import gql from 'graphql-tag'; 4 | 5 | import { FLIGHT_INFO_MUTATION } from '../mutation/flight.mutations'; 6 | import { FlightInfo } from '../../../../shared/model/flight-info.model'; 7 | 8 | @Injectable() 9 | export class FlightService { 10 | constructor(private apollo: Apollo) { 11 | } 12 | 13 | sendFlightInfo(flightInfo: FlightInfo) { 14 | return this.apollo.mutate({ 15 | mutation: gql`${FLIGHT_INFO_MUTATION}`, 16 | variables: { 17 | flightInfo: flightInfo 18 | } 19 | }); 20 | } 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './airports.service'; 2 | export * from './flight.service'; 3 | export * from './tweet.service'; 4 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/service/tweet.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import { Apollo } from 'apollo-angular'; 3 | import gql from 'graphql-tag'; 4 | /* tslint:disable */ 5 | import { Observable } from 'rxjs/Rx'; 6 | 7 | import { GET_TWEETS } from '../subscription/tweet.subscription'; 8 | 9 | @Injectable() 10 | export class TweetService { 11 | constructor(private apollo: Apollo) { 12 | } 13 | 14 | getTweets(): Observable { 15 | return this.apollo.subscribe({ 16 | query: gql`${GET_TWEETS}` 17 | }); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/gql/subscription/tweet.subscription.ts: -------------------------------------------------------------------------------- 1 | export const GET_TWEETS = ` 2 | subscription GetTweets { 3 | getTweets { 4 | id 5 | } 6 | }`; 7 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/search-flight.component.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | Purchase a ticket 4 | 5 | 6 |
7 | 8 | 9 | Round trip 10 | One way 11 | 12 | 13 |
14 | 15 | 17 | 18 | 19 | 20 | 21 | {{ airport.Name }}, {{ airport.City }}, {{ airport.Country }} 22 | 23 | 24 | 25 | 26 |
27 | 28 | 29 | 31 | 32 | 33 | 34 | 35 | {{ airport.Name }}, {{ airport.City }}, {{ airport.Country }} 36 | 37 | 38 | 39 | 40 |
41 | 42 | 43 | 45 | 46 | 47 | 48 | 49 |
50 | 51 | 52 | 54 | 55 | 56 | 57 | 58 |
59 | 60 | 61 | 62 | {{ passengerNumber }} 63 | 64 | 65 | 66 | 67 | 68 | {{ cabinClass }} 69 | 70 | 71 | 72 |
73 | 74 | 75 |
76 | 77 | 78 |
-------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/search-flight.component.scss: -------------------------------------------------------------------------------- 1 | .search-flight-box { 2 | width: 100%; 3 | .mat-card { 4 | margin: 16px 0; 5 | } 6 | &-form { 7 | min-width: 150px; 8 | max-width: 500px; 9 | width: 100%; 10 | } 11 | &-full-width { 12 | width: 100%; 13 | } 14 | button:disabled, 15 | button[disabled] { 16 | color: black !important; 17 | } 18 | 19 | mat-form-field { 20 | width: 500px; 21 | } 22 | } -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/search-flight.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit, ElementRef, ViewChild } from '@angular/core'; 2 | import { FormControl, FormGroup, FormBuilder, Validators } from '@angular/forms'; 3 | /* tslint:disable */ 4 | import { Observable } from 'rxjs/Rx'; 5 | import { Subscription } from 'rxjs/Subscription'; 6 | import { map } from 'rxjs/operators/map'; 7 | import { switchMap } from 'rxjs/operators/switchMap'; 8 | import { debounceTime } from 'rxjs/operators/debounceTime'; 9 | import { distinctUntilChanged } from 'rxjs/operators/distinctUntilChanged'; 10 | 11 | import { FlightInfo, TripType, CabinClass } from '../../shared/model/flight-info.model'; 12 | import { Airport } from '../../shared/model/airport.model'; 13 | import { AirportsService, FlightService, TweetService } from './gql/service'; 14 | import { AirportDtoMapper, AirportMapper } from './util/'; 15 | import * as moment from 'moment'; 16 | 17 | @Component({ 18 | moduleId: module.id, 19 | selector: 'sf-search-flight', 20 | templateUrl: './search-flight.component.html', 21 | styleUrls: ['./search-flight.component.scss'] 22 | }) 23 | export class SearchFlightComponent implements OnInit { 24 | 25 | @ViewChild('departureAirportInput') departureAirportInput: ElementRef; 26 | @ViewChild('arrivalAirportInput') arrivalAirportInput: ElementRef; 27 | searchFlightForm: FormGroup; 28 | passengersNumberOptions = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; 29 | cabinsClassOptions = Object.keys(CabinClass).filter(k => typeof CabinClass[k as any] === 'string'); 30 | tripTypeOptions = Object.keys(TripType).filter(k => typeof TripType[k as any] === 'string'); 31 | minDate = new Date(); 32 | maxDate = new Date(2020, 0, 1); 33 | minDateTo = new Date(); 34 | maxDateTo = new Date(2020, 0, 1); 35 | filteredAirports: Observable>; 36 | filteredDestinationAirports: Observable>; 37 | 38 | private departureAirport: Airport; 39 | private arrivalAirport: Airport; 40 | private tweetSubscriber: Subscription; 41 | 42 | constructor(private flightService: FlightService, 43 | private airportsService: AirportsService, 44 | private tweetService: TweetService, 45 | private formBuilder: FormBuilder, 46 | ) { 47 | } 48 | 49 | ngOnInit() { 50 | this.searchFlightForm = this.formBuilder.group({ 51 | hideRequired: false, 52 | departingFrom: [null, [Validators.required]], 53 | arrivingAt: [null, [Validators.required]], 54 | departureDate: [null, [Validators.required]], 55 | arrivalDate: [null, [Validators.required]], 56 | passengerNumber: [1, [Validators.required]], 57 | cabinClass: [CabinClass.ECONOMY, [Validators.required]], 58 | tripType: [TripType.ROUND_TRIP, [Validators.required]] 59 | }); 60 | 61 | // load departure airports 62 | this.filteredAirports = this.searchFlightForm.get('departingFrom').valueChanges.pipe( 63 | distinctUntilChanged(), 64 | debounceTime(300), 65 | switchMap((airportToSearch: any) => this.loadAirports(airportToSearch, this.arrivalAirport)) 66 | ); 67 | 68 | // load destinations airports 69 | this.filteredDestinationAirports = this.searchFlightForm.get('arrivingAt').valueChanges.pipe( 70 | distinctUntilChanged(), 71 | debounceTime(300), 72 | switchMap((airportToSearch: any) => this.loadAirports(airportToSearch, this.departureAirport)) 73 | ); 74 | 75 | // arrivalDate must be greater than the departureDate 76 | this.searchFlightForm.get('departureDate').valueChanges.subscribe(response => { 77 | this.minDateTo = response === null ? new Date() : response; 78 | }); 79 | 80 | this.checkDepartureAirportInputChanges(); 81 | this.checkArrivalAirportInputChanges(); 82 | } 83 | /** 84 | * track departure airtport input for changes 85 | */ 86 | private checkDepartureAirportInputChanges() { 87 | return Observable.fromEvent(this.departureAirportInput.nativeElement, 'keyup') 88 | .debounceTime(150) 89 | .distinctUntilChanged() 90 | .subscribe((res) => { 91 | this.departureAirport = this.departureAirportInput.nativeElement.value === '' ? undefined : this.departureAirport; 92 | }); 93 | } 94 | 95 | /** 96 | * track arrival airtport input for changes 97 | */ 98 | private checkArrivalAirportInputChanges() { 99 | return Observable.fromEvent(this.arrivalAirportInput.nativeElement, 'keyup') 100 | .debounceTime(150) 101 | .distinctUntilChanged() 102 | .subscribe((res) => { 103 | this.arrivalAirport = this.arrivalAirportInput.nativeElement.value === '' ? undefined : this.arrivalAirport; 104 | }); 105 | } 106 | /** 107 | * method called when on submitting the form 108 | */ 109 | searchFlight() { 110 | const flightInfo: FlightInfo = new FlightInfo(); 111 | flightInfo.departingId = this.departureAirport.AirportID; 112 | flightInfo.arrivingId = this.arrivalAirport.AirportID; 113 | const dateFormat = 'YYYY-MM-DD'; 114 | const departureDate = this.searchFlightForm.get('departureDate').value; 115 | flightInfo.departureDate = moment.utc(departureDate, dateFormat).local().format(dateFormat); 116 | const arrivalDate = this.searchFlightForm.get('arrivalDate').value; 117 | flightInfo.arrivalDate = moment.utc(arrivalDate, dateFormat).local().format(dateFormat); 118 | flightInfo.passengerNumber = this.searchFlightForm.get('passengerNumber').value; 119 | flightInfo.cabinClass = this.searchFlightForm.get('cabinClass').value; 120 | flightInfo.tripType = this.searchFlightForm.get('tripType').value; 121 | this.flightService.sendFlightInfo(flightInfo).subscribe(res => { 122 | if (this.tweetSubscriber) { 123 | this.tweetSubscriber.unsubscribe(); 124 | } 125 | this.tweetSubscriber = this.tweetService.getTweets().subscribe({ 126 | next(data) { 127 | const tweets = data.getTweets; 128 | } 129 | }); 130 | }); 131 | } 132 | 133 | /** 134 | * 135 | * @param airport put the departingAirport in a property to use it 136 | * for loading the destination airports 137 | */ 138 | setDepartureAirport(airport: Airport) { 139 | this.departureAirport = airport; 140 | } 141 | /** 142 | * 143 | * @param airport put the arrivalAirport in a property to use it 144 | * for loading the departure airports 145 | */ 146 | setArrivalAirport(airport: Airport) { 147 | this.arrivalAirport = airport; 148 | } 149 | 150 | private loadAirports(airportToSearch: string, airportToFilterBy: Airport) { 151 | const airportId = airportToFilterBy === undefined ? undefined : airportToFilterBy.AirportID; 152 | return this.airportsService.getAirports(airportToSearch, airportId).pipe( 153 | map(response => { 154 | const airportsData = (response).data.fetchAirports; 155 | const airportsDtoList = AirportDtoMapper.toAirportsDto(airportsData); 156 | return AirportMapper.toAirports(airportsDtoList); 157 | })); 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/util/airport-dto-mapper.ts: -------------------------------------------------------------------------------- 1 | import { AirportDto } from './../gql/dto/'; 2 | 3 | /** 4 | * Details of a AirportDtoMapper 5 | */ 6 | 7 | export class AirportDtoMapper { 8 | 9 | /** 10 | * Map response to Array 11 | * @param airports 12 | */ 13 | public static toAirportsDto(airports: any): Array { 14 | let airportDto; 15 | if (airports) { 16 | airportDto = new Array(); 17 | airports.forEach(airport => { 18 | airportDto.push(this.toAirportDto(airport)); 19 | }); 20 | } 21 | return airportDto; 22 | } 23 | 24 | /** 25 | * Map response to AirportDto 26 | * @param airport 27 | */ 28 | public static toAirportDto(airport: AirportDto): AirportDto { 29 | let airportDto; 30 | if (airport) { 31 | airportDto = new AirportDto(); 32 | airportDto.AirportID = airport.AirportID; 33 | airportDto.City = airport.City; 34 | airportDto.Country = airport.Country; 35 | airportDto.Name = airport.Name; 36 | airportDto.destinations = airport.destinations; 37 | } 38 | return airportDto; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/util/airport-mapper.ts: -------------------------------------------------------------------------------- 1 | import { AirportDto } from './../gql/dto/'; 2 | import { Airport } from './../../../shared/model/'; 3 | 4 | /** 5 | * Details of a AirportMapper 6 | */ 7 | 8 | export class AirportMapper { 9 | 10 | /** 11 | * Map Array to Array 12 | * @param airportsDtoList 13 | */ 14 | public static toAirports(airportsDtoList: Array): Array { 15 | let airports; 16 | if (airportsDtoList) { 17 | airports = new Array(); 18 | airportsDtoList.forEach(airportDtoList => { 19 | airports.push(this.toAirport(airportDtoList)); 20 | }); 21 | } 22 | return airports; 23 | } 24 | 25 | /** 26 | * Map Array to Airport 27 | * @param airportDtoList 28 | */ 29 | public static toAirport(airportDtoList: AirportDto): Airport { 30 | let airport; 31 | if (airportDtoList) { 32 | airport = new Airport(); 33 | airport.AirportID = airportDtoList.AirportID; 34 | airport.City = airportDtoList.City; 35 | airport.Country = airportDtoList.Country; 36 | airport.Name = airportDtoList.Name; 37 | airport.destinations = airportDtoList.destinations; 38 | } 39 | return airport; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /webapp/client/src/app/dashboard/search-flight/util/index.ts: -------------------------------------------------------------------------------- 1 | export * from './airport-dto-mapper'; 2 | export * from './airport-mapper'; 3 | -------------------------------------------------------------------------------- /webapp/client/src/app/shared/material.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | 3 | import { MatMomentDateModule } from '@angular/material-moment-adapter'; 4 | 5 | import { CdkTableModule } from '@angular/cdk/table'; 6 | import { 7 | MatAutocompleteModule, MatButtonModule, MatButtonToggleModule, MatPaginatorModule, 8 | MatCardModule, MatCheckboxModule, MatChipsModule, MatDatepickerModule, 9 | MatDialogModule, MatGridListModule, MatIconModule, MatInputModule, 10 | MatListModule, MatMenuModule, MatProgressBarModule, MatProgressSpinnerModule, 11 | MatRadioModule, MatSelectModule, MatSidenavModule, MatSliderModule, MatSortModule, 12 | MatSlideToggleModule, MatSnackBarModule, MatTableModule, MatTabsModule, MatToolbarModule, 13 | MatTooltipModule, MatFormFieldModule, MatExpansionModule, MatStepperModule, MatNativeDateModule 14 | } from '@angular/material'; 15 | 16 | @NgModule({ 17 | exports: [ 18 | CdkTableModule, 19 | MatNativeDateModule, 20 | MatMomentDateModule, 21 | MatAutocompleteModule, 22 | MatButtonModule, 23 | MatButtonToggleModule, 24 | MatCardModule, 25 | MatCheckboxModule, 26 | MatChipsModule, 27 | MatDatepickerModule, 28 | MatDialogModule, 29 | MatExpansionModule, 30 | MatFormFieldModule, 31 | MatGridListModule, 32 | MatIconModule, 33 | MatInputModule, 34 | MatListModule, 35 | MatMenuModule, 36 | MatProgressBarModule, 37 | MatProgressSpinnerModule, 38 | MatRadioModule, 39 | MatSelectModule, 40 | MatSlideToggleModule, 41 | MatSliderModule, 42 | MatSidenavModule, 43 | MatSnackBarModule, 44 | MatStepperModule, 45 | MatTabsModule, 46 | MatToolbarModule, 47 | MatTooltipModule, 48 | MatPaginatorModule, 49 | MatSortModule, 50 | MatTableModule 51 | ] 52 | }) 53 | export class MaterialModule { } 54 | -------------------------------------------------------------------------------- /webapp/client/src/app/shared/model/airport.model.ts: -------------------------------------------------------------------------------- 1 | export class Airport { 2 | AirportID: string; 3 | City: string; 4 | Country: string; 5 | Name: string; 6 | destinations: Array; 7 | } 8 | -------------------------------------------------------------------------------- /webapp/client/src/app/shared/model/flight-info.model.ts: -------------------------------------------------------------------------------- 1 | import { Airport } from './airport.model'; 2 | 3 | export enum TripType { 4 | ONE_WAY = 'ONE_WAY', 5 | ROUND_TRIP = 'ROUND_TRIP' 6 | } 7 | 8 | export enum CabinClass { 9 | 10 | ECONOMY = 'ECONOMY', 11 | PRENIUM = 'PRENIUM', 12 | BUSINESS = 'BUSINESS' 13 | } 14 | 15 | export class FlightInfo { 16 | departingId: string; 17 | arrivingId: string; 18 | tripType: TripType; 19 | departureDate: string; 20 | arrivalDate: string; 21 | passengerNumber: number; 22 | cabinClass: CabinClass; 23 | } 24 | -------------------------------------------------------------------------------- /webapp/client/src/app/shared/model/index.ts: -------------------------------------------------------------------------------- 1 | export * from './flight-info.model'; 2 | export * from './airport.model'; 3 | -------------------------------------------------------------------------------- /webapp/client/src/assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Chabane/bigdata-playground/6838b5e33fefe81b7e1514c057dad5ebc48f4132/webapp/client/src/assets/.gitkeep -------------------------------------------------------------------------------- /webapp/client/src/browserslist: -------------------------------------------------------------------------------- 1 | # This file is currently used by autoprefixer to adjust CSS to support the below specified browsers 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | # 5 | # For IE 9-11 support, please remove 'not' from the last line of the file and adjust as needed 6 | 7 | > 0.5% 8 | last 2 versions 9 | Firefox ESR 10 | not dead 11 | not IE 9-11 -------------------------------------------------------------------------------- /webapp/client/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true 3 | }; 4 | -------------------------------------------------------------------------------- /webapp/client/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false 7 | }; 8 | 9 | /* 10 | * For easier debugging in development mode, you can import the following file 11 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 12 | * 13 | * This import should be commented out in production mode because it will have a negative impact 14 | * on performance if an error is thrown. 15 | */ 16 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI. 17 | -------------------------------------------------------------------------------- /webapp/client/src/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Chabane/bigdata-playground/6838b5e33fefe81b7e1514c057dad5ebc48f4132/webapp/client/src/favicon.ico -------------------------------------------------------------------------------- /webapp/client/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Search Flight 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /webapp/client/src/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration file, see link for more information 2 | // https://karma-runner.github.io/1.0/config/configuration-file.html 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | basePath: '', 7 | frameworks: ['jasmine', '@angular/cli'], 8 | plugins: [ 9 | require('karma-jasmine'), 10 | require('karma-chrome-launcher'), 11 | require('karma-jasmine-html-reporter'), 12 | require('karma-coverage-istanbul-reporter'), 13 | require('@angular/cli/plugins/karma'), 14 | require('karma-phantomjs-launcher'), 15 | require('phantomjs-prebuilt') 16 | ], 17 | client: { 18 | clearContext: false // leave Jasmine Spec Runner output visible in browser 19 | }, 20 | coverageIstanbulReporter: { 21 | reports: ['html', 'lcovonly'], 22 | fixWebpackSourcePaths: true 23 | }, 24 | angularCli: { 25 | environment: 'dev' 26 | }, 27 | reporters: ['progress', 'kjhtml'], 28 | port: 9876, 29 | colors: true, 30 | logLevel: config.LOG_INFO, 31 | autoWatch: true, 32 | browsers: ['PhantomJS'], 33 | singleRun: true 34 | }); 35 | }; -------------------------------------------------------------------------------- /webapp/client/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.log(err)); 13 | -------------------------------------------------------------------------------- /webapp/client/src/polyfills.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file includes polyfills needed by Angular and is loaded before the app. 3 | * You can add your own extra polyfills to this file. 4 | * 5 | * This file is divided into 2 sections: 6 | * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. 7 | * 2. Application imports. Files imported after ZoneJS that should be loaded before your main 8 | * file. 9 | * 10 | * The current setup is for so-called "evergreen" browsers; the last versions of browsers that 11 | * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), 12 | * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. 13 | * 14 | * Learn more in https://angular.io/guide/browser-support 15 | */ 16 | 17 | /*************************************************************************************************** 18 | * BROWSER POLYFILLS 19 | */ 20 | 21 | /** IE9, IE10 and IE11 requires all of the following polyfills. **/ 22 | // import 'core-js/es6/symbol'; 23 | // import 'core-js/es6/object'; 24 | // import 'core-js/es6/function'; 25 | // import 'core-js/es6/parse-int'; 26 | // import 'core-js/es6/parse-float'; 27 | // import 'core-js/es6/number'; 28 | // import 'core-js/es6/math'; 29 | // import 'core-js/es6/string'; 30 | // import 'core-js/es6/date'; 31 | // import 'core-js/es6/array'; 32 | // import 'core-js/es6/regexp'; 33 | // import 'core-js/es6/map'; 34 | // import 'core-js/es6/weak-map'; 35 | // import 'core-js/es6/set'; 36 | 37 | /** 38 | * If the application will be indexed by Google Search, the following is required. 39 | * Googlebot uses a renderer based on Chrome 41. 40 | * https://developers.google.com/search/docs/guides/rendering 41 | **/ 42 | // import 'core-js/es6/array'; 43 | 44 | /** IE10 and IE11 requires the following for NgClass support on SVG elements */ 45 | // import 'classlist.js'; // Run `npm install --save classlist.js`. 46 | 47 | /** IE10 and IE11 requires the following for the Reflect API. */ 48 | // import 'core-js/es6/reflect'; 49 | 50 | /** 51 | * Web Animations `@angular/platform-browser/animations` 52 | * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. 53 | * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). 54 | **/ 55 | // import 'web-animations-js'; // Run `npm install --save web-animations-js`. 56 | 57 | /** 58 | * By default, zone.js will patch all possible macroTask and DomEvents 59 | * user can disable parts of macroTask/DomEvents patch by setting following flags 60 | */ 61 | 62 | // (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame 63 | // (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick 64 | // (window as any).__zone_symbol__BLACK_LISTED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames 65 | 66 | /* 67 | * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js 68 | * with the following flag, it will bypass `zone.js` patch for IE/Edge 69 | */ 70 | // (window as any).__Zone_enable_cross_context_check = true; 71 | 72 | /*************************************************************************************************** 73 | * Zone JS is required by default for Angular itself. 74 | */ 75 | import 'zone.js/dist/zone'; // Included with Angular CLI. 76 | 77 | 78 | /*************************************************************************************************** 79 | * APPLICATION IMPORTS 80 | */ 81 | -------------------------------------------------------------------------------- /webapp/client/src/styles.scss: -------------------------------------------------------------------------------- 1 | @import '~@angular/material/prebuilt-themes/deeppurple-amber.css'; 2 | -------------------------------------------------------------------------------- /webapp/client/src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/dist/zone-testing'; 4 | import { getTestBed } from '@angular/core/testing'; 5 | import { 6 | BrowserDynamicTestingModule, 7 | platformBrowserDynamicTesting 8 | } from '@angular/platform-browser-dynamic/testing'; 9 | 10 | declare const require: any; 11 | 12 | // First, initialize the Angular testing environment. 13 | getTestBed().initTestEnvironment( 14 | BrowserDynamicTestingModule, 15 | platformBrowserDynamicTesting() 16 | ); 17 | // Then we find all the tests. 18 | const context = require.context('./', true, /\.spec\.ts$/); 19 | // And load the modules. 20 | context.keys().map(context); 21 | -------------------------------------------------------------------------------- /webapp/client/src/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "types": [] 6 | }, 7 | "exclude": [ 8 | "test.ts", 9 | "**/*.spec.ts" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /webapp/client/src/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/spec", 5 | "types": [ 6 | "jasmine", 7 | "node" 8 | ] 9 | }, 10 | "files": [ 11 | "test.ts", 12 | "polyfills.ts" 13 | ], 14 | "include": [ 15 | "**/*.spec.ts", 16 | "**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /webapp/client/src/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tslint.json", 3 | "rules": { 4 | "directive-selector": [ 5 | true, 6 | "attribute", 7 | "app", 8 | "camelCase" 9 | ], 10 | "component-selector": [ 11 | true, 12 | "element", 13 | "app", 14 | "kebab-case" 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /webapp/client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "sourceMap": true, 7 | "declaration": false, 8 | "module": "es2015", 9 | "moduleResolution": "node", 10 | "emitDecoratorMetadata": true, 11 | "experimentalDecorators": true, 12 | "target": "es5", 13 | "typeRoots": [ 14 | "node_modules/@types" 15 | ], 16 | "lib": [ 17 | "es2018", 18 | "esnext.asynciterable", 19 | "dom" 20 | ] 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /webapp/client/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "rulesDirectory": [ 3 | "node_modules/codelyzer" 4 | ], 5 | "rules": { 6 | "arrow-return-shorthand": true, 7 | "callable-types": true, 8 | "class-name": true, 9 | "comment-format": [ 10 | true, 11 | "check-space" 12 | ], 13 | "curly": true, 14 | "deprecation": { 15 | "severity": "warn" 16 | }, 17 | "eofline": true, 18 | "forin": true, 19 | "import-blacklist": [ 20 | true, 21 | "rxjs/Rx" 22 | ], 23 | "import-spacing": true, 24 | "indent": [ 25 | true, 26 | "spaces" 27 | ], 28 | "interface-over-type-literal": true, 29 | "label-position": true, 30 | "max-line-length": [ 31 | true, 32 | 140 33 | ], 34 | "member-access": false, 35 | "member-ordering": [ 36 | true, 37 | { 38 | "order": [ 39 | "static-field", 40 | "instance-field", 41 | "static-method", 42 | "instance-method" 43 | ] 44 | } 45 | ], 46 | "no-arg": true, 47 | "no-bitwise": true, 48 | "no-console": [ 49 | true, 50 | "debug", 51 | "info", 52 | "time", 53 | "timeEnd", 54 | "trace" 55 | ], 56 | "no-construct": true, 57 | "no-debugger": true, 58 | "no-duplicate-super": true, 59 | "no-empty": false, 60 | "no-empty-interface": true, 61 | "no-eval": true, 62 | "no-inferrable-types": [ 63 | true, 64 | "ignore-params" 65 | ], 66 | "no-misused-new": true, 67 | "no-non-null-assertion": true, 68 | "no-redundant-jsdoc": true, 69 | "no-shadowed-variable": true, 70 | "no-string-literal": false, 71 | "no-string-throw": true, 72 | "no-switch-case-fall-through": true, 73 | "no-trailing-whitespace": true, 74 | "no-unnecessary-initializer": true, 75 | "no-unused-expression": true, 76 | "no-use-before-declare": true, 77 | "no-var-keyword": true, 78 | "object-literal-sort-keys": false, 79 | "one-line": [ 80 | true, 81 | "check-open-brace", 82 | "check-catch", 83 | "check-else", 84 | "check-whitespace" 85 | ], 86 | "prefer-const": true, 87 | "quotemark": [ 88 | true, 89 | "single" 90 | ], 91 | "radix": true, 92 | "semicolon": [ 93 | true, 94 | "always" 95 | ], 96 | "triple-equals": [ 97 | true, 98 | "allow-null-check" 99 | ], 100 | "typedef-whitespace": [ 101 | true, 102 | { 103 | "call-signature": "nospace", 104 | "index-signature": "nospace", 105 | "parameter": "nospace", 106 | "property-declaration": "nospace", 107 | "variable-declaration": "nospace" 108 | } 109 | ], 110 | "unified-signatures": true, 111 | "variable-name": false, 112 | "whitespace": [ 113 | true, 114 | "check-branch", 115 | "check-decl", 116 | "check-operator", 117 | "check-separator", 118 | "check-type" 119 | ], 120 | "no-output-on-prefix": true, 121 | "use-input-property-decorator": true, 122 | "use-output-property-decorator": true, 123 | "use-host-property-decorator": true, 124 | "no-input-rename": true, 125 | "no-output-rename": true, 126 | "use-life-cycle-interface": true, 127 | "use-pipe-transform-interface": true, 128 | "component-class-suffix": true, 129 | "directive-class-suffix": true 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /webapp/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "search-flight-webapp", 3 | "version": "0.1.0", 4 | "description": "flight Patch Test", 5 | "main": "index.js", 6 | "scripts": { 7 | "lint": "cd client && npm run lint && cd ../server && npm run lint", 8 | "test": "cd client && npm run test && cd ../server && npm run test", 9 | "clear:client": "cd client && rimraf public && cd ..", 10 | "clear:server": "cd server && rimraf public dist && cd ..", 11 | "build:client": "cd client && npm run build && cd ..", 12 | "build:client-prod": "cd client && npm run build:prod && cd ..", 13 | "build:server": "cd server && npm run build && cd ..", 14 | "start:server": "cd server && nodemon server.js", 15 | "start:server-prod": "cd server && node server.js", 16 | "copy:public": "copyfiles -u 1 client/public/* server/dist && rimraf client/public", 17 | "build:dev": "npm run clear:client && npm run clear:server && npm run build:client && npm run build:server && npm run copy:public", 18 | "start:dev": "npm run start:server", 19 | "build:prod": "npm run clear:client && npm run clear:server && npm run build:client-prod && npm run build:server && npm run copy:public", 20 | "start:prod": "npm run start:server-prod" 21 | }, 22 | "keywords": [ 23 | "flight", 24 | "PATCH", 25 | "ANGULAR", 26 | "TEST", 27 | "NODEJS" 28 | ], 29 | "private": true, 30 | "author": "Chabane REFES", 31 | "license": "ISC", 32 | "devDependencies": { 33 | "husky": "1.3.1", 34 | "validate-commit-msg": "2.14.0", 35 | "copy": "0.3.2", 36 | "copyfiles": "2.1.0", 37 | "rimraf": "2.6.3" 38 | }, 39 | "dependencies": { 40 | "nodemon": "1.18.9" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /webapp/server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "search-flight-webapp-server", 3 | "version": "0.1.0", 4 | "description": "flight Patch Test Server", 5 | "main": "index.js", 6 | "author": "Chabane REFES", 7 | "license": "ISC", 8 | "scripts": { 9 | "build": "rimraf dist && tsc", 10 | "tsc": "tsc", 11 | "lint": "tslint 'src/**/*.ts'", 12 | "test": "echo working on it" 13 | }, 14 | "engines": { 15 | "node": ">= 6.9.0", 16 | "npm": ">= 3.0.0" 17 | }, 18 | "private": true, 19 | "dependencies": { 20 | "apollo-errors": "1.9.0", 21 | "apollo-link": "1.2.6", 22 | "apollo-server-express": "2.3.3", 23 | "avsc": "5.4.7", 24 | "body-parser": "1.18.3", 25 | "express": "4.16.4", 26 | "express-static": "1.2.6", 27 | "graphql": "14.1.1", 28 | "graphql-subscriptions": "1.0.0", 29 | "graphql-tools": "4.0.4", 30 | "kafka-node": "4.0.1", 31 | "mongoose": "5.4.8", 32 | "morgan": "1.9.1", 33 | "path": "0.12.7", 34 | "subscriptions-transport-ws": "0.9.15", 35 | "winston": "3.2.1" 36 | }, 37 | "devDependencies": { 38 | "@types/body-parser": "1.17.0", 39 | "@types/chai": "4.1.7", 40 | "@types/chai-http": "3.0.5", 41 | "@types/express": "4.16.1", 42 | "@types/graphql": "14.0.5", 43 | "@types/kafka-node": "2.0.7", 44 | "@types/mocha": "5.2.5", 45 | "@types/mongoose": "5.3.11", 46 | "@types/morgan": "1.7.35", 47 | "@types/node": "10.12.20", 48 | "@types/winston": "2.4.4", 49 | "chai": "4.2.0", 50 | "chai-http": "4.2.1", 51 | "concurrently": "4.1.0", 52 | "istanbul": "0.4.5", 53 | "mocha": "5.2.0", 54 | "nodemon": "1.18.9", 55 | "rimraf": "2.6.3", 56 | "tslint": "5.12.1", 57 | "typescript": "3.3.1" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /webapp/server/server.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict'; 3 | 4 | //module dependencies 5 | var server = require('./dist/src'); 6 | var debug = require('debug')('express:server'); 7 | var http = require('http'); 8 | var winston = require('winston'); 9 | 10 | //create http server 11 | var port = normalizePort(process.env.PORT || 4000); 12 | 13 | var bootstrap = server.Server.bootstrap(); 14 | var app = bootstrap.app; 15 | var apolloServer = bootstrap.apolloServer; 16 | 17 | apolloServer.applyMiddleware({ app }); 18 | 19 | app.set('port', port); 20 | 21 | app.listen({ port: port }, () => 22 | console.log(`🚀 Server ready at http://localhost:4000${apolloServer.graphqlPath}`) 23 | ); 24 | 25 | var httpServer = http.createServer(app); 26 | 27 | //add error handler 28 | httpServer.on('error', onError); 29 | 30 | //start listening on port 31 | httpServer.on('listening', onListening); 32 | 33 | /** 34 | * Normalize a port into a number, string, or false. 35 | */ 36 | function normalizePort(val) { 37 | var port = parseInt(val, 10); 38 | 39 | if (isNaN(port)) { 40 | // named pipe 41 | return val; 42 | } 43 | 44 | if (port >= 0) { 45 | // port number 46 | return port; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | /** 53 | * Event listener for HTTP server 'error' event. 54 | */ 55 | function onError(error) { 56 | if (error.syscall !== 'listen') { 57 | throw error; 58 | } 59 | 60 | var bind = typeof port === 'string' 61 | ? 'Pipe ' + port 62 | : 'Port ' + port; 63 | 64 | // handle specific listen errors with friendly messages 65 | switch (error.code) { 66 | case 'EACCES': 67 | winston.error(bind + ' requires elevated privileges'); 68 | process.exit(1); 69 | break; 70 | case 'EADDRINUSE': 71 | winston.error(bind + ' is already in use'); 72 | process.exit(1); 73 | break; 74 | default: 75 | throw error; 76 | } 77 | } 78 | 79 | /** 80 | * Event listener for HTTP server 'listening' event. 81 | */ 82 | function onListening() { 83 | var addr = httpServer.address(); 84 | var bind = typeof addr === 'string' 85 | ? 'pipe ' + addr 86 | : 'port ' + addr.port; 87 | winston.info('Listening on ' + bind); 88 | } -------------------------------------------------------------------------------- /webapp/server/src/db/config/database.ts: -------------------------------------------------------------------------------- 1 | import * as mongoose from 'mongoose'; 2 | import * as winston from 'winston'; 3 | 4 | mongoose.connect('mongodb://mongo:27017/mitosis'); 5 | const db = mongoose.connection; 6 | 7 | db.on('error', console.error.bind(console, 'connection error:')); 8 | db.once('openUri', function () { 9 | winston.info('Connected to MongoDB'); 10 | }); 11 | 12 | export { mongoose }; 13 | 14 | 15 | -------------------------------------------------------------------------------- /webapp/server/src/db/index.ts: -------------------------------------------------------------------------------- 1 | export * from './config/database'; 2 | export * from './models/flight-info'; 3 | export * from './models/airport'; 4 | export * from './models/tweet'; 5 | -------------------------------------------------------------------------------- /webapp/server/src/db/models/airport.ts: -------------------------------------------------------------------------------- 1 | import { mongoose } from '../config/database'; 2 | import { Schema, Document, Model } from 'mongoose'; 3 | import * as winston from 'winston'; 4 | import { Decimal128 } from 'bson'; 5 | 6 | export interface IAirport extends Document { 7 | AirportID: string; 8 | City: string; 9 | Country: string; 10 | Name: string; 11 | destinations: Array; 12 | } 13 | 14 | export interface IAirportModel extends Model { 15 | findAirports(airportToSearch: string, airportId?: string): Promise; 16 | } 17 | 18 | // create a schema 19 | const schema = new Schema({ 20 | AirportID: String, 21 | City: String, 22 | Country: String, 23 | Name: String, 24 | destinations: Array() 25 | }); 26 | 27 | schema.index( 28 | { 29 | City: 'text', 30 | Country: 'text', 31 | Name: 'text' 32 | }, 33 | ); 34 | 35 | // retrieve list of airports 36 | schema.static('findAirports', (airportToSearch, airportId) => { 37 | 38 | if (airportId === undefined) { 39 | return Airport 40 | .find() 41 | .or([ 42 | { 'Name': new RegExp(airportToSearch, 'i') }, 43 | { 'Country': new RegExp(airportToSearch, 'i') }, 44 | { 'City': new RegExp(airportToSearch, 'i') }]) 45 | .limit(10) 46 | .exec(); 47 | } else { 48 | return Airport 49 | .find() 50 | .where('destinations', airportId) 51 | .or([ 52 | { 'Name': new RegExp(airportToSearch, 'i') }, 53 | { 'Country': new RegExp(airportToSearch, 'i') }, 54 | { 'City': new RegExp(airportToSearch, 'i') }]) 55 | .limit(10) 56 | .exec(); 57 | } 58 | 59 | }); 60 | export const Airport = mongoose.model('Airport', schema) as IAirportModel; 61 | 62 | -------------------------------------------------------------------------------- /webapp/server/src/db/models/flight-info.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface IFlightInfo { 3 | departingId: string; 4 | arrivingId: string; 5 | tripType: string; 6 | departureDate: string; 7 | arrivalDate: string; 8 | passengerNumber: number; 9 | cabinClass: string; 10 | } 11 | -------------------------------------------------------------------------------- /webapp/server/src/db/models/tweet.ts: -------------------------------------------------------------------------------- 1 | export interface ITweet { 2 | id: number; 3 | } 4 | -------------------------------------------------------------------------------- /webapp/server/src/index.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import { ApolloServer, gql } from 'apollo-server-express'; 3 | import { makeExecutableSchema } from 'graphql-tools'; 4 | 5 | import { SubscriptionServer } from 'subscriptions-transport-ws'; 6 | import { execute, subscribe } from 'graphql'; 7 | import { createServer } from 'http'; 8 | 9 | import * as path from 'path'; 10 | import * as morgan from 'morgan'; 11 | import * as bodyParser from 'body-parser'; 12 | import * as winston from 'winston'; 13 | 14 | import { typeDefs, resolvers } from './schema'; 15 | import { KafkaTweetConsumer } from './kafka'; 16 | 17 | export class Server { 18 | app: express.Application; 19 | apolloServer: ApolloServer; 20 | 21 | /** 22 | * Bootstrap the application. 23 | * 24 | * @class Server 25 | * @method bootstrap 26 | * @static 27 | * @return {ng.auto.IInjectorService} Returns the newly created injector for this app. 28 | */ 29 | static bootstrap(): Server { 30 | return new Server(); 31 | } 32 | 33 | constructor() { 34 | this.app = express(); 35 | this.app.use(bodyParser.json()); 36 | this.app.use(bodyParser.urlencoded({ extended: false })); 37 | 38 | this.app.use(morgan('dev')); 39 | 40 | const executableSchema = makeExecutableSchema({ 41 | typeDefs: typeDefs as any, 42 | resolvers: resolvers as any 43 | }); 44 | 45 | this.apolloServer = new ApolloServer({ 46 | typeDefs: typeDefs as any, 47 | resolvers: resolvers as any 48 | }); 49 | 50 | const websocketServer = createServer((request, response) => { 51 | response.writeHead(404); 52 | response.end(); 53 | }); 54 | 55 | websocketServer.listen(5000, () => winston.info('Listening on port ' + 5000)); 56 | 57 | const subscriptionServer = SubscriptionServer.create( 58 | { 59 | schema: executableSchema, 60 | execute: execute, 61 | subscribe: subscribe 62 | }, 63 | { 64 | server: websocketServer, 65 | path: '/gql-ws', 66 | }, 67 | ); 68 | 69 | this.app.use('/', express.static(path.join(__dirname, '../public'))); 70 | // all other routes are handled by Angular 71 | this.app.get('/*', function (req, res) { 72 | res.sendFile(path.join(__dirname, '../public/index.html')); 73 | }); 74 | 75 | const consumer = new KafkaTweetConsumer(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /webapp/server/src/kafka/flight-info-avro.mapper.ts: -------------------------------------------------------------------------------- 1 | import { IFlightInfo } from '../db'; 2 | import { FlightInfoAvro } from './flight-info-avro'; 3 | 4 | export class FlightInfoAvroMapper { 5 | 6 | public static toFlightInfoAvro(flightInfo: IFlightInfo): FlightInfoAvro { 7 | let flightInfoAvro: FlightInfoAvro; 8 | if (flightInfo) { 9 | flightInfoAvro = new FlightInfoAvro(); 10 | flightInfoAvro.arrivalDate = flightInfo.arrivalDate; 11 | flightInfoAvro.arrivingId = flightInfo.arrivingId; 12 | flightInfoAvro.cabinClass = flightInfo.cabinClass; 13 | flightInfoAvro.departingId = flightInfo.departingId; 14 | flightInfoAvro.departureDate = flightInfo.departureDate; 15 | flightInfoAvro.ipAddress = undefined; 16 | flightInfoAvro.latitude = undefined; 17 | flightInfoAvro.longitude = undefined; 18 | flightInfoAvro.passengerNumber = flightInfo.passengerNumber; 19 | flightInfoAvro.tripType = flightInfo.tripType; 20 | flightInfoAvro.eventTime = new Date().getTime(); 21 | } 22 | return flightInfoAvro; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /webapp/server/src/kafka/flight-info-avro.ts: -------------------------------------------------------------------------------- 1 | 2 | export class FlightInfoAvro { 3 | departingId: string; 4 | ipAddress: string; 5 | latitude: string; 6 | longitude: string; 7 | arrivingId: string; 8 | tripType: string; 9 | departureDate: string; 10 | arrivalDate: string; 11 | passengerNumber: number; 12 | cabinClass: string; 13 | eventTime: number; 14 | } 15 | -------------------------------------------------------------------------------- /webapp/server/src/kafka/flight-info.producer.ts: -------------------------------------------------------------------------------- 1 | import { Producer, KeyedMessage, KafkaClient } from 'kafka-node'; 2 | import { Type as AvroType } from 'avsc/lib'; 3 | import * as winston from 'winston'; 4 | import { FlightInfoAvro } from './flight-info-avro'; 5 | import { FlightInfoAvroMapper } from './flight-info-avro.mapper'; 6 | import { IFlightInfo } from '../db'; 7 | 8 | export class KafkaFlightInfoProducer { 9 | 10 | private client: KafkaClient; 11 | private producer: Producer; 12 | private topic = 'flightInfoTopic'; 13 | 14 | constructor() { 15 | this.client = new KafkaClient({ kafkaHost: 'kafka:9092' }); 16 | this.client.on('connect', () => { 17 | winston.info('Flight-Info Kafka Client connected to Kafka'); 18 | }); 19 | this.client.on('error', (error) => { 20 | winston.error('Flight-Info Kafka Client - error > ', error); 21 | }); 22 | 23 | this.producer = new Producer(this.client, { requireAcks: 1 }); 24 | this.producer.on('error', (error) => { 25 | winston.error('Flight-Info Kafka Producer - error > ', error); 26 | }); 27 | this.producer.on('ready', () => { 28 | winston.info('Flight-Info Kafka Producer ready'); 29 | }); 30 | } 31 | 32 | sendFlightInfo(flightInfo: IFlightInfo) { 33 | const schemaType = AvroType.forSchema({ 34 | type: 'record', 35 | name: 'flightInfo', 36 | fields: [ 37 | { name: 'departingId', type: 'string' }, 38 | { name: 'arrivingId', type: 'string' }, 39 | { name: 'tripType', type: { type: 'enum', name: 'TripType', symbols: ['ONE_WAY', 'ROUND_TRIP'] } }, 40 | { name: 'departureDate', type: 'string' }, 41 | { name: 'arrivalDate', type: 'string' }, 42 | { name: 'passengerNumber', type: 'int' }, 43 | { name: 'cabinClass', type: { type: 'enum', name: 'CabinClass', symbols: ['ECONOMY', 'PRENIUM', 'BUSINESS'] } } 44 | ] 45 | }); 46 | 47 | const flightInfoAvro: FlightInfoAvro = FlightInfoAvroMapper.toFlightInfoAvro(flightInfo); 48 | // {\"departingId\":\"test\", \"arrivingId\":\"test\", \"tripType\":\"ONE_WAY\", \"departureDate\":\"tesss\", \"arrivalDate\":\"tezzz\", \"passengerNumber\":3, \"cabinClass\":\"ECONOMY\"}"); 49 | const buffer = schemaType.toBuffer(flightInfoAvro); 50 | const keyedMessage = new KeyedMessage('flightInfo', buffer); 51 | console.log('keyedMessage is >' + buffer); 52 | this.producer.send([ 53 | { topic: this.topic, partition: 0, messages: keyedMessage } 54 | ], (error, result) => { 55 | if (error) { 56 | winston.error('Flight-Info Kafka Producer - Message was not sent to consumers > ', error); 57 | } 58 | if (result) { 59 | winston.info('Flight-Info Kafka Producer - Message sent to consumers > ', result); 60 | } 61 | }); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /webapp/server/src/kafka/index.ts: -------------------------------------------------------------------------------- 1 | export * from './flight-info.producer'; 2 | export * from './tweet.consumer'; 3 | -------------------------------------------------------------------------------- /webapp/server/src/kafka/tweet.consumer.ts: -------------------------------------------------------------------------------- 1 | import { Consumer, KeyedMessage, KafkaClient } from 'kafka-node'; 2 | import { Type as AvroType } from 'avsc/lib'; 3 | import * as winston from 'winston'; 4 | import { pubsub } from '../schema'; 5 | import { ITweet } from '../db'; 6 | 7 | export class KafkaTweetConsumer { 8 | 9 | private client: KafkaClient; 10 | private consumer: Consumer; 11 | private topic = 'tweetsTopic'; 12 | 13 | constructor() { 14 | this.client = new KafkaClient({ kafkaHost: 'kafka:9092' }); 15 | this.client.on('connect', () => { 16 | winston.info('Tweet Kafka Client connected to Kafka'); 17 | }); 18 | this.client.on('error', (error) => { 19 | winston.error('Tweet Kafka Client - error > ', error); 20 | }); 21 | 22 | this.consumer = new Consumer(this.client, 23 | [ 24 | { topic: this.topic, partition: 0 } 25 | ], 26 | { 27 | autoCommit: false 28 | }); 29 | 30 | this.consumer.on('error', (error) => { 31 | winston.error('Tweet Kafka Consumer - error > ', error); 32 | }); 33 | this.consumer.on('message', (message: any) => { 34 | this.consumeTweets(message); 35 | }); 36 | } 37 | 38 | async consumeTweets(message) { 39 | let tweets = message.value as Array; 40 | pubsub.publish('tweets', { getTweets: tweets }); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /webapp/server/src/schema/index.ts: -------------------------------------------------------------------------------- 1 | export * from './resolvers/resolvers'; 2 | export * from './types/types'; 3 | -------------------------------------------------------------------------------- /webapp/server/src/schema/resolvers/resolvers.ts: -------------------------------------------------------------------------------- 1 | import { IFlightInfo } from '../../db'; 2 | import { KafkaFlightInfoProducer } from '../../kafka'; 3 | import { Airport } from '../../db/models/airport'; 4 | import { PubSub, withFilter } from 'graphql-subscriptions'; 5 | 6 | const producer = new KafkaFlightInfoProducer(); 7 | export const pubsub = new PubSub(); 8 | 9 | export const resolvers = { 10 | Query: { 11 | fetchAirports: async (_, { airportToSearch, airportId }) => { 12 | const airports = await Airport.findAirports(airportToSearch, airportId); 13 | return airports; 14 | } 15 | }, 16 | Mutation: { 17 | sendFlightInfo: async (_, { flightInfo }) => { 18 | producer.sendFlightInfo(flightInfo as IFlightInfo); 19 | return flightInfo; 20 | } 21 | }, 22 | Subscription: { 23 | getTweets: { 24 | subscribe: () => pubsub.asyncIterator('tweets') 25 | } 26 | } 27 | }; 28 | -------------------------------------------------------------------------------- /webapp/server/src/schema/types/types.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | export const typeDefs = ` 3 | 4 | scalar Date 5 | 6 | type FlightInfoType { 7 | departingId: String 8 | arrivingId: String 9 | tripType: String 10 | departureDate: String 11 | arrivalDate: String 12 | passengerNumber: Int 13 | cabinClass: String 14 | } 15 | 16 | input FlightInfoInput { 17 | departingId: String 18 | arrivingId: String 19 | tripType: String 20 | departureDate: String 21 | arrivalDate: String 22 | passengerNumber: Int 23 | cabinClass: String 24 | } 25 | 26 | type Airport { 27 | AirportID: String 28 | City: String 29 | Country: String 30 | Name: String 31 | destinations: [String] 32 | } 33 | 34 | type Tweet { 35 | id: Int 36 | } 37 | 38 | type Query { 39 | fetchAirports(airportToSearch: String, airportId: String): [Airport] 40 | } 41 | type Mutation { 42 | sendFlightInfo(flightInfo: FlightInfoInput): FlightInfoType 43 | } 44 | type Subscription { 45 | getTweets: [Tweet] 46 | } 47 | 48 | schema { 49 | query: Query, 50 | mutation : Mutation, 51 | subscription : Subscription 52 | } 53 | `; -------------------------------------------------------------------------------- /webapp/server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "outDir": "./dist/", 5 | "sourceMap": true, 6 | "declaration": false, 7 | "allowJs": true, 8 | "moduleResolution": "node", 9 | "emitDecoratorMetadata": true, 10 | "experimentalDecorators": true, 11 | "allowSyntheticDefaultImports": true, 12 | "target": "es5", 13 | "typeRoots": [ 14 | "node_modules/@types" 15 | ], 16 | "lib": [ 17 | "es2016", 18 | "esnext.asynciterable", 19 | "dom" 20 | ] 21 | }, 22 | "exclude": [ 23 | "node_modules", 24 | "dist" 25 | ] 26 | } -------------------------------------------------------------------------------- /webapp/server/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "member-access": false, 4 | "member-ordering": [ 5 | true, 6 | { 7 | "order": [ 8 | "public-static-field", 9 | "public-instance-field", 10 | "private-static-field", 11 | "private-instance-field", 12 | "public-constructor", 13 | "private-constructor", 14 | "public-instance-method", 15 | "protected-instance-method", 16 | "private-instance-method" 17 | ] 18 | } 19 | ], 20 | "no-any": false, 21 | "no-inferrable-types": false, 22 | "no-internal-module": true, 23 | "no-var-requires": false, 24 | "typedef": false, 25 | "typedef-whitespace": [ 26 | true, 27 | { 28 | "call-signature": "nospace", 29 | "index-signature": "nospace", 30 | "parameter": "nospace", 31 | "property-declaration": "nospace", 32 | "variable-declaration": "nospace" 33 | }, 34 | { 35 | "call-signature": "space", 36 | "index-signature": "space", 37 | "parameter": "space", 38 | "property-declaration": "space", 39 | "variable-declaration": "space" 40 | } 41 | ], 42 | 43 | "ban": false, 44 | "curly": false, 45 | "forin": true, 46 | "label-position": true, 47 | "no-arg": true, 48 | "no-bitwise": true, 49 | "no-conditional-assignment": true, 50 | "no-console": [ 51 | true, 52 | "debug", 53 | "info", 54 | "time", 55 | "timeEnd", 56 | "trace" 57 | ], 58 | "no-construct": true, 59 | "no-debugger": true, 60 | "no-duplicate-variable": true, 61 | "no-empty": false, 62 | "no-eval": true, 63 | "no-null-keyword": false, 64 | "no-shadowed-variable": true, 65 | "no-string-literal": false, 66 | "no-switch-case-fall-through": true, 67 | "no-unused-expression": true, 68 | "no-use-before-declare": true, 69 | "no-var-keyword": true, 70 | "radix": true, 71 | "switch-default": true, 72 | "triple-equals": [ 73 | true, 74 | "allow-null-check" 75 | ], 76 | 77 | "eofline": true, 78 | "indent": [ 79 | true, 80 | "spaces" 81 | ], 82 | "max-line-length": [ 83 | true, 84 | 200 85 | ], 86 | "no-require-imports": false, 87 | "no-trailing-whitespace": true, 88 | "object-literal-sort-keys": false, 89 | "trailing-comma": [ 90 | true, 91 | { 92 | "multiline": false, 93 | "singleline": "never" 94 | } 95 | ], 96 | 97 | "align": false, 98 | "class-name": true, 99 | "comment-format": [ 100 | true, 101 | "check-space" 102 | ], 103 | "interface-name": false, 104 | "jsdoc-format": true, 105 | "no-consecutive-blank-lines": false, 106 | "one-line": [ 107 | true, 108 | "check-open-brace", 109 | "check-catch", 110 | "check-else", 111 | "check-whitespace" 112 | ], 113 | "quotemark": [ 114 | true, 115 | "single", 116 | "avoid-escape" 117 | ], 118 | "semicolon": true, 119 | "variable-name": [ 120 | true, 121 | "check-format", 122 | "allow-leading-underscore", 123 | "ban-keywords", 124 | "allow-pascal-case" 125 | ], 126 | "whitespace": [ 127 | true, 128 | "check-branch", 129 | "check-decl", 130 | "check-operator", 131 | "check-separator", 132 | "check-type" 133 | ] 134 | } 135 | } --------------------------------------------------------------------------------