├── .gitignore ├── LICENSE.txt ├── NiFiDeploy.groovy ├── README.md ├── TemplateInspector.groovy ├── assets └── HelloNiFi_screenshot.png ├── nifi-deploy.yml └── test-yaml.groovy /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NiFiDeploy.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonBuilder 2 | import groovyx.net.http.RESTClient 3 | import org.apache.http.entity.mime.MultipartEntity 4 | import org.apache.http.entity.mime.content.StringBody 5 | import org.yaml.snakeyaml.Yaml 6 | 7 | import static groovy.json.JsonOutput.prettyPrint 8 | import static groovy.json.JsonOutput.toJson 9 | import static groovyx.net.http.ContentType.JSON 10 | import static groovyx.net.http.ContentType.URLENC 11 | import static groovyx.net.http.Method.POST 12 | 13 | 14 | 15 | @Grab(group='org.codehaus.groovy.modules.http-builder', 16 | module='http-builder', 17 | version='0.7.1') 18 | @Grab(group='org.yaml', 19 | module='snakeyaml', 20 | version='1.17') 21 | @Grab(group='org.apache.httpcomponents', 22 | module='httpmime', 23 | version='4.2.1') 24 | 25 | // see actual script content at the bottom of the text, 26 | // after every implementation method. Groovy compiler likes these much better 27 | 28 | 29 | def cli = new CliBuilder(usage: 'groovy NiFiDeploy.groovy [options]', 30 | header: 'Options:') 31 | cli.with { 32 | f longOpt: 'file', 33 | 'Deployment specification file in a YAML format', 34 | args:1, argName:'name', type:String.class 35 | h longOpt: 'help', 'This usage screen' 36 | d longOpt: 'debug', 'Debug underlying HTTP wire communication' 37 | n longOpt: 'nifi-api', 'NiFi REST API (override), e.g. http://example.com:9090', 38 | args:1, argName:'http://host:port', type:String.class 39 | t longOpt: 'template', 'Template URI (override)', 40 | args:1, argName:'uri', type:String.class 41 | c longOpt: 'client-id', 'Client ID for API calls, any unique string (override)', 42 | args:1, argName:'id', type:String.class 43 | } 44 | 45 | def opts = cli.parse(args) 46 | if (!opts) { return } 47 | if (opts.help) { 48 | cli.usage() 49 | return 50 | } 51 | 52 | 53 | def deploymentSpec 54 | if (opts.file) { 55 | deploymentSpec = opts.file 56 | } else { 57 | println "ERROR: Missing a file argument\n" 58 | cli.usage() 59 | System.exit(-1) 60 | } 61 | 62 | if (opts.debug) { 63 | System.setProperty('org.apache.commons.logging.Log', 'org.apache.commons.logging.impl.SimpleLog') 64 | System.setProperty('org.apache.commons.logging.simplelog.showdatetime', 'true') 65 | System.setProperty('org.apache.commons.logging.simplelog.log.org.apache.http', 'DEBUG') 66 | } 67 | 68 | // implementation methods below 69 | 70 | def handleUndeploy() { 71 | if (!conf.nifi.undeploy) { 72 | return 73 | } 74 | 75 | // stop & remove controller services 76 | // stop & remove process groups 77 | // delete templates 78 | 79 | // TODO not optimal (would rather save all CS in state), but ok for now 80 | conf.nifi?.undeploy?.controllerServices?.each { csName -> 81 | print "Undeploying Controller Service: $csName" 82 | def cs = lookupControllerService(csName) 83 | if (cs) { 84 | println " ($cs.id)" 85 | stopControllerService(cs.id) 86 | updateToLatestRevision() 87 | def resp = nifi.delete( 88 | path: "controller/controller-services/NODE/$cs.id", 89 | query: [ 90 | clientId: client, 91 | version: currentRevision 92 | ] 93 | ) 94 | assert resp.status == 200 95 | } else { 96 | println '' 97 | } 98 | } 99 | 100 | conf.nifi?.undeploy?.processGroups?.each { pgName -> 101 | println "Undeploying Process Group: $pgName" 102 | def pg = processGroups.findAll { it.name == pgName } 103 | if (pg.isEmpty()) { 104 | println "[WARN] No such process group found in NiFi" 105 | return 106 | } 107 | assert pg.size() == 1 : "Ambiguous process group name" 108 | 109 | def id = pg[0].id 110 | 111 | stopProcessGroup(id) 112 | 113 | // now delete it 114 | updateToLatestRevision() 115 | resp = nifi.delete( 116 | path: "controller/process-groups/root/process-group-references/$id", 117 | query: [ 118 | clientId: client, 119 | version: currentRevision 120 | ] 121 | ) 122 | assert resp.status == 200 123 | } 124 | 125 | conf.nifi?.undeploy?.templates?.each { tName -> 126 | println "Deleting template: $tName" 127 | def t = lookupTemplate(tName) 128 | if (t) { 129 | updateToLatestRevision() 130 | def resp = nifi.delete( 131 | path: "controller/templates/$t.id", 132 | query: [ 133 | clientId: client, 134 | version: currentRevision 135 | ] 136 | ) 137 | assert resp.status == 200 138 | } 139 | } 140 | } 141 | 142 | /** 143 | Returns a json-backed controller service structure from NiFi 144 | */ 145 | def lookupControllerService(String name) { 146 | def resp = nifi.get( 147 | path: 'controller/controller-services/NODE' 148 | ) 149 | assert resp.status == 200 150 | 151 | if (resp.data.controllerServices.name.grep(name).isEmpty()) { 152 | return 153 | } 154 | 155 | assert resp.data.controllerServices.name.grep(name).size() == 1 : 156 | "Multiple controller services found named '$name'" 157 | // println prettyPrint(toJson(resp.data)) 158 | 159 | def cs = resp.data.controllerServices.find { it.name == name } 160 | assert cs != null 161 | 162 | return cs 163 | } 164 | 165 | /** 166 | Returns a json-backed template structure from NiFi. Null if not found. 167 | */ 168 | def lookupTemplate(String name) { 169 | def resp = nifi.get( 170 | path: 'controller/templates' 171 | ) 172 | assert resp.status == 200 173 | 174 | if (resp.data.templates.name.grep(name).isEmpty()) { 175 | return null 176 | } 177 | 178 | assert resp.data.templates.name.grep(name).size() == 1 : 179 | "Multiple templates found named '$name'" 180 | // println prettyPrint(toJson(resp.data)) 181 | 182 | def t = resp.data.templates.find { it.name == name } 183 | assert t != null 184 | 185 | return t 186 | } 187 | 188 | def importTemplate(String templateUri) { 189 | println "Loading template from URI: $templateUri" 190 | def templateBody = templateUri.toURL().text 191 | 192 | nifi.request(POST) { request -> 193 | uri.path = '/nifi-api/controller/templates' 194 | 195 | requestContentType = 'multipart/form-data' 196 | MultipartEntity entity = new MultipartEntity() 197 | entity.addPart("template", new StringBody(templateBody)) 198 | request.entity = entity 199 | 200 | response.success = { resp, xml -> 201 | switch (resp.statusLine.statusCode) { 202 | case 200: 203 | println "[WARN] Template already exists, skipping for now" 204 | // TODO delete template, CS and, maybe a PG 205 | break 206 | case 201: 207 | // grab the trailing UUID part of the location URL header 208 | def location = resp.headers.Location 209 | templateId = location[++location.lastIndexOf('/')..-1] 210 | println "Template successfully imported into NiFi. ID: $templateId" 211 | updateToLatestRevision() // ready to make further changes 212 | break 213 | default: 214 | throw new Exception("Error importing template") 215 | break 216 | } 217 | } 218 | } 219 | } 220 | 221 | def instantiateTemplate(String id) { 222 | updateToLatestRevision() 223 | def resp = nifi.post ( 224 | path: 'controller/process-groups/root/template-instance', 225 | body: [ 226 | templateId: id, 227 | // TODO add slight randomization to the XY to avoid hiding PG behind each other 228 | originX: 100, 229 | originY: 100, 230 | version: currentRevision 231 | ], 232 | requestContentType: URLENC 233 | ) 234 | 235 | assert resp.status == 201 236 | } 237 | 238 | def loadProcessGroups() { 239 | println "Loading Process Groups from NiFi" 240 | def resp = nifi.get( 241 | path: 'controller/process-groups/root/process-group-references' 242 | ) 243 | assert resp.status == 200 244 | // println resp.data 245 | processGroups = resp.data.processGroups 246 | } 247 | 248 | /** 249 | - read the desired pgConfig 250 | - locate the processor according to the nesting structure in YAML 251 | (intentionally not using 'search') to pick up a specific PG->Proc 252 | - update via a partial PUT constructed from the pgConfig 253 | */ 254 | def handleProcessGroup(Map.Entry pgConfig) { 255 | //println pgConfig 256 | 257 | if (!pgConfig.value) { 258 | return 259 | } 260 | 261 | updateToLatestRevision() 262 | 263 | def pgName = pgConfig.key 264 | def pg = processGroups.find { it.name == pgName } 265 | assert pg : "Processing Group '$pgName' not found in this instance, check your deployment config?" 266 | def pgId = pg.id 267 | 268 | println "Process Group: $pgConfig.key ($pgId)" 269 | //println pgConfig 270 | 271 | if (!pg.comments) { 272 | updateToLatestRevision() 273 | // update process group comments with a deployment timestamp 274 | def builder = new JsonBuilder() 275 | builder { 276 | revision { 277 | clientId client 278 | version currentRevision 279 | } 280 | processGroup { 281 | id pgId 282 | comments defaultComment 283 | } 284 | } 285 | 286 | // println builder.toPrettyString() 287 | 288 | updateToLatestRevision() 289 | 290 | resp = nifi.put ( 291 | path: "controller/process-groups/$pgId", 292 | body: builder.toPrettyString(), 293 | requestContentType: JSON 294 | ) 295 | assert resp.status == 200 296 | } 297 | 298 | // load processors in this group 299 | resp = nifi.get(path: "controller/process-groups/$pgId/processors") 300 | assert resp.status == 200 301 | 302 | // construct a quick map of "procName -> [id, fullUri]" 303 | def processors = resp.data.processors.collectEntries { 304 | [(it.name): [it.id, it.uri, it.comments]] 305 | } 306 | 307 | pgConfig.value.processors.each { proc -> 308 | // check for any duplicate processors in the remote NiFi instance 309 | def result = processors.findAll { remote -> remote.key == proc.key } 310 | assert result.entrySet().size() == 1 : "Ambiguous processor name '$proc.key'" 311 | 312 | def procId = processors[proc.key][0] 313 | def existingComments = processors[proc.key][2] 314 | 315 | println "Stopping Processor '$proc.key' ($procId)" 316 | stopProcessor(pgId, procId) 317 | 318 | def procProps = proc.value.config.entrySet() 319 | 320 | println "Applying processor configuration" 321 | def builder = new JsonBuilder() 322 | builder { 323 | revision { 324 | clientId client 325 | version currentRevision 326 | } 327 | processor { 328 | id procId 329 | config { 330 | comments existingComments ?: defaultComment 331 | properties { 332 | procProps.each { p -> 333 | // check if it's a ${referenceToControllerServiceName} 334 | def ref = p.value =~ /\$\{(.*)}/ 335 | if (ref) { 336 | def name = ref[0][1] // grab the first capture group (nested inside ArrayList) 337 | // lookup the CS by name and get the newly generated ID instead of the one in a template 338 | def newCS = lookupControllerService(name) 339 | assert newCS : "Couldn't locate Controller Service with the name: $name" 340 | "$p.key" newCS.id 341 | } else { 342 | "$p.key" p.value 343 | } 344 | } 345 | } 346 | } 347 | } 348 | } 349 | 350 | // println builder.toPrettyString() 351 | 352 | updateToLatestRevision() 353 | 354 | resp = nifi.put ( 355 | path: "controller/process-groups/$pgId/processors/$procId", 356 | body: builder.toPrettyString(), 357 | requestContentType: JSON 358 | ) 359 | assert resp.status == 200 360 | 361 | // check if pgConfig tells us to start this processor 362 | if (proc.value.state == 'RUNNING') { 363 | println "Will start it up next" 364 | startProcessor(pgId, procId) 365 | } else { 366 | println "Processor wasn't configured to be running, not starting it up" 367 | } 368 | } 369 | 370 | println "Starting Process Group: $pgName ($pgId)" 371 | startProcessGroup(pgId) 372 | } 373 | 374 | def handleControllerService(Map.Entry cfg) { 375 | //println config 376 | def name = cfg.key 377 | println "Looking up a controller service '$name'" 378 | 379 | def cs = lookupControllerService(name) 380 | updateToLatestRevision() 381 | 382 | println "Found the controller service '$cs.name'. Current state is ${cs.state}." 383 | 384 | if (cs.state == cfg.value.state) { 385 | println "$cs.name is already in a requested state: '$cs.state'" 386 | return 387 | } 388 | 389 | if (cfg.value?.config) { 390 | println "Applying controller service '$cs.name' configuration" 391 | def builder = new JsonBuilder() 392 | builder { 393 | revision { 394 | clientId client 395 | version currentRevision 396 | } 397 | controllerService { 398 | id cs.id 399 | comments cs.comments ?: defaultComment 400 | properties { 401 | cfg.value.config.each { p -> 402 | "$p.key" p.value 403 | } 404 | } 405 | } 406 | } 407 | 408 | 409 | // println builder.toPrettyString() 410 | 411 | updateToLatestRevision() 412 | 413 | resp = nifi.put ( 414 | path: "controller/controller-services/NODE/$cs.id", 415 | body: builder.toPrettyString(), 416 | requestContentType: JSON 417 | ) 418 | assert resp.status == 200 419 | } 420 | 421 | 422 | println "Enabling $cs.name (${cs.id})" 423 | startControllerService(cs.id) 424 | } 425 | 426 | def updateToLatestRevision() { 427 | def resp = nifi.get( 428 | path: 'controller/revision' 429 | ) 430 | assert resp.status == 200 431 | currentRevision = resp.data.revision.version 432 | } 433 | 434 | def stopProcessor(processGroupId, processorId) { 435 | _changeProcessorState(processGroupId, processorId, false) 436 | } 437 | 438 | def startProcessor(processGroupId, processorId) { 439 | _changeProcessorState(processGroupId, processorId, true) 440 | } 441 | 442 | private _changeProcessorState(processGroupId, processorId, boolean running) { 443 | updateToLatestRevision() 444 | def builder = new JsonBuilder() 445 | builder { 446 | revision { 447 | clientId client 448 | version currentRevision 449 | } 450 | processor { 451 | id processorId 452 | state running ? 'RUNNING' : 'STOPPED' 453 | } 454 | } 455 | 456 | //println builder.toPrettyString() 457 | resp = nifi.put ( 458 | path: "controller/process-groups/$processGroupId/processors/$processorId", 459 | body: builder.toPrettyString(), 460 | requestContentType: JSON 461 | ) 462 | assert resp.status == 200 463 | currentRevision = resp.data.revision.version 464 | } 465 | 466 | def startProcessGroup(pgId) { 467 | _changeProcessGroupState(pgId, true) 468 | } 469 | 470 | def stopProcessGroup(pgId) { 471 | print "Waiting for a Process Group to stop: $pgId " 472 | _changeProcessGroupState(pgId, false) 473 | 474 | 475 | int maxWait = 1000 * 30 // up to X seconds 476 | def resp = nifi.get(path: "controller/process-groups/$pgId/status") 477 | assert resp.status == 200 478 | long start = System.currentTimeMillis() 479 | 480 | // keep polling till active threads shut down, but no more than maxWait time 481 | while ((System.currentTimeMillis() < (start + maxWait)) && 482 | resp.data.processGroupStatus.activeThreadCount > 0) { 483 | sleep(1000) 484 | resp = nifi.get(path: "controller/process-groups/$pgId/status") 485 | assert resp.status == 200 486 | print '.' 487 | } 488 | if (resp.data.processGroupStatus.activeThreadCount == 0) { 489 | println 'Done' 490 | } else { 491 | println "Failed to stop the processing group, request timed out after ${maxWait/1000} seconds" 492 | System.exit(-1) 493 | } 494 | } 495 | 496 | private _changeProcessGroupState(pgId, boolean running) { 497 | updateToLatestRevision() 498 | def resp = nifi.put( 499 | path: "controller/process-groups/root/process-group-references/$pgId", 500 | body: [ 501 | running: running, 502 | client: client, 503 | version: currentRevision 504 | ], 505 | requestContentType: URLENC 506 | ) 507 | assert resp.status == 200 508 | } 509 | 510 | def stopControllerService(csId) { 511 | _changeControllerServiceState(csId, false) 512 | } 513 | 514 | def startControllerService(csId) { 515 | _changeControllerServiceState(csId, true) 516 | } 517 | 518 | private _changeControllerServiceState(csId, boolean enabled) { 519 | updateToLatestRevision() 520 | 521 | if (!enabled) { 522 | // gotta stop all CS references first when disabling a CS 523 | def resp = nifi.put ( 524 | path: "controller/controller-services/node/$csId/references", 525 | body: [ 526 | clientId: client, 527 | version: currentRevision, 528 | state: 'STOPPED' 529 | ], 530 | requestContentType: URLENC 531 | ) 532 | assert resp.status == 200 533 | } 534 | 535 | def builder = new JsonBuilder() 536 | builder { 537 | revision { 538 | clientId client 539 | version currentRevision 540 | } 541 | controllerService { 542 | id csId 543 | state enabled ? 'ENABLED' : 'DISABLED' 544 | } 545 | } 546 | 547 | // println builder.toPrettyString() 548 | 549 | resp = nifi.put( 550 | path: "controller/controller-services/NODE/$csId", 551 | body: builder.toPrettyString(), 552 | requestContentType: JSON 553 | ) 554 | assert resp.status == 200 555 | } 556 | 557 | // script flow below 558 | 559 | conf = new Yaml().load(new File(deploymentSpec).text) 560 | assert conf 561 | 562 | def nifiHostPort = opts.'nifi-api' ?: conf.nifi.url 563 | if (!nifiHostPort) { 564 | println 'Please specify a NiFi instance URL in the deployment spec file or via CLI' 565 | System.exit(-1) 566 | } 567 | nifiHostPort = nifiHostPort.endsWith('/') ? nifiHostPort[0..-2] : nifiHostPort 568 | assert nifiHostPort : "No NiFI REST API endpoint provided" 569 | 570 | nifi = new RESTClient("$nifiHostPort/nifi-api/") 571 | nifi.handler.failure = { resp, data -> 572 | resp.setData(data?.text) 573 | println "[ERROR] HTTP call failed. Status code: $resp.statusLine: $resp.data" 574 | // fail gracefully with a more sensible groovy stacktrace 575 | assert null : "Terminated script execution" 576 | } 577 | 578 | 579 | client = opts.'client-id' ?: conf.nifi.clientId 580 | assert client : 'Client ID must be provided' 581 | 582 | thisHost = InetAddress.localHost 583 | defaultComment = "Last updated by '$client' on ${new Date()} from $thisHost" 584 | 585 | currentRevision = -1 // used for optimistic concurrency throughout the REST API 586 | 587 | processGroups = null 588 | loadProcessGroups() 589 | 590 | handleUndeploy() 591 | 592 | templateId = null // will be assigned on import into NiFi 593 | 594 | def tUri = opts.template ?: conf.nifi.templateUri 595 | assert tUri : "Template URI not provided" 596 | importTemplate(tUri) 597 | instantiateTemplate(templateId) 598 | 599 | // reload after template instantiation 600 | loadProcessGroups() 601 | 602 | println "Configuring Controller Services" 603 | 604 | // controller services are dependencies of processors, 605 | // configure them first 606 | conf.controllerServices.each { handleControllerService(it) } 607 | 608 | println "Configuring Process Groups and Processors" 609 | conf.processGroups.each { handleProcessGroup(it) } 610 | 611 | println 'All Done.' 612 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Note: this was originally created for NiFi 0.x. REST APIs and concepts have changed significantly in NiFi 1.x 2 | # NiFi Deployment Automation 3 | 4 | - Deploy & configure NiFi templates with a touch of a button (or, rather, a single command) 5 | - Specify a URI to fetch a Template from - meaning it can be a local file system, remote HTTP URL, or any other exotic location for which you have a URLHandler installed 6 | - Describe NiFi state and configuration properties for things you want tweaked in a template. YAML format came out to be the cleanest and most usable option (YAML is a subset of JSON) 7 | - (Recommended) Tell NiFi what things are in your way and have them undeployed as part of the process. Good idea if one wants a deployment to be **idempotent**. 8 | 9 | # NiFi Template Inspector 10 | 11 | A utility to list available components and properties in a template. Also serves as a great starting point to customize a deployment. See https://github.com/aperepel/nifi-api-deploy/wiki 12 | 13 | ### Wait, what's a template? 14 | Template is a NiFi means to share and exchange re-usable parts of a flow. It can be trivial or very complex, works the same way all along. Documentation at https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#templates 15 | 16 | # 1-minute How-To 17 | ``` 18 | git clone https://github.com/aperepel/nifi-api-deploy.git 19 | cd nifi-api-deploy 20 | 21 | # edit nifi-deploy.yml and point nifi.url to your NiFi instance or cluster 22 | 23 | groovy NiFiDeploy.groovy --file nifi-deploy.yml 24 | ... 25 | # after deployment completes 26 | nifi-api-deploy ♨ > curl http://192.168.99.102:10000 27 | Dynamically Configured NiFi! 28 | 29 | # bonus item, see 'undeploy' in action 30 | groovy NiFiDeploy.groovy --file nifi-deploy.yml 31 | ``` 32 | 33 | 34 | 35 | When things finish one ends up with the following in NiFi: 36 | 37 | - `Hello_NiFi_Web_Service` template imported. See more here: https://cwiki.apache.org/confluence/display/NIFI/Example+Dataflow+Templates 38 | - Template's listen port and service return message reconfigured as per our deployment recipe 39 | - Template is instantiated and its `Processing Group` is added to the canvas 40 | - Things are started up and an HTTP endpoint is listening on port 10000 41 | 42 | ![Image of the Template Running](/assets/HelloNiFi_screenshot.png) 43 | 44 | # 5-minute Introduction 45 | 46 | The `nifi-deploy.yml` has several major sections: 47 | 48 | - Basics, like your NiFi address and where to get the template from 49 | - Undeploy instructions, for idempotent script runs 50 | - Controller Services to be instantiated 51 | - Processor configurations 52 | 53 | Best way to grasp things is to dissect the YAML file: 54 | ``` 55 | nifi: 56 | url: http://192.168.99.103:9091 57 | 58 | # when making changes via API, need a unique client ID, can be anything 59 | clientId: Deployment Script v1 60 | 61 | # Where to fetch the actual template XML data from 62 | # Escape complex URLs with quotes 63 | templateUri: "any file: , http://..., etc URL" 64 | 65 | # Tell NiFi we want some things removed to make way for this (re-) deployment 66 | undeploy: 67 | 68 | # Names of controller services to remove. Ignores any missing ones 69 | controllerServices: 70 | - StandardHttpContextMap 71 | - SomeOtherControllerService 72 | 73 | # Names of process groups to remove. These are in your template 74 | processGroups: 75 | - Hello NiFi Web Service 76 | 77 | # Template names to remove. Because we're updating with a new version 78 | templates: 79 | - Hello NiFi Web Service 80 | ``` 81 | 82 | 83 | Next, one describes what configuration changes need to be applied to the template in this deployment: 84 | ``` 85 | # Instantiate these controller services, our template uses them 86 | controllerServices: 87 | StandardHttpContextMap: 88 | state: ENABLED 89 | 90 | # Processors belong to process groups. 91 | # This way random ones won't be picked up (unlike a search api, 92 | # which returns every occurence) 93 | processGroups: 94 | 95 | # Empty in this case, as our template puts everything in a group 96 | root: ~ 97 | 98 | # Process group name from a template 99 | Hello NiFi Web Service: 100 | 101 | # processors we want to reconfigure from template defaults 102 | processors: 103 | 104 | # processor by name 105 | Receive request and data: 106 | state: RUNNING 107 | 108 | # These match the Properties tab in the processor UI 109 | config: 110 | Listening Port: 10000 111 | 112 | # another processor, but name is escaped with quotes 113 | "Update Request Body with a greeting!": 114 | config: 115 | Replacement Value: Dynamically Configured NiFi! 116 | 117 | ``` 118 | 119 | # Troubleshooting 120 | ### Proxy 121 | The script automatically downloads several dependencies from a Maven central repository (via Grape annotation). If you are behind a firewall, and can't reach that server directly, try adding these system properties on the command line: 122 | ``` 123 | groovy -Dhttp.proxyHost=myproxy.mycompany.com -Dhttp.proxyPort=3128 NiFiDeploy.groovy 124 | ``` 125 | See more at http://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html 126 | 127 | ### REST API Issues 128 | Start troubleshooting by enabling the HTTP debug option via the `--debug` 129 | -------------------------------------------------------------------------------- /TemplateInspector.groovy: -------------------------------------------------------------------------------- 1 | import org.yaml.snakeyaml.Yaml 2 | import org.yaml.snakeyaml.DumperOptions 3 | 4 | @Grab(group='org.yaml', module='snakeyaml', version='1.17') 5 | 6 | def cli = new CliBuilder(usage: 'groovy TemplateInspector.groovy [options]', 7 | header: 'Options:') 8 | cli.with { 9 | f longOpt: 'file', 10 | 'Template file to inspect (can be a URL). E.g. try this hello world template at https://goo.gl/M1vmvS', 11 | args:1, argName: 'template', type:String.class 12 | h longOpt: 'help', 'This usage screen' 13 | } 14 | 15 | def opts = cli.parse(args) 16 | if (!opts) { return } 17 | if (opts.help) { 18 | cli.usage() 19 | return 20 | } 21 | 22 | 23 | def templateUri 24 | if (opts.file) { 25 | templateUri = opts.file 26 | def scheme = new URI(templateUri).scheme 27 | if (!scheme) { 28 | // assume a local file 29 | templateUri = 'file:' + templateUri 30 | } 31 | } else { 32 | println "ERROR: Missing a file argument\n" 33 | cli.usage() 34 | System.exit(-1) 35 | } 36 | 37 | t = new XmlSlurper().parse(templateUri) 38 | 39 | // create a data structure 40 | y = [:] 41 | y.nifi = [:] 42 | y.nifi.templateUri = templateUri 43 | y.nifi.templateName = t.name.text() 44 | 45 | if (t.snippet.controllerServices.size() > 0) { 46 | y.controllerServices = [:] 47 | t.snippet.controllerServices.each { xCs -> 48 | def yC = y.controllerServices 49 | yC[xCs.name.text()] = [:] 50 | yC[xCs.name.text()].state = 'ENABLED' 51 | def xProps = xCs.properties?.entry 52 | if (xProps.size() > 0) { 53 | yC[xCs.name.text()].config = [:] 54 | xProps.each { xProp -> 55 | if (xProp.value.size() > 0) { 56 | yC[xCs.name.text()].config[xProp.key.text()] = xProp.value.text() 57 | } 58 | } 59 | } 60 | } 61 | } 62 | 63 | y.processGroups = [:] 64 | 65 | if (t.snippet.processors.size() > 0) { 66 | // special handling for root-level processors 67 | parseGroup(t.snippet) 68 | } 69 | 70 | t.snippet.processGroups.each { 71 | parseGroup(it) 72 | } 73 | 74 | def parseGroup(node) { 75 | def pgName = node?.name.text() 76 | if (!pgName) { 77 | pgName = 'root' 78 | } 79 | 80 | y.processGroups[pgName] = [:] 81 | y.processGroups[pgName].processors = [:] 82 | 83 | parseProcessors(pgName, node) 84 | } 85 | 86 | def parseProcessors(groupName, node) { 87 | def processors = node.contents.isEmpty() ? node.processors // root process group 88 | : node.contents.processors // regular process group 89 | processors.each { p -> 90 | y.processGroups[groupName].processors[p.name.text()] = [:] 91 | y.processGroups[groupName].processors[p.name.text()].config = [:] 92 | 93 | p.config.properties?.entry?.each { 94 | def c = y.processGroups[groupName].processors[p.name.text()].config 95 | // check if it's a UUID and try lookup the CS to get the name 96 | if (it.value.text() ==~ /[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}/) { 97 | def n = t.snippet.controllerServices.find { cs -> cs.id.text() == it.value.text() } 98 | assert !n.isEmpty() : "Couldn't resolve a Controller Service with ID: ${it.value.text()}" 99 | c[it.key.text()] = '\${' + n.name.text() + "}" 100 | } else if (it.value.size() > 0) { 101 | c[it.key.text()] = it.value.size() == 0 ? null : it.value.text() 102 | } 103 | } 104 | } 105 | } 106 | 107 | // serialize to yaml 108 | def yamlOpts = new DumperOptions() 109 | yamlOpts.defaultFlowStyle = DumperOptions.FlowStyle.BLOCK 110 | yamlOpts.prettyFlow = true 111 | println new Yaml(yamlOpts).dump(y) 112 | -------------------------------------------------------------------------------- /assets/HelloNiFi_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aperepel/nifi-api-deploy/ea1aae25115f043c0974809f109c8e2fbe2b3335/assets/HelloNiFi_screenshot.png -------------------------------------------------------------------------------- /nifi-deploy.yml: -------------------------------------------------------------------------------- 1 | nifi: 2 | url: http://192.168.99.103:9091 3 | clientId: Deployment Script v1 4 | templateUri: "https://cwiki.apache.org/confluence/download/attachments/57904847/Hello_NiFi_Web_Service.xml?version=1&modificationDate=1449369797000&api=v2" 5 | # templateUri: file:./Hello_nifi.xml 6 | undeploy: 7 | controllerServices: 8 | - StandardHttpContextMap 9 | - SomeOtherControllerService 10 | processGroups: 11 | - Hello NiFi Web Service 12 | templates: 13 | - Hello NiFi Web Service 14 | 15 | controllerServices: 16 | StandardHttpContextMap: 17 | state: ENABLED 18 | config: 19 | Maximum Outstanding Requests: 20 20 | 21 | processGroups: 22 | root: ~ 23 | 24 | Hello NiFi Web Service: 25 | processors: 26 | Receive request and data: 27 | state: RUNNING 28 | config: 29 | Listening Port: 10000 30 | 31 | "Update Request Body with a greeting!": 32 | config: 33 | Replacement Value: Dynamically Configured NiFi! 34 | -------------------------------------------------------------------------------- /test-yaml.groovy: -------------------------------------------------------------------------------- 1 | import org.yaml.snakeyaml.Yaml 2 | 3 | @Grab(group='org.yaml', module='snakeyaml', version='1.17') 4 | 5 | def config = new Yaml().load(new File('nifi-deploy.yml').text) 6 | assert config 7 | 8 | assert config.nifi.url == 'http://192.168.99.103:9091' 9 | assert config.nifi.templateUri == 'https://cwiki.apache.org/confluence/download/attachments/57904847/Hello_NiFi_Web_Service.xml?version=1&modificationDate=1449369797000&api=v2' 10 | assert 'ENABLED' == config.controllerServices.StandardHttpContextMap.state 11 | 12 | assert config.nifi.undeploy.controllerServices == ['StandardHttpContextMap', 'SomeOtherControllerService'] 13 | assert config.nifi.undeploy.processGroups.size() == 1 14 | assert config.nifi.undeploy.processGroups[0] == 'Hello NiFi Web Service' 15 | assert config.nifi.undeploy.templates.size() == 1 16 | assert config.nifi.undeploy.templates[0] == 'Hello NiFi Web Service' 17 | 18 | def cs = config.controllerServices.StandardHttpContextMap 19 | assert cs 20 | def csConfig = cs.config 21 | assert csConfig?.entrySet().size() == 1 22 | def r = csConfig.entrySet()[0] 23 | assert r.key == 'Maximum Outstanding Requests' 24 | assert r.value == 20 25 | 26 | 27 | assert config.processGroups.size() == 2 28 | 29 | def pg = config.processGroups['Hello NiFi Web Service'] 30 | assert pg.processors.entrySet().size() == 2 31 | 32 | def p = pg.processors.entrySet()[0] 33 | assert p.key == 'Receive request and data' 34 | assert p.value.state == 'RUNNING' 35 | 36 | def c = p.value.config 37 | assert c 38 | 39 | assert c.'Listening Port' == 10000 40 | 41 | def s = 'Location: http://192.168.99.103:9091/nifi-api/controller/templates/1c6bd4ca-c934-36fd-98cd-397d0dc0f27d' 42 | assert '1c6bd4ca-c934-36fd-98cd-397d0dc0f27d' == s[++s.lastIndexOf('/')..-1] // grabs the trailing UUID only 43 | --------------------------------------------------------------------------------