├── .gitattributes ├── .github └── FUNDING.yml ├── .gitignore ├── README.md ├── build.gradle ├── collect-stream-logs ├── README.md ├── collect-stream-logs-flow.xml ├── dashboard │ ├── draw.html │ ├── log.html │ ├── reqscanvas.js │ └── testEB.html ├── data │ ├── in │ │ └── example.log │ └── out │ │ └── .gitkeep ├── log-generator │ ├── build.gradle │ └── src │ │ └── main │ │ ├── groovy │ │ └── com.crossbusiness.loggen │ │ │ ├── AccessLogGenerator.groovy │ │ │ └── AppLogGenerator.groovy │ │ ├── java │ │ └── .gitkeep │ │ └── resources │ │ ├── example.log │ │ ├── logback-access.xml │ │ └── logback.xml ├── logs-demo.png └── logs-flow.png ├── csv-to-json ├── README.md └── csv-to-json-flow.xml ├── decompression ├── README.md └── decompression-circular-flow.xml ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── http-get-route ├── README.md └── simple-httpget-route-flow.xml ├── invoke-http-route ├── README.md └── invokeHttp-and-route-original-on-status-flow.xml ├── iot-activity-tracker ├── README.md ├── dashboard │ └── heartrate.html ├── iot-demo.png ├── iot-flow.png └── iot-flow.xml ├── oltp-cdc-olap ├── Dockerfile ├── README.md ├── cdc-architecture.jpg ├── cdc-flow.png ├── cdc-flow.xml ├── kafka │ ├── README.md │ ├── kafka │ │ └── .gitignore │ ├── server.properties │ ├── zookeeper.properties │ └── zookeeper │ │ └── .gitignore ├── maxwell │ ├── .gitignore │ └── config.properties └── mysql │ ├── README.md │ ├── data │ └── .gitignore │ └── my.cnf ├── retry ├── README.md └── retry-count-loop.xml ├── settings.gradle ├── split-route ├── README.md ├── data │ ├── in │ │ └── sample-input.txt │ └── out │ │ └── .gitkeep └── split-route-merge-flow.xml ├── twitter-garden-hose ├── README.md └── pull-from-twitter-garden-hose-flow.xml └── twitter-solr ├── README.md └── twitter-solr-flow.xml /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | CONTRIBUTING.md export-ignore -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [xmlking] 4 | open_collective: xmlking 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/gradle,osx,windows 2 | 3 | ## App ## 4 | logs 5 | *.log 6 | NOTES.md 7 | 8 | ### Gradle ### 9 | .gradle 10 | build/ 11 | 12 | # Ignore Gradle GUI config 13 | gradle-app.setting 14 | 15 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 16 | !gradle-wrapper.jar 17 | 18 | 19 | ### Intellij ### 20 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio 21 | 22 | *.iml 23 | 24 | ## Directory-based project format: 25 | .idea/ 26 | # if you remove the above rule, at least ignore the following: 27 | 28 | # User-specific stuff: 29 | # .idea/workspace.xml 30 | # .idea/tasks.xml 31 | # .idea/dictionaries 32 | 33 | # Sensitive or high-churn files: 34 | # .idea/dataSources.ids 35 | # .idea/dataSources.xml 36 | # .idea/sqlDataSources.xml 37 | # .idea/dynamic.xml 38 | # .idea/uiDesigner.xml 39 | 40 | # Gradle: 41 | # .idea/gradle.xml 42 | # .idea/libraries 43 | 44 | # Mongo Explorer plugin: 45 | # .idea/mongoSettings.xml 46 | 47 | ## File-based project format: 48 | *.ipr 49 | *.iws 50 | 51 | ## Plugin-specific files: 52 | 53 | # IntelliJ 54 | /out/ 55 | 56 | # mpeltonen/sbt-idea plugin 57 | .idea_modules/ 58 | 59 | # JIRA plugin 60 | atlassian-ide-plugin.xml 61 | 62 | # Crashlytics plugin (for Android Studio and IntelliJ) 63 | com_crashlytics_export_strings.xml 64 | crashlytics.properties 65 | crashlytics-build.properties 66 | 67 | 68 | ### OSX ### 69 | .DS_Store 70 | .AppleDouble 71 | .LSOverride 72 | 73 | # Icon must end with two \r 74 | Icon 75 | 76 | 77 | # Thumbnails 78 | ._* 79 | 80 | # Files that might appear in the root of a volume 81 | .DocumentRevisions-V100 82 | .fseventsd 83 | .Spotlight-V100 84 | .TemporaryItems 85 | .Trashes 86 | .VolumeIcon.icns 87 | 88 | # Directories potentially created on remote AFP share 89 | .AppleDB 90 | .AppleDesktop 91 | Network Trash Folder 92 | Temporary Items 93 | .apdisk 94 | 95 | 96 | ### Windows ### 97 | # Windows image file caches 98 | Thumbs.db 99 | ehthumbs.db 100 | 101 | # Folder config file 102 | Desktop.ini 103 | 104 | # Recycle Bin used on file shares 105 | $RECYCLE.BIN/ 106 | 107 | # Windows Installer files 108 | *.cab 109 | *.msi 110 | *.msm 111 | *.msp 112 | 113 | # Windows shortcuts 114 | *.lnk -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | NiFi Examples 2 | ================= 3 | 4 | Apache NiFi example flows. 5 | 6 | #### collect-stream-logs 7 | 8 | This [flow](./collect-stream-logs/) shows workflow for log collection, aggregation, store and display. 9 | 10 | 1. Ingest logs from folders. 11 | 2. Listen for syslogs on UDP port. 12 | 3. Merge syslogs and drop-in logs and persist merged logs to Solr for historical search. 13 | 4. Dashboard: stream real-time log events to dashboard and enable cross-filter search on historical logs data. 14 | 15 | 16 | #### iot-activity-tracker 17 | 18 | This [flow](./iot-activity-tracker/) shows how to bring IoT data into Enterprise. 19 | 20 | 1. Ingest IoT data over WebSocket and HTTP 21 | 3. Store all data to Hadoop(HDFS) and summary data to NoSQL(MarkLogic) for historical data search. 22 | 4. Route data based on pre-set thresholds (vital signs like `pulse rate` and `blood pressure`) to alert users and physicians. 23 | 5. Inactivity Reporting 24 | 25 | 26 | #### oltp-to-olap 27 | 28 | A low latency *Change Data Capture* [flow](./oltp-cdc-olap/) to continuously replicate data from OLTP(MySQL) to OLAP(NoSQL) systems with no impact to the source. 29 | 30 | 1. Multi-tenant: can contain data from many different databases, support multiple consumers. 31 | 2. Flexible CDC: Capture changes from many data sources and types. 32 | 1. Source consistency preservation. No impact to the source. 33 | 2. Both DML (INSERT/UPDATE/DELETE) and DDL (ALTER/CREATE/DROP) are captured non invasively. 34 | 3. Produce Logical Change Records (LCR) in JSON format. 35 | 4. Commits at the source are grouped by transaction. 36 | 3. Flexible Consumer Dataflows: consumer dataflows can be implemented in Apache NiFi, Flink, Spark or Apex 37 | 1. Parallel processing data filtering, transformation and loading. 38 | 4. Flexible Databus: store LCRs in **Kafka** streams for durability and pub-sub semantics. 39 | 1. Use *only* Kafka as input for all consumer dataflows. 40 | 1. Feed data to many client types (real-time, slow/catch-up, full bootstrap). 41 | 2. Consumption from an arbitrary time point in the change stream including full bootstrap capability of the entire data. 42 | 3. Guaranteed in-commit-order and at-least-once delivery. 43 | 4. Partitioned consumption (partitioned data to different Kafka topics based on database name, table or any field of LCR) 44 | 5. Both batch and near real time delivery. 45 | 46 | 47 | #### csv-to-json 48 | 49 | This [flow](./csv-to-json/) shows how to convert a CSV entry to a JSON document using ExtractText and ReplaceText. 50 | 51 | #### decompression 52 | 53 | This [flow](./decompression/) demonstrates taking an archive that is created with several levels of compression and then continuously 54 | decompressing it using a loop until the archived file is extracted out. 55 | 56 | #### http-get-route 57 | 58 | his [flow](./http-get-route/) pulls from a web service (example is nifi itself), extracts text from a specific section, makes a routing decision 59 | on that extracted value, prepares to write to disk using PutFile. 60 | 61 | #### invoke-http-route 62 | 63 | This [flow](./invoke-http-route/) demonstrates how to call an HTTP service based on an incoming FlowFile, and route the original FlowFile 64 | based on the status code returned from the invocation. In this example, every 30 seconds a FlowFile is produced, 65 | an attribute is added to the FlowFile that sets q=nifi, the google.com is invoked for that FlowFile, and any response 66 | with a 200 is routed to a relationship called 200. 67 | 68 | #### retry-count-loop 69 | 70 | This [process group](./retry/) can be used to maintain a count of how many times a flowfile goes through it. If it reaches some 71 | configured threshold it will route to a 'Limit Exceeded' relationship otherwise it will route to 'retry'. 72 | Great for processes which you only want to run X number of times before you give up. 73 | 74 | #### split-route 75 | 76 | This [flow](./split-route/) demonstrates splitting a file on line boundaries, routing the splits based on a regex in the content, 77 | merging the less important files together for storage somewhere, and sending the higher priority files down 78 | another path to take immediate action. 79 | 80 | #### twitter-garden-hose 81 | 82 | This [flow](./twitter-garden-hose/) pulls from Twitter using the garden hose setting; it pulls out some basic attributes from the Json and 83 | then routes only those items that are actually tweets. 84 | 85 | #### twitter-solr 86 | 87 | This [flow](./twitter-solr/) shows how to index tweets with Solr using NiFi. Pre-requisites for this flow are NiFi 0.3.0 or later, 88 | the creation of a Twitter application, and a running instance of Solr 5.1 or later with a tweets collection: 89 | 90 | 91 | ### Install NiFi 92 | 1. Manual: Download [Apache NiFi](https://nifi.apache.org/download.html) binaries and unpack to a folder. 93 | 2. On Mac: `brew install nifi` 94 | 95 | ### Run NiFi 96 | ```bash 97 | cd /Developer/Applications/nifi 98 | ./bin/nifi.sh start 99 | ./bin/nifi.sh stop 100 | ``` 101 | On Mac 102 | ```bash 103 | # nifi start|stop|run|restart|status|dump|install 104 | nifi start 105 | nifi status 106 | nifi stop 107 | # Working Directory: /usr/local/Cellar/nifi/0.3.0/libexec 108 | ``` -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | allprojects { 2 | repositories { 3 | mavenCentral() 4 | } 5 | version = rootVersion 6 | } 7 | 8 | subprojects { 9 | 10 | } 11 | 12 | task wrapper(type: Wrapper) { 13 | description = 'Generates gradlew[.bat] scripts' 14 | gradleVersion = '2.7' 15 | } 16 | -------------------------------------------------------------------------------- /collect-stream-logs/README.md: -------------------------------------------------------------------------------- 1 | collect-stream-logs 2 | =================== 3 | 4 | 1. Ingest logs from folders. 5 | 2. Listen for syslogs on UDP port. 6 | 3. Merge syslogs and drop-in logs and persist merged logs to Solr for historical search. 7 | 4. Dashboard: stream real-time log events to dashboard and enable cross-filter search on historical logs data. 8 | 9 | Note: this flow depends on **nifi-websocket** module, download [nar](https://github.com/xmlking/nifi-websocket/releases/download/0.1.0/nifi-websocket-0.1.0-SNAPSHOT.nar) and copy to `$NIFI_HOME/lib` 10 | 11 | ### Run log generator 12 | ```bash 13 | gradle :collect-stream-logs:log-generator:run 14 | ``` 15 | 16 | ### Flow 17 | ![logs dataflow](./logs-flow.png) 18 | 19 | ### Demo 20 | ![streaming logs](./logs-demo.png) 21 | 22 | ### Reference 23 | 1. [Collecting Logs with Apache NiFi](http://bryanbende.com/development/2015/05/17/collecting-logs-with-apache-nifi/) -------------------------------------------------------------------------------- /collect-stream-logs/dashboard/draw.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Real time drawing App 6 | 9 | 10 | 11 | 12 | 13 | Your browser needs to support canvas for this to work! 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /collect-stream-logs/dashboard/log.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Cloud Management Platform: CMP Console 6 | 7 | 8 | 9 | 10 | 36 | 37 | 38 |
39 | 40 | 41 |
42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /collect-stream-logs/dashboard/reqscanvas.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Nodestalgia websocket & canvas experiment 3 | * 2012 fcsonline 4 | * Sumo: This file is copied from https://github.com/fcsonline/nodestalgia/tree/develop/public/javascripts and modified to use Vert.X EventBus 5 | */ 6 | (function () { 7 | 8 | var PI_2 = Math.PI * 2; 9 | var MAX_MSG_TTL = 50; 10 | var MARGIN_LEFT = 150; 11 | var MARGIN_RIGHT = 150; 12 | var MARGIN_TOP = 50; 13 | var MARGIN_BOTTOM = 50; 14 | var BULLET_POPUP_DIST = 20; 15 | var BULLET_SIZE = 4; 16 | var DEFAULT_FONT = "10pt Arial"; 17 | var PONG_HEIGHT = 50; 18 | 19 | var canvasW = 1000; 20 | var canvasH = 560; 21 | var friction = 0.99; 22 | var requests = []; 23 | var typerequests = {}; 24 | var messages = []; 25 | var srcslots = []; 26 | var dstslots = []; 27 | var origins = []; 28 | var total = 0; 29 | var pongy = 0; 30 | var bulletpopup = -1; 31 | var pauseinfo = {}; 32 | 33 | var canvas; 34 | var ctx; 35 | 36 | var longDays = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; 37 | var longMonths = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; 38 | 39 | var intervalId = null; 40 | var intervalLoopTime = 30; 41 | 42 | var speed = 0; 43 | var sumarize = true; 44 | var colorize = true; 45 | var time = true; 46 | 47 | function init() { 48 | $canvas = $("#mainCanvas"); 49 | canvas = $canvas[0]; 50 | 51 | // Load dynamic properties 52 | intervalLoopTime = $canvas.data("frame-rate"); 53 | speed = $canvas.data("speed"); 54 | colorize = $canvas.data("colorize"); 55 | sumarize = $canvas.data("sumarize"); 56 | time = $canvas.data("time"); 57 | 58 | if (canvas.getContext) { 59 | setup(); 60 | resetcounters(); 61 | intervalId = setInterval(run, intervalLoopTime); 62 | } 63 | else { 64 | alert("Sorry, needs a recent version of Chrome, Firefox, Opera, Safari, or Internet Explorer 9."); 65 | } 66 | } 67 | 68 | function setup() { 69 | var canvasDiv = $("#canvasContainer"); 70 | 71 | canvasW = canvasDiv.outerWidth(); 72 | canvasH = canvasDiv.outerHeight(); 73 | 74 | canvas.setAttribute("width", canvasW); 75 | canvas.setAttribute("height", canvasH); 76 | 77 | pongy = canvasH / 2 - PONG_HEIGHT / 2; 78 | 79 | console.log("Initialized canvas with size: " + canvasW + "x" + canvasH); 80 | console.log("Initialized " + Math.floor(canvasH / 20) + " request and resource vertical slots"); 81 | 82 | ctx = canvas.getContext("2d"); 83 | } 84 | 85 | function resetcounters() { 86 | // Init label display responses 87 | typerequests['200'] = 0; 88 | typerequests['404'] = 0; 89 | typerequests['304'] = 0; 90 | } 91 | 92 | function run(readonly) { 93 | 94 | if (readonly === undefined || readonly === false) { 95 | // For paused information 96 | pauseinfo ['200'] = typerequests['200']; 97 | pauseinfo ['304'] = typerequests['304']; 98 | pauseinfo ['404'] = typerequests['404']; 99 | pauseinfo.date = new Date(); 100 | } 101 | 102 | ctx.globalCompositeOperation = "source-over"; 103 | ctx.fillStyle = "rgb(0,0,0)"; 104 | ctx.fillRect(0, 0, canvasW, canvasH); 105 | ctx.font = DEFAULT_FONT; 106 | ctx.globalCompositeOperation = "lighter"; 107 | 108 | var Mrnd = Math.random; 109 | var Mabs = Math.abs; 110 | 111 | // Obsolete arrays 112 | var orequests = []; 113 | var omessages = []; 114 | var osrcslots = []; 115 | var odstslots = []; 116 | 117 | var i = requests.length; 118 | while (i--) { 119 | var m = requests[i]; 120 | var x = m.x; 121 | var y = m.y; 122 | var vX = m.vX; 123 | var vY = m.vY; 124 | 125 | var nextX = x + vX; 126 | var nextY = y + vY; 127 | 128 | if (nextX > canvasW - MARGIN_RIGHT) { 129 | nextX = canvasW - MARGIN_RIGHT; 130 | vX *= -1; 131 | 132 | // Establish the repplied attribute for pong 133 | m.repplied = true; 134 | 135 | // Push a new message 136 | var msg = new Message(); 137 | msg.x = nextX - 50; 138 | msg.y = nextY; 139 | msg.color = m.color; 140 | msg.text = m.req.result; 141 | msg.ttl = MAX_MSG_TTL; // Aprox: 1.5s 142 | messages.push(msg); 143 | 144 | var g = ctx.createRadialGradient(nextX, nextY, BULLET_SIZE, nextX, nextY, BULLET_SIZE * 20); 145 | g.addColorStop(0, "rgba(" + m.color.r + "," + m.color.g + "," + m.color.b + "," + 1 + ")"); 146 | g.addColorStop(0.2, "rgba(" + m.color.r + "," + m.color.g + "," + m.color.b + ", 0.4)"); 147 | g.addColorStop(1.0, "rgba(255,255,255,0)"); 148 | 149 | ctx.save(); 150 | ctx.fillStyle = g; 151 | ctx.beginPath(); 152 | ctx.arc(nextX, nextY, BULLET_SIZE, 0, PI_2, false); 153 | ctx.fill(); 154 | ctx.restore(); 155 | 156 | // Search the source slot, for removing 157 | var dstslotpos = findSlotByTarget(m.req.path); 158 | 159 | if (dstslotpos >= 0) { 160 | dstslots[dstslotpos].count--; 161 | if (dstslots[dstslotpos].count === 0) { 162 | console.log('Removed obsoleted resource slot at: ' + dstslotpos); 163 | odstslots.push(dstslotpos); 164 | } 165 | } 166 | 167 | } else if (nextX < MARGIN_LEFT) { 168 | // Remove the request from the stack 169 | orequests.push(i); 170 | 171 | // Search the request slot, for removing 172 | var srcslotpos = findSlotByIp(m.req.ip); 173 | 174 | if (srcslotpos >= 0) { 175 | srcslots[srcslotpos].count--; 176 | if (srcslots[srcslotpos].count === 0) { 177 | console.log('Removed obsoleted request slot at: ' + srcslotpos); 178 | osrcslots.push(srcslotpos); 179 | } 180 | } 181 | } 182 | 183 | if (nextY > canvasH) { 184 | nextY = canvasH; 185 | vY *= -1; 186 | } else if (nextY < 0) { 187 | nextY = 0; 188 | vY *= -1; 189 | } 190 | 191 | if (readonly === undefined || readonly === false) { 192 | m.vX = vX; 193 | m.vY = vY; 194 | m.x = nextX; 195 | m.y = nextY; 196 | } 197 | 198 | ctx.save(); 199 | ctx.fillStyle = colorDef(m.color); 200 | ctx.beginPath(); 201 | ctx.arc(nextX, nextY, BULLET_SIZE, 0, PI_2, true); 202 | ctx.closePath(); 203 | ctx.fill(); 204 | ctx.restore(); 205 | } 206 | 207 | // DNS Source ip label 208 | var j = srcslots.length; 209 | ctx.save(); 210 | ctx.font = DEFAULT_FONT; 211 | ctx.shadowColor = "#fff"; 212 | ctx.shadowOffsetX = 0; 213 | ctx.shadowOffsetY = 0; 214 | ctx.shadowBlur = 0; 215 | ctx.fillStyle = "#ffffff"; 216 | 217 | function fitText(text, maxwidth) { 218 | 219 | while (ctx.measureText(text).width > maxwidth) { 220 | text = text.substring(0, text.length - 1); 221 | } 222 | 223 | return text; 224 | } 225 | 226 | while (j--) { 227 | var s = srcslots[j]; 228 | 229 | 230 | ctx.fillText(fitText(s.ip, MARGIN_LEFT), 10, s.y); 231 | } 232 | 233 | ctx.restore(); 234 | 235 | // Target label 236 | var k = dstslots.length; 237 | ctx.save(); 238 | ctx.font = DEFAULT_FONT; 239 | ctx.shadowColor = "#fff"; 240 | ctx.shadowOffsetX = 0; 241 | ctx.shadowOffsetY = 0; 242 | ctx.shadowBlur = 0; 243 | ctx.fillStyle = "#ffffff"; 244 | 245 | while (k--) { 246 | var t = dstslots[k]; 247 | ctx.fillText(t.path, canvasW - MARGIN_RIGHT + 10, t.y); 248 | } 249 | 250 | ctx.restore(); 251 | 252 | // HTTP Result labels 253 | var n = messages.length; 254 | ctx.save(); 255 | ctx.font = DEFAULT_FONT; 256 | ctx.shadowColor = "#fff"; 257 | ctx.shadowOffsetX = 0; 258 | ctx.shadowOffsetY = 0; 259 | 260 | while (n--) { 261 | var title = messages[n]; 262 | 263 | if (--title.ttl > 0) { 264 | ctx.fillStyle = colorDef(title.color, title.ttl / MAX_MSG_TTL); 265 | ctx.shadowBlur = title.ttl / 5; 266 | ctx.fillText(title.text, title.x, title.y); 267 | } else { 268 | omessages.push(n); 269 | } 270 | 271 | } 272 | 273 | ctx.restore(); 274 | 275 | // Next Pong position 276 | var nrpos = findNextNonReppliedRequest(); 277 | if (nrpos >= 0) { 278 | pongy = Math.max(requests[nrpos].y - PONG_HEIGHT / 2, 0); 279 | } 280 | 281 | // Display pong 282 | ctx.save(); 283 | ctx.fillStyle = "rgb(150,29,28)"; 284 | ctx.fillRect(canvasW - MARGIN_RIGHT, pongy, 10, PONG_HEIGHT); 285 | ctx.restore(); 286 | 287 | // Type requests and total label 288 | if (sumarize) { 289 | var st = ''; 290 | 291 | st += ' HTTP OK: ' + pad(pauseinfo['200'], 7); 292 | st += ' HTTP NOT FOUND: ' + pad(pauseinfo['404'], 5); 293 | st += ' HTTP NOT MODIFIED: ' + pad(pauseinfo['304'], 5); 294 | st += ' TOTAL: ' + pad(total, 8); 295 | 296 | var tx = canvasW - 600; 297 | var ty = canvasH - 5; 298 | ctx.save(); 299 | ctx.font = DEFAULT_FONT; 300 | ctx.shadowColor = "#fff"; 301 | ctx.shadowOffsetX = 0; 302 | ctx.shadowOffsetY = 0; 303 | ctx.shadowBlur = 0; 304 | ctx.fillStyle = "#ffffff"; 305 | ctx.fillText(st, tx, ty); 306 | ctx.restore(); 307 | } 308 | 309 | // Date & Time display 310 | if (time) { 311 | var date = pauseinfo.date; 312 | ctx.save(); 313 | ctx.font = DEFAULT_FONT; 314 | ctx.shadowColor = "#fff"; 315 | ctx.shadowOffsetX = 0; 316 | ctx.shadowOffsetY = 0; 317 | ctx.shadowBlur = 0; 318 | ctx.fillStyle = "#ffffff"; 319 | ctx.fillText(getDateDisplay(date), 5, 15); 320 | ctx.fillText(getTimeDisplay(date), 5, 35); 321 | ctx.restore(); 322 | } 323 | 324 | // Remove obsolete requests 325 | requests = $.grep(requests, function (n, i) { 326 | return $.inArray(i, orequests) < 0; 327 | }); 328 | 329 | // Remove obsolete messages 330 | messages = $.grep(messages, function (n, i) { 331 | return $.inArray(i, omessages) < 0; 332 | }); 333 | 334 | // Remove obsolete resources slots 335 | dstslots = $.grep(dstslots, function (n, i) { 336 | return $.inArray(i, odstslots) < 0; 337 | }); 338 | 339 | // Remove obsolete requests slots 340 | srcslots = $.grep(srcslots, function (n, i) { 341 | return $.inArray(i, osrcslots) < 0; 342 | }); 343 | 344 | } 345 | 346 | function RemoteRequest() { 347 | this.color = ''; // Defined by origin color o random for only one origin 348 | this.x = 0; 349 | this.y = 0; 350 | this.vX = 0; 351 | this.vY = 0; 352 | this.req = null; // Filled by websocket 353 | this.repplied = false; // For pong targets 354 | } 355 | 356 | function Slot() { 357 | this.x = 0; 358 | this.y = 0; 359 | this.count = 0; 360 | this.ip = ''; // For request slots 361 | this.path = ''; // For resource slots 362 | } 363 | 364 | function Origin() { 365 | this.color = ''; 366 | this.path = ''; 367 | } 368 | 369 | function colorDef(obj, alpha) { 370 | if (alpha !== undefined) { 371 | return "rgba(" + obj.r + "," + obj.g + "," + obj.b + "," + alpha + ")"; 372 | } else { 373 | return "rgb(" + obj.r + "," + obj.g + "," + obj.b + ")"; 374 | } 375 | } 376 | 377 | function Message() { 378 | this.x = 0; 379 | this.y = 0; 380 | this.text = ""; 381 | } 382 | 383 | function rect(context, x, y, w, h) { 384 | context.beginPath(); 385 | context.rect(x, y, w, h); 386 | context.closePath(); 387 | context.fill(); 388 | } 389 | 390 | function pad(num, length) { 391 | var str = '' + num; 392 | while (str.length < length) { 393 | str = '0' + str; 394 | } 395 | 396 | return str; 397 | } 398 | 399 | function getDateDisplay(date) { 400 | return longDays[date.getDay()] + ', ' + longMonths[date.getMonth()] + ' ' + date.getDate() + ', ' + date.getFullYear() + '\r\n'; 401 | } 402 | 403 | function getTimeDisplay(date) { 404 | return (date.getHours() < 10 ? '0' : '') + date.getHours() + ':' + 405 | (date.getMinutes() < 10 ? '0' : '') + date.getMinutes() + ':' + 406 | (date.getSeconds() < 10 ? '0' : '') + date.getSeconds(); 407 | } 408 | 409 | function findNextNonReppliedRequest() { 410 | var j; 411 | for (j = 0; j < requests.length; j++) { 412 | if (!requests[j].repplied) { 413 | return j; 414 | } 415 | } 416 | 417 | return -1; 418 | } 419 | 420 | function findSlotByIp(ip) { 421 | var j; 422 | for (j = 0; j < srcslots.length; j++) { 423 | if (ip === srcslots[j].ip) { 424 | return j; 425 | } 426 | } 427 | 428 | return -1; 429 | } 430 | 431 | function findSlotByTarget(target) { 432 | var j; 433 | for (j = 0; j < dstslots.length; j++) { 434 | if (target === dstslots[j].path) { 435 | return j; 436 | } 437 | } 438 | 439 | return -1; 440 | } 441 | 442 | function findOriginByName(filename) { 443 | var j; 444 | for (j = 0; j < origins.length; j++) { 445 | if (filename === origins[j].path) { 446 | return j; 447 | } 448 | } 449 | 450 | return -1; 451 | } 452 | 453 | window.onload = init; 454 | 455 | // Establish the websocket connection 456 | //var eb = new EventBus('http://localhost:5555/eventbus'); 457 | var eb = new EventBus('http://apsrt1451:5555/eventbus'); 458 | 459 | eb.onopen = function () { 460 | 461 | eb.registerHandler('log', function (err, res) { 462 | console.log(res.body); 463 | var robj = res.body; 464 | 465 | 466 | if (typerequests[robj.result] === undefined) { 467 | typerequests[robj.result] = 0; 468 | } 469 | 470 | typerequests[robj.result]++; 471 | 472 | // if not paused, then add it to buffer 473 | if (intervalId) { 474 | var m = new RemoteRequest(); 475 | m.x = MARGIN_LEFT; // canvasW * 0.5; 476 | m.vX = speed; 477 | m.req = robj; 478 | requests.push(m); 479 | 480 | if (robj.origin !== undefined) { 481 | // Find pre generated origin, for inherit color 482 | var originpos = findOriginByName(robj.origin); 483 | 484 | if (originpos < 0) { 485 | // New origin assignment 486 | var origin = new Origin(); 487 | origin.path = robj.origin; 488 | origin.color = { 489 | r: Math.floor(Math.random() * 155 + 100), 490 | g: Math.floor(Math.random() * 155 + 100), 491 | b: Math.floor(Math.random() * 155 + 100) 492 | }; 493 | originpos = origins.push(origin) - 1; 494 | console.log('New origin: ' + robj.origin); 495 | } 496 | 497 | m.color = origins[originpos].color; 498 | } else { 499 | // For no origin request, only one origin, then random color 500 | m.color = { 501 | r: Math.floor(Math.random() * 155 + 100), 502 | g: Math.floor(Math.random() * 155 + 100), 503 | b: Math.floor(Math.random() * 155 + 100) 504 | }; 505 | } 506 | 507 | // Search a request slot 508 | var srcslotpos = findSlotByIp(robj.ip); 509 | 510 | if (srcslotpos < 0) { 511 | // New slot assignment 512 | var sslot = new Slot(); 513 | sslot.ip = robj.ip; 514 | sslot.count = 1; 515 | sslot.y = Math.floor(Math.random() * (canvasH - MARGIN_TOP - MARGIN_BOTTOM) + MARGIN_TOP); // TODO: Find a correct slot vertical position 516 | srcslotpos = srcslots.push(sslot) - 1; 517 | console.log('New request sslot at: ' + sslot.y); 518 | } else { 519 | srcslots[srcslotpos].count++; 520 | console.log('Recycled request slot at: ' + srcslots[srcslotpos].y); 521 | } 522 | 523 | // Search a resource slot 524 | var dstslotpos = findSlotByTarget(robj.path); 525 | 526 | if (dstslotpos < 0) { 527 | // New slot assignment 528 | var dslot = new Slot(); 529 | dslot.path = robj.path; 530 | dslot.count = 1; 531 | dslot.y = Math.floor(Math.random() * (canvasH - MARGIN_TOP - MARGIN_BOTTOM) + MARGIN_TOP); // TODO: Find a correct slot vertical position 532 | dstslotpos = dstslots.push(dslot) - 1; 533 | console.log('New resource dslot at: ' + dslot.y); 534 | } else { 535 | dstslots[dstslotpos].count++; 536 | console.log('Recycled resource slot at: ' + dstslots[dstslotpos].y); 537 | } 538 | 539 | m.y = srcslots[srcslotpos].y; 540 | 541 | // When the origin and target slots are set, then the vertical speed can be calculated 542 | m.vY = (dstslots[dstslotpos].y - srcslots[srcslotpos].y) / (canvasW - MARGIN_LEFT - MARGIN_RIGHT) * speed; 543 | console.log('New request with vertical speed at: ' + m.vY); 544 | 545 | } 546 | 547 | ++total; 548 | }); 549 | 550 | }; 551 | 552 | function bulletInfoPopup(e) { 553 | var minpos; 554 | var mindist; 555 | var j; 556 | 557 | for (j = 0; j < requests.length; j++) { 558 | var tr = requests[j]; 559 | var dist = Math.sqrt(Math.pow(e.pageX - tr.x, 2) + Math.pow(e.pageY - tr.y, 2)); 560 | 561 | if (mindist === undefined || dist < mindist) { 562 | minpos = j; 563 | mindist = dist; 564 | } 565 | } 566 | 567 | if (mindist < BULLET_POPUP_DIST) { 568 | if (minpos !== bulletpopup || bulletpopup < 0) { 569 | console.log('Showing popup of request: ' + minpos); 570 | console.log(requests[minpos]); 571 | bulletpopup = minpos; 572 | 573 | // Refresh canvas 574 | run(true); 575 | 576 | // Display popup rectangle 577 | var r = requests[bulletpopup]; 578 | var rx = r.x + 20; 579 | var ry = r.y + 20; 580 | 581 | ctx.save(); 582 | ctx.fillStyle = colorDef(r.color); 583 | ctx.fillRect(rx, ry, 220, 95); 584 | ctx.font = DEFAULT_FONT; 585 | ctx.fillStyle = "#ffffff"; 586 | ctx.fillText(r.req.method + ' ' + r.req.path, rx + 10, ry + 20); 587 | ctx.fillText("From: " + r.req.ip, rx + 10, ry + 35); 588 | ctx.fillText("Result: " + r.req.result, rx + 10, ry + 50); 589 | ctx.fillText("Size: " + r.req.size, rx + 10, ry + 65); 590 | ctx.fillText("Time: " + r.req.time, rx + 10, ry + 80); 591 | ctx.restore(); 592 | 593 | $canvas.css('cursor', 'pointer'); 594 | } 595 | } else { 596 | if (bulletpopup >= 0) { 597 | console.log('Hidding popup'); 598 | bulletpopup = -1; 599 | 600 | // Refresh canvas 601 | run(true); 602 | 603 | $canvas.css('cursor', 'default'); 604 | } 605 | } 606 | 607 | } 608 | 609 | // Pause 610 | $(document).bind('keypress', function (e) { 611 | var unicode = e.keyCode ? e.keyCode : e.charCode; 612 | 613 | if (unicode == 32) { // Space - Pause 614 | if (intervalId) { 615 | clearInterval(intervalId); 616 | intervalId = null; 617 | 618 | // Refresh canvas 619 | run(true); 620 | 621 | // Bind the mousemove event 622 | $canvas.bind('mousemove', bulletInfoPopup); 623 | } else { 624 | // Unbind the mousemove event 625 | $canvas.unbind('mousemove', bulletInfoPopup); 626 | intervalId = setInterval(run, intervalLoopTime); 627 | } 628 | } else if (unicode == 43) { // + more horizontal speed 629 | speed = Math.min(speed + 5, 200); 630 | console.log("Speed set to: " + speed); 631 | } else if (unicode == 45) { // - less horizontal speed 632 | speed = Math.max(speed - 5, 10); 633 | console.log("Speed set to: " + speed); 634 | } 635 | }); 636 | 637 | // Resize window event 638 | $(window).resize(function () { 639 | setup(); 640 | }); 641 | 642 | })(); 643 | -------------------------------------------------------------------------------- /collect-stream-logs/dashboard/testEB.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 27 | 28 | 29 | 30 |
31 | 32 |
33 | Address:
34 | Message: 35 | 36 |
37 | 38 |
39 | Sent messages:
40 | 41 |
42 |
43 |
44 | 45 |
46 | 47 |
48 | Address: 49 | 50 |
51 | 52 |
53 | Subscriptions:
54 | 55 |
56 |
57 |
58 | 59 |
60 | 61 |
62 | 63 | Received messages:
64 | 65 |
66 |
67 | 68 |
69 | 70 |
71 |
72 |
73 | Connection Status:  74 |
Not connected
75 |
76 | 77 | 150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /collect-stream-logs/data/out/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xmlking/nifi-examples/3a12b86d7a288285af39142cb3f56c0cba0ee312/collect-stream-logs/data/out/.gitkeep -------------------------------------------------------------------------------- /collect-stream-logs/log-generator/build.gradle: -------------------------------------------------------------------------------- 1 | group 'com.crossbusiness.loggen' 2 | 3 | apply plugin: 'groovy' 4 | apply plugin: 'application' 5 | 6 | sourceCompatibility = 1.8 7 | 8 | // mainClassName = "com.crossbusiness.loggen.AppLogGenerator" 9 | mainClassName = "com.crossbusiness.loggen.AccessLogGenerator" 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | compile 'org.codehaus.groovy:groovy-all:+' 17 | compile 'ch.qos.logback:logback-classic:1.1.3' 18 | compile 'ch.qos.logback:logback-access:1.1.3' 19 | compile 'net.logstash.logback:logstash-logback-encoder:4.5.1' 20 | } 21 | -------------------------------------------------------------------------------- /collect-stream-logs/log-generator/src/main/groovy/com.crossbusiness.loggen/AccessLogGenerator.groovy: -------------------------------------------------------------------------------- 1 | package com.crossbusiness.loggen 2 | 3 | import groovy.json.JsonBuilder 4 | import groovy.util.logging.Slf4j 5 | 6 | @Slf4j 7 | class AccessLogGenerator { 8 | 9 | public static final String[] ips=["123.221.14.56","16.180.70.237","10.182.189.79","218.193.16.244","198.122.118.164","114.214.178.92","233.192.62.103","244.157.45.12","81.73.150.239","237.43.24.118"] 10 | public static final String[] referers=["-","http://www.casualcyclist.com","http://bestcyclingreviews.com/top_online_shops","http://bleater.com","http://searchengine.com"] 11 | public static final String[] resources=["/handle-bars","/stems","/wheelsets","/forks","/seatposts","/saddles","/shifters","/Store/cart.jsp?productID="] 12 | public static final String[] useragents=["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36","Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1","Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25","Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201","Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0","Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"] 13 | 14 | AccessLogGenerator() { 15 | println 'Hello World' 16 | } 17 | 18 | def produce() { 19 | println "starting log events..........." 20 | def ranDumb = new Random(); 21 | //127.0.0.1 - - [07/Mar/2012:23:21:47 +0100] "GET / HTTP/1.0" 200 454 "-" "ApacheBench/2.3" 22 | //def regex = ~/([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+).+\[(.+)\] "(\w+) ([^ ]+) .*" (\w+) (\w+)/; 23 | def regex = ~/^([^ ]*).+\[(.+)\] "(\w+) ([^ ]+) .*" (\w+) ([^ ]*)/ 24 | 25 | while (true) { 26 | AccessLogGenerator.class.getResource( '/example.log' ).eachLine { line -> 27 | def matcher = regex.matcher(line) 28 | if (matcher.find()) { 29 | def logJson = new JsonBuilder(["ip": matcher.group(1), "time": matcher.group(2), "method": matcher.group(3), "path": matcher.group(4), "result": matcher.group(5), "size": matcher.group(6)]) 30 | 31 | log.info logJson.toString() 32 | } else { 33 | println "no match: " + line 34 | } 35 | //sleep(ranDumb.nextInt(2000-1000+1)+100) 36 | sleep(ranDumb.nextInt(200-100+1)+100) 37 | //sleep(ranDumb.nextInt(20-10+1)+50) 38 | } 39 | println "re-starting log events..........." 40 | } 41 | } 42 | 43 | static main(args) { 44 | def generator = new AccessLogGenerator() 45 | generator.produce() 46 | } 47 | } -------------------------------------------------------------------------------- /collect-stream-logs/log-generator/src/main/groovy/com.crossbusiness.loggen/AppLogGenerator.groovy: -------------------------------------------------------------------------------- 1 | package com.crossbusiness.loggen 2 | 3 | import groovy.json.JsonBuilder 4 | import groovy.util.logging.Slf4j 5 | 6 | @Slf4j 7 | class AppLogGenerator { 8 | 9 | static final int DEFAULT_NUM_LOGS = 100; 10 | static final long DEFAULT_DELAY = 100; 11 | 12 | public static final String ERROR = "ERROR"; 13 | public static final String WARN = "WARN"; 14 | public static final String INFO = "INFO"; 15 | public static final String DEBUG = "DEBUG"; 16 | public static final String TRACE = "TRACE"; 17 | 18 | static final String[] LEVELS = [ERROR, WARN, INFO, DEBUG, TRACE]; 19 | 20 | static final String[] MESSAGES =["Solr is cool", "NiFi rocks!", "Lucene is awesome" ]; 21 | 22 | static final Exception[] EXCEPTIONS =[ 23 | new IllegalStateException("Uh-oh something went wrong"), 24 | new IllegalArgumentException("Invalid value"), 25 | new NullPointerException("Value was null") 26 | ]; 27 | 28 | 29 | AppLogGenerator() { 30 | println 'Hello World' 31 | } 32 | 33 | def produce(long numLogs, long delay) { 34 | Random rand = new Random(); 35 | for (int i=0; i < numLogs; i++) { 36 | switch(LEVELS[rand.nextInt(5)]) { 37 | case ERROR: 38 | final Exception e = EXCEPTIONS[rand.nextInt(EXCEPTIONS.length)]; 39 | log.error(e.getMessage(), e); 40 | break; 41 | case WARN: 42 | log.warn(MESSAGES[rand.nextInt(MESSAGES.length)]); 43 | break; 44 | case INFO: 45 | log.info(MESSAGES[rand.nextInt(MESSAGES.length)]); 46 | break; 47 | case DEBUG: 48 | log.debug(MESSAGES[rand.nextInt(MESSAGES.length)]); 49 | break; 50 | case TRACE: 51 | log.trace(MESSAGES[rand.nextInt(MESSAGES.length)]); 52 | break; 53 | default: 54 | log.debug("Default message"); 55 | } 56 | 57 | if (delay > 0) { 58 | try { 59 | Thread.sleep(delay); 60 | } catch (InterruptedException e) { 61 | log.error(e.getMessage(), e); 62 | } 63 | } 64 | } 65 | } 66 | 67 | static main(args) { 68 | def generator = new AppLogGenerator() 69 | 70 | int numLogs = DEFAULT_NUM_LOGS; 71 | long delay = DEFAULT_DELAY; 72 | 73 | if (args.length == 2) { 74 | numLogs = Integer.parseInt(args[0]); 75 | delay = Long.parseLong(args[1]); 76 | } 77 | 78 | generator.produce(numLogs, delay); 79 | } 80 | } -------------------------------------------------------------------------------- /collect-stream-logs/log-generator/src/main/java/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xmlking/nifi-examples/3a12b86d7a288285af39142cb3f56c0cba0ee312/collect-stream-logs/log-generator/src/main/java/.gitkeep -------------------------------------------------------------------------------- /collect-stream-logs/log-generator/src/main/resources/logback-access.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | %h %l %u %user %date "%r" %s %b 13 | 14 | 15 | 16 | 17 | ./logs/access.log 18 | true 19 | 20 | ./logs/archive/access-%d{yyyyMMdd}-%i.log.zip 21 | ${max.retention.days} 22 | 23 | 20MB 24 | 25 | 26 | 27 | %h %l %u [%t] "%r" %s %b "%i{Referer}" "%i{User-Agent}" 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /collect-stream-logs/log-generator/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | %-4relative [%thread] %-5level %logger{35} - %msg %n 11 | 12 | 13 | 14 | 15 | localhost 16 | 8888 17 | 18 | 19 | 20 | ./logs/access.log 21 | true 22 | 23 | ./logs/archive/access-%d{yyyyMMdd}-%i.log.zip 24 | ${max.retention.days} 25 | 26 | 20MB 27 | 28 | 29 | 30 | %h %l %u [%t] "%r" %s %b "%i{Referer}" "%i{User-Agent}" 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /collect-stream-logs/logs-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xmlking/nifi-examples/3a12b86d7a288285af39142cb3f56c0cba0ee312/collect-stream-logs/logs-demo.png -------------------------------------------------------------------------------- /collect-stream-logs/logs-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xmlking/nifi-examples/3a12b86d7a288285af39142cb3f56c0cba0ee312/collect-stream-logs/logs-flow.png -------------------------------------------------------------------------------- /csv-to-json/README.md: -------------------------------------------------------------------------------- 1 | csv-to-json 2 | =========== 3 | 4 | This flow shows how to convert a CSV entry to a JSON document using ExtractText and ReplaceText. -------------------------------------------------------------------------------- /decompression/README.md: -------------------------------------------------------------------------------- 1 | decompression 2 | ============= 3 | 4 | This flow demonstrates taking an archive that is created with several levels of compression and then continuously 5 | decompressing it using a loop until the archived file is extracted out. -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | rootVersion=0.1.0-SNAPSHOT 2 | 3 | #Override by USER_HOME/.gradle/gradle.properties 4 | sonatypeUsername= 5 | sonatypePassword= 6 | gitUsername= 7 | gitPassword= -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xmlking/nifi-examples/3a12b86d7a288285af39142cb3f56c0cba0ee312/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Sun Oct 04 11:47:38 PDT 2015 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.7-bin.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # Attempt to set APP_HOME 46 | # Resolve links: $0 may be a link 47 | PRG="$0" 48 | # Need this for relative symlinks. 49 | while [ -h "$PRG" ] ; do 50 | ls=`ls -ld "$PRG"` 51 | link=`expr "$ls" : '.*-> \(.*\)$'` 52 | if expr "$link" : '/.*' > /dev/null; then 53 | PRG="$link" 54 | else 55 | PRG=`dirname "$PRG"`"/$link" 56 | fi 57 | done 58 | SAVED="`pwd`" 59 | cd "`dirname \"$PRG\"`/" >&- 60 | APP_HOME="`pwd -P`" 61 | cd "$SAVED" >&- 62 | 63 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 64 | 65 | # Determine the Java command to use to start the JVM. 66 | if [ -n "$JAVA_HOME" ] ; then 67 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 68 | # IBM's JDK on AIX uses strange locations for the executables 69 | JAVACMD="$JAVA_HOME/jre/sh/java" 70 | else 71 | JAVACMD="$JAVA_HOME/bin/java" 72 | fi 73 | if [ ! -x "$JAVACMD" ] ; then 74 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 75 | 76 | Please set the JAVA_HOME variable in your environment to match the 77 | location of your Java installation." 78 | fi 79 | else 80 | JAVACMD="java" 81 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 82 | 83 | Please set the JAVA_HOME variable in your environment to match the 84 | location of your Java installation." 85 | fi 86 | 87 | # Increase the maximum file descriptors if we can. 88 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 89 | MAX_FD_LIMIT=`ulimit -H -n` 90 | if [ $? -eq 0 ] ; then 91 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 92 | MAX_FD="$MAX_FD_LIMIT" 93 | fi 94 | ulimit -n $MAX_FD 95 | if [ $? -ne 0 ] ; then 96 | warn "Could not set maximum file descriptor limit: $MAX_FD" 97 | fi 98 | else 99 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 100 | fi 101 | fi 102 | 103 | # For Darwin, add options to specify how the application appears in the dock 104 | if $darwin; then 105 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 106 | fi 107 | 108 | # For Cygwin, switch paths to Windows format before running java 109 | if $cygwin ; then 110 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 111 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 112 | JAVACMD=`cygpath --unix "$JAVACMD"` 113 | 114 | # We build the pattern for arguments to be converted via cygpath 115 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 116 | SEP="" 117 | for dir in $ROOTDIRSRAW ; do 118 | ROOTDIRS="$ROOTDIRS$SEP$dir" 119 | SEP="|" 120 | done 121 | OURCYGPATTERN="(^($ROOTDIRS))" 122 | # Add a user-defined pattern to the cygpath arguments 123 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 124 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 125 | fi 126 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 127 | i=0 128 | for arg in "$@" ; do 129 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 130 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 131 | 132 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 133 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 134 | else 135 | eval `echo args$i`="\"$arg\"" 136 | fi 137 | i=$((i+1)) 138 | done 139 | case $i in 140 | (0) set -- ;; 141 | (1) set -- "$args0" ;; 142 | (2) set -- "$args0" "$args1" ;; 143 | (3) set -- "$args0" "$args1" "$args2" ;; 144 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 145 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 146 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 147 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 148 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 149 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 150 | esac 151 | fi 152 | 153 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 154 | function splitJvmOpts() { 155 | JVM_OPTS=("$@") 156 | } 157 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 158 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 159 | 160 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 161 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /http-get-route/README.md: -------------------------------------------------------------------------------- 1 | http-get-route 2 | ============== 3 | 4 | Pulls from a web service (example is nifi itself), extracts text from a specific section, makes a routing decision 5 | on that extracted value, prepares to write to disk using PutFile. -------------------------------------------------------------------------------- /invoke-http-route/README.md: -------------------------------------------------------------------------------- 1 | invoke-http-route 2 | ================= 3 | 4 | This flow demonstrates how to call an HTTP service based on an incoming FlowFile, and route the original FlowFile 5 | based on the status code returned from the invocation. In this example, every 30 seconds a FlowFile is produced, 6 | an attribute is added to the FlowFile that sets q=nifi, the google.com is invoked for that FlowFile, and any response 7 | with a 200 is routed to a relationship called 200. -------------------------------------------------------------------------------- /invoke-http-route/invokeHttp-and-route-original-on-status-flow.xml: -------------------------------------------------------------------------------- 1 | 2 |