├── Kippo2ElasticSearch.json ├── LICENSE ├── README.md ├── geoip └── GeoIP.dat └── kippo2elasticsearch.py /Kippo2ElasticSearch.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Kippo2ElasticSearch", 3 | "services": { 4 | "query": { 5 | "list": { 6 | "0": { 7 | "query": "*", 8 | "alias": "", 9 | "color": "#7EB26D", 10 | "id": 0, 11 | "pin": false, 12 | "type": "lucene", 13 | "enable": true 14 | } 15 | }, 16 | "ids": [ 17 | 0 18 | ] 19 | }, 20 | "filter": { 21 | "list": {}, 22 | "ids": [] 23 | } 24 | }, 25 | "rows": [ 26 | { 27 | "title": "Graph", 28 | "height": "250px", 29 | "editable": true, 30 | "collapse": false, 31 | "collapsable": true, 32 | "panels": [ 33 | { 34 | "error": false, 35 | "span": 3, 36 | "editable": true, 37 | "group": [ 38 | "default" 39 | ], 40 | "type": "terms", 41 | "queries": { 42 | "mode": "all", 43 | "ids": [ 44 | 0 45 | ] 46 | }, 47 | "field": "_type", 48 | "exclude": [], 49 | "missing": true, 50 | "other": true, 51 | "size": 10, 52 | "order": "count", 53 | "style": { 54 | "font-size": "10pt" 55 | }, 56 | "donut": false, 57 | "tilt": false, 58 | "labels": true, 59 | "arrangement": "horizontal", 60 | "chart": "table", 61 | "counter_pos": "above", 62 | "spyable": true, 63 | "title": "Document Types", 64 | "tmode": "terms", 65 | "tstat": "total", 66 | "valuefield": "" 67 | }, 68 | { 69 | "error": false, 70 | "span": 3, 71 | "editable": true, 72 | "type": "terms", 73 | "loadingEditor": false, 74 | "field": "success", 75 | "exclude": [], 76 | "missing": true, 77 | "other": true, 78 | "size": 5, 79 | "order": "count", 80 | "style": { 81 | "font-size": "10pt" 82 | }, 83 | "donut": false, 84 | "tilt": false, 85 | "labels": true, 86 | "arrangement": "horizontal", 87 | "chart": "table", 88 | "counter_pos": "above", 89 | "spyable": true, 90 | "queries": { 91 | "mode": "all", 92 | "ids": [ 93 | 0 94 | ] 95 | }, 96 | "tmode": "terms", 97 | "tstat": "total", 98 | "valuefield": "", 99 | "title": "Successes" 100 | }, 101 | { 102 | "error": false, 103 | "span": 3, 104 | "editable": true, 105 | "type": "terms", 106 | "loadingEditor": false, 107 | "field": "sensor", 108 | "exclude": [], 109 | "missing": false, 110 | "other": false, 111 | "size": 5, 112 | "order": "count", 113 | "style": { 114 | "font-size": "10pt" 115 | }, 116 | "donut": false, 117 | "tilt": false, 118 | "labels": true, 119 | "arrangement": "horizontal", 120 | "chart": "table", 121 | "counter_pos": "above", 122 | "spyable": true, 123 | "queries": { 124 | "mode": "all", 125 | "ids": [ 126 | 0 127 | ] 128 | }, 129 | "tmode": "terms", 130 | "tstat": "total", 131 | "valuefield": "", 132 | "title": "Sensors" 133 | } 134 | ], 135 | "notice": false 136 | }, 137 | { 138 | "title": "Histogram", 139 | "height": "300px", 140 | "editable": true, 141 | "collapse": false, 142 | "collapsable": true, 143 | "panels": [ 144 | { 145 | "span": 12, 146 | "editable": true, 147 | "type": "histogram", 148 | "loadingEditor": false, 149 | "mode": "count", 150 | "time_field": "timestamp", 151 | "value_field": null, 152 | "x-axis": true, 153 | "y-axis": true, 154 | "scale": 1, 155 | "y_format": "none", 156 | "grid": { 157 | "max": null, 158 | "min": 0 159 | }, 160 | "queries": { 161 | "mode": "all", 162 | "ids": [ 163 | 0 164 | ] 165 | }, 166 | "annotate": { 167 | "enable": false, 168 | "query": "*", 169 | "size": 20, 170 | "field": "_type", 171 | "sort": [ 172 | "_score", 173 | "desc" 174 | ] 175 | }, 176 | "auto_int": false, 177 | "resolution": 100, 178 | "interval": "1d", 179 | "intervals": [ 180 | "auto", 181 | "1s", 182 | "1m", 183 | "5m", 184 | "10m", 185 | "30m", 186 | "1h", 187 | "3h", 188 | "12h", 189 | "1d", 190 | "1w", 191 | "1y" 192 | ], 193 | "lines": true, 194 | "fill": 0, 195 | "linewidth": 3, 196 | "points": false, 197 | "pointradius": 5, 198 | "bars": false, 199 | "stack": true, 200 | "spyable": true, 201 | "zoomlinks": true, 202 | "options": true, 203 | "legend": true, 204 | "show_query": true, 205 | "interactive": true, 206 | "legend_counts": true, 207 | "timezone": "browser", 208 | "percentage": false, 209 | "zerofill": true, 210 | "derivative": false, 211 | "tooltip": { 212 | "value_type": "cumulative", 213 | "query_as_alias": true 214 | }, 215 | "title": "Histogram", 216 | "scaleSeconds": false 217 | } 218 | ], 219 | "notice": false 220 | }, 221 | { 222 | "title": "Usernames", 223 | "height": "300px", 224 | "editable": true, 225 | "collapse": false, 226 | "collapsable": true, 227 | "panels": [ 228 | { 229 | "error": false, 230 | "span": 6, 231 | "editable": true, 232 | "type": "terms", 233 | "loadingEditor": false, 234 | "field": "username", 235 | "exclude": [], 236 | "missing": false, 237 | "other": false, 238 | "size": 20, 239 | "order": "count", 240 | "style": { 241 | "font-size": "10pt" 242 | }, 243 | "donut": false, 244 | "tilt": false, 245 | "labels": true, 246 | "arrangement": "horizontal", 247 | "chart": "bar", 248 | "counter_pos": "above", 249 | "spyable": true, 250 | "queries": { 251 | "mode": "all", 252 | "ids": [ 253 | 0 254 | ] 255 | }, 256 | "tmode": "terms", 257 | "tstat": "total", 258 | "valuefield": "", 259 | "title": "Usernames (top 20)" 260 | }, 261 | { 262 | "error": false, 263 | "span": 6, 264 | "editable": true, 265 | "type": "terms", 266 | "loadingEditor": false, 267 | "field": "username", 268 | "exclude": [], 269 | "missing": false, 270 | "other": false, 271 | "size": 20, 272 | "order": "count", 273 | "style": { 274 | "font-size": "10pt" 275 | }, 276 | "donut": false, 277 | "tilt": false, 278 | "labels": true, 279 | "arrangement": "horizontal", 280 | "chart": "pie", 281 | "counter_pos": "above", 282 | "spyable": true, 283 | "queries": { 284 | "mode": "all", 285 | "ids": [ 286 | 0 287 | ] 288 | }, 289 | "tmode": "terms", 290 | "tstat": "total", 291 | "valuefield": "", 292 | "title": "Usernames (top 20)" 293 | } 294 | ], 295 | "notice": false 296 | }, 297 | { 298 | "title": "Passwords", 299 | "height": "300px", 300 | "editable": true, 301 | "collapse": false, 302 | "collapsable": true, 303 | "panels": [ 304 | { 305 | "error": false, 306 | "span": 6, 307 | "editable": true, 308 | "type": "terms", 309 | "loadingEditor": false, 310 | "field": "password", 311 | "exclude": [], 312 | "missing": false, 313 | "other": false, 314 | "size": 20, 315 | "order": "count", 316 | "style": { 317 | "font-size": "10pt" 318 | }, 319 | "donut": false, 320 | "tilt": false, 321 | "labels": true, 322 | "arrangement": "horizontal", 323 | "chart": "bar", 324 | "counter_pos": "above", 325 | "spyable": true, 326 | "queries": { 327 | "mode": "all", 328 | "ids": [ 329 | 0 330 | ] 331 | }, 332 | "tmode": "terms", 333 | "tstat": "total", 334 | "valuefield": "", 335 | "title": "Passwords (top 20)" 336 | }, 337 | { 338 | "error": false, 339 | "span": 6, 340 | "editable": true, 341 | "type": "terms", 342 | "loadingEditor": false, 343 | "field": "password", 344 | "exclude": [], 345 | "missing": false, 346 | "other": false, 347 | "size": 20, 348 | "order": "count", 349 | "style": { 350 | "font-size": "10pt" 351 | }, 352 | "donut": false, 353 | "tilt": false, 354 | "labels": true, 355 | "arrangement": "horizontal", 356 | "chart": "pie", 357 | "counter_pos": "above", 358 | "spyable": true, 359 | "queries": { 360 | "mode": "all", 361 | "ids": [ 362 | 0 363 | ] 364 | }, 365 | "tmode": "terms", 366 | "tstat": "total", 367 | "valuefield": "", 368 | "title": "Passwords (top 20)" 369 | } 370 | ], 371 | "notice": false 372 | }, 373 | { 374 | "title": "Clients", 375 | "height": "300px", 376 | "editable": true, 377 | "collapse": false, 378 | "collapsable": true, 379 | "panels": [ 380 | { 381 | "error": false, 382 | "span": 6, 383 | "editable": true, 384 | "type": "terms", 385 | "loadingEditor": false, 386 | "field": "client", 387 | "exclude": [], 388 | "missing": false, 389 | "other": false, 390 | "size": 20, 391 | "order": "count", 392 | "style": { 393 | "font-size": "10pt" 394 | }, 395 | "donut": false, 396 | "tilt": false, 397 | "labels": true, 398 | "arrangement": "horizontal", 399 | "chart": "bar", 400 | "counter_pos": "above", 401 | "spyable": true, 402 | "queries": { 403 | "mode": "all", 404 | "ids": [ 405 | 0 406 | ] 407 | }, 408 | "tmode": "terms", 409 | "tstat": "total", 410 | "valuefield": "", 411 | "title": "SSH clients (top 20)" 412 | }, 413 | { 414 | "error": false, 415 | "span": 6, 416 | "editable": true, 417 | "type": "terms", 418 | "loadingEditor": false, 419 | "field": "client", 420 | "exclude": [], 421 | "missing": false, 422 | "other": false, 423 | "size": 20, 424 | "order": "count", 425 | "style": { 426 | "font-size": "10pt" 427 | }, 428 | "donut": false, 429 | "tilt": false, 430 | "labels": true, 431 | "arrangement": "horizontal", 432 | "chart": "pie", 433 | "counter_pos": "above", 434 | "spyable": true, 435 | "queries": { 436 | "mode": "all", 437 | "ids": [ 438 | 0 439 | ] 440 | }, 441 | "tmode": "terms", 442 | "tstat": "total", 443 | "valuefield": "", 444 | "title": "SSH clients (top 20)" 445 | } 446 | ], 447 | "notice": false 448 | }, 449 | { 450 | "title": "Maps", 451 | "height": "450px", 452 | "editable": true, 453 | "collapse": false, 454 | "collapsable": true, 455 | "panels": [ 456 | { 457 | "error": false, 458 | "span": 8, 459 | "editable": true, 460 | "type": "map", 461 | "loadingEditor": false, 462 | "map": "world", 463 | "colors": [ 464 | "#A0E2E2", 465 | "#265656" 466 | ], 467 | "size": 100, 468 | "exclude": [], 469 | "spyable": true, 470 | "queries": { 471 | "mode": "all", 472 | "ids": [ 473 | 0 474 | ] 475 | }, 476 | "title": "Attack map (world)", 477 | "field": "country" 478 | }, 479 | { 480 | "error": false, 481 | "span": 4, 482 | "editable": true, 483 | "type": "map", 484 | "loadingEditor": false, 485 | "map": "europe", 486 | "colors": [ 487 | "#A0E2E2", 488 | "#265656" 489 | ], 490 | "size": 100, 491 | "exclude": [], 492 | "spyable": true, 493 | "queries": { 494 | "mode": "all", 495 | "ids": [ 496 | 0 497 | ] 498 | }, 499 | "title": "Attack map (Europe)", 500 | "field": "country" 501 | } 502 | ], 503 | "notice": false 504 | }, 505 | { 506 | "title": "Events", 507 | "height": "650px", 508 | "editable": true, 509 | "collapse": false, 510 | "collapsable": true, 511 | "panels": [ 512 | { 513 | "error": false, 514 | "span": 12, 515 | "editable": true, 516 | "group": [ 517 | "default" 518 | ], 519 | "type": "table", 520 | "size": 100, 521 | "pages": 5, 522 | "offset": 0, 523 | "sort": [ 524 | "_score", 525 | "desc" 526 | ], 527 | "style": { 528 | "font-size": "9pt" 529 | }, 530 | "overflow": "min-height", 531 | "fields": [], 532 | "highlight": [], 533 | "sortable": true, 534 | "header": true, 535 | "paging": true, 536 | "spyable": true, 537 | "queries": { 538 | "mode": "all", 539 | "ids": [ 540 | 0 541 | ] 542 | }, 543 | "field_list": true, 544 | "status": "Stable", 545 | "trimFactor": 300, 546 | "normTimes": true, 547 | "title": "Documents", 548 | "all_fields": false, 549 | "localTime": false, 550 | "timeField": "@timestamp" 551 | } 552 | ], 553 | "notice": false 554 | } 555 | ], 556 | "editable": true, 557 | "index": { 558 | "interval": "none", 559 | "pattern": "[logstash-]YYYY.MM.DD", 560 | "default": "_all", 561 | "warm_fields": false 562 | }, 563 | "style": "dark", 564 | "failover": false, 565 | "panel_hints": true, 566 | "loader": { 567 | "save_gist": false, 568 | "save_elasticsearch": true, 569 | "save_local": true, 570 | "save_default": true, 571 | "save_temp": true, 572 | "save_temp_ttl_enable": true, 573 | "save_temp_ttl": "30d", 574 | "load_gist": true, 575 | "load_elasticsearch": true, 576 | "load_elasticsearch_size": 20, 577 | "load_local": true, 578 | "hide": false 579 | }, 580 | "pulldowns": [ 581 | { 582 | "type": "query", 583 | "collapse": false, 584 | "notice": false, 585 | "query": "*", 586 | "pinned": true, 587 | "history": [], 588 | "remember": 10, 589 | "enable": true 590 | }, 591 | { 592 | "type": "filtering", 593 | "collapse": false, 594 | "notice": true, 595 | "enable": true 596 | } 597 | ], 598 | "nav": [ 599 | { 600 | "type": "timepicker", 601 | "collapse": false, 602 | "notice": false, 603 | "status": "Stable", 604 | "time_options": [ 605 | "5m", 606 | "15m", 607 | "1h", 608 | "6h", 609 | "12h", 610 | "24h", 611 | "2d", 612 | "7d", 613 | "30d" 614 | ], 615 | "refresh_intervals": [ 616 | "5s", 617 | "10s", 618 | "30s", 619 | "1m", 620 | "5m", 621 | "15m", 622 | "30m", 623 | "1h", 624 | "2h", 625 | "1d" 626 | ], 627 | "timefield": "timestamp", 628 | "enable": true 629 | } 630 | ], 631 | "refresh": false 632 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Ion 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kippo2ElasticSearch 2 | =================== 3 | 4 | Transfer Kippo data to ElasticSearch 5 | -------------------------------------------------------------------------------- /geoip/GeoIP.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ikoniaris/kippo2elasticsearch/7a156ff26d6efaa583a546d8991a21c22b4a3176/geoip/GeoIP.dat -------------------------------------------------------------------------------- /kippo2elasticsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import collections 4 | import json 5 | import GeoIP 6 | 7 | import pony.orm 8 | import pony.options 9 | import pyes 10 | 11 | 12 | # MYSQL CONFIGURATION 13 | mysql_host = '127.0.0.1' 14 | mysql_port = 3306 15 | mysql_user = 'username' 16 | mysql_pass = 'password' 17 | mysql_db = 'database' 18 | 19 | # ELASTICSEARCH CONFIGURATION 20 | es_host = '127.0.0.1' 21 | es_port = 9200 22 | es_index = 'index' # suggested: kippo 23 | es_type = 'type' # suggested: kippo 24 | 25 | # We need this, otherwise pony returns an error during the SELECT because it enforces a limit 26 | pony.options.MAX_FETCH_COUNT = 999999 27 | 28 | db = pony.orm.Database('mysql', host=mysql_host, port=mysql_port, user=mysql_user, passwd=mysql_pass, db=mysql_db) 29 | 30 | with pony.orm.db_session: 31 | rows = db.select('SELECT auth.*, sessions.ip, clients.version, sensors.ip ' 32 | 'FROM auth INNER JOIN sessions ON auth.session = sessions.id ' 33 | 'INNER JOIN clients ON sessions.client = clients.id ' 34 | 'INNER JOIN sensors ON sessions.sensor = sensors.id') 35 | 36 | geoip = GeoIP.open("geoip/GeoIP.dat", GeoIP.GEOIP_STANDARD) 37 | 38 | # This is the ES mapping, we mostly need it to mark specific fields as "not_analyzed" 39 | kippo_mapping = { 40 | "client": { 41 | "type": "string", 42 | "index": "not_analyzed" 43 | }, 44 | "country": { 45 | "type": "string" 46 | }, 47 | "id": { 48 | "type": "long" 49 | }, 50 | "ip": { 51 | "type": "string", 52 | "index": "not_analyzed", 53 | "fields": { 54 | "ipv4": { 55 | "type": "ip", 56 | } 57 | } 58 | }, 59 | "log_type": { 60 | "type": "string" 61 | }, 62 | "password": { 63 | "type": "string", 64 | "index": "not_analyzed" 65 | }, 66 | "sensor": { 67 | "type": "string", 68 | "index": "not_analyzed" 69 | }, 70 | "session": { 71 | "type": "string", 72 | "index": "not_analyzed" 73 | }, 74 | "success": { 75 | "type": "boolean" 76 | }, 77 | "timestamp": { 78 | "type": "date", 79 | "format": "dateOptionalTime" 80 | }, 81 | "username": { 82 | "type": "string", 83 | "index": "not_analyzed" 84 | } 85 | } 86 | 87 | # Setup ES connection, flush index and put mapping 88 | es = pyes.ES('{0}:{1}'.format(es_host, es_port)) 89 | es.indices.delete_index_if_exists(es_index) 90 | es.indices.create_index_if_missing(es_index) 91 | es.indices.put_mapping(es_type, {'properties': kippo_mapping}, [es_index]) 92 | 93 | # Parse rows from MySQL, create dict/json for each and index it in ES 94 | for row in rows: 95 | row_dict = collections.OrderedDict() 96 | row_dict['id'] = row[0] 97 | row_dict['log_type'] = "login_attempt" 98 | row_dict['session'] = row[1] 99 | row_dict['success'] = row[2] 100 | row_dict['username'] = row[3] 101 | row_dict['password'] = row[4] 102 | row_dict['timestamp'] = row[5].strftime("%Y-%m-%dT%H:%M:%S") 103 | row_dict['country'] = geoip.country_code_by_addr(row[6]) 104 | row_dict['ip'] = row[6] 105 | row_dict['client'] = row[7] 106 | row_dict['sensor'] = row[8] 107 | auth_json = json.dumps(row_dict) 108 | print auth_json 109 | es.index(auth_json, es_index, es_type) 110 | --------------------------------------------------------------------------------