├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bin └── adenium-2.1.0.jar ├── docs └── scaladocs │ ├── com │ ├── adenium │ │ ├── app │ │ │ ├── config │ │ │ │ ├── Conf$.html │ │ │ │ ├── Conf.html │ │ │ │ ├── ConfZkPaths.html │ │ │ │ ├── Opts$.html │ │ │ │ ├── Opts.html │ │ │ │ └── package.html │ │ │ ├── framework │ │ │ │ ├── AdeniumBroadcast$$Caster.html │ │ │ │ ├── AdeniumBroadcast$$Ref$.html │ │ │ │ ├── AdeniumBroadcast$$Ref.html │ │ │ │ ├── AdeniumBroadcast$.html │ │ │ │ ├── AdeniumBroadcastHelper.html │ │ │ │ ├── AdeniumContext$.html │ │ │ │ ├── AdeniumContext.html │ │ │ │ ├── AdeniumControl$.html │ │ │ │ ├── AdeniumControl.html │ │ │ │ ├── AdeniumMessage.html │ │ │ │ └── package.html │ │ │ ├── logNormalizer │ │ │ │ ├── LNBroadcast$.html │ │ │ │ ├── LNBroadcast.html │ │ │ │ ├── LNBroadcastHelper$.html │ │ │ │ ├── LNBroadcastHelper.html │ │ │ │ ├── LogNormalizer$.html │ │ │ │ ├── LogNormalizerHelper$.html │ │ │ │ └── package.html │ │ │ └── package.html │ │ ├── common │ │ │ ├── FBoolean.html │ │ │ ├── FDateMillis.html │ │ │ ├── FDateString.html │ │ │ ├── FDouble.html │ │ │ ├── FNull$.html │ │ │ ├── Field$.html │ │ │ ├── Field.html │ │ │ ├── FieldFormat$.html │ │ │ ├── FieldFormat.html │ │ │ ├── FieldFormatter$$DateFieldFormatter$.html │ │ │ ├── FieldFormatter$$DatePath$.html │ │ │ ├── FieldFormatter$.html │ │ │ ├── FieldFormatter.html │ │ │ ├── FieldGetter.html │ │ │ ├── FieldKey.html │ │ │ ├── FieldOrder$.html │ │ │ ├── FieldOrder.html │ │ │ └── FieldType$.html │ │ ├── externals │ │ │ ├── kafka │ │ │ │ ├── KfConsumer$.html │ │ │ │ ├── KfConsumer.html │ │ │ │ ├── KfPaths$.html │ │ │ │ ├── KfProducer$.html │ │ │ │ ├── KfProducer.html │ │ │ │ ├── KfQueue$.html │ │ │ │ ├── KfQueue.html │ │ │ │ └── package.html │ │ │ ├── package.html │ │ │ ├── spark │ │ │ │ ├── SparkUtil$$Accumulate.html │ │ │ │ ├── SparkUtil$.html │ │ │ │ └── package.html │ │ │ └── zookeeper │ │ │ │ ├── ZkClient$$Retry$.html │ │ │ │ ├── ZkClient$$Retry.html │ │ │ │ ├── ZkClient$.html │ │ │ │ ├── ZkClient.html │ │ │ │ ├── ZkUtil$.html │ │ │ │ ├── ZkWatcher$.html │ │ │ │ └── package.html │ │ ├── package.html │ │ ├── parser │ │ │ ├── Parser$.html │ │ │ ├── ParserHelper$.html │ │ │ ├── ParserPref$$Implicits$.html │ │ │ ├── ParserPref$.html │ │ │ ├── ParserTester$.html │ │ │ ├── devices │ │ │ │ ├── SOCDeviceKind$.html │ │ │ │ ├── Syslog$$Header3164.html │ │ │ │ ├── Syslog$$Header5424.html │ │ │ │ ├── Syslog$$RFC3164Msg$.html │ │ │ │ ├── Syslog$$RFC3164Msg.html │ │ │ │ ├── Syslog$$RFC5424Msg$.html │ │ │ │ ├── Syslog$$RFC5424Msg.html │ │ │ │ ├── Syslog$$SyslogEncodings.html │ │ │ │ ├── Syslog$$SyslogHeader.html │ │ │ │ ├── Syslog$$SyslogMsg.html │ │ │ │ ├── Syslog$.html │ │ │ │ └── package.html │ │ │ ├── package.html │ │ │ ├── reference │ │ │ │ ├── File2ParserRef$$ParserRefFiles$.html │ │ │ │ ├── File2ParserRef$$ParserRefFiles.html │ │ │ │ ├── File2ParserRef$.html │ │ │ │ ├── File2ParserRef.html │ │ │ │ ├── ParserRef$.html │ │ │ │ ├── ParserRef.html │ │ │ │ ├── ParserRefMaker.html │ │ │ │ ├── SOCServer.html │ │ │ │ ├── SOCServerRegex.html │ │ │ │ ├── Zk2ParserRef.html │ │ │ │ ├── logEncoding.html │ │ │ │ └── package.html │ │ │ └── structs │ │ │ │ ├── Agent$.html │ │ │ │ ├── Agent.html │ │ │ │ ├── ArrangeRule$.html │ │ │ │ ├── ArrangeRule.html │ │ │ │ ├── CompanyHost$.html │ │ │ │ ├── CompanyHost.html │ │ │ │ ├── CompanyIp$.html │ │ │ │ ├── CompanyIp.html │ │ │ │ ├── CompanyIpRange$.html │ │ │ │ ├── CompanyIpRange.html │ │ │ │ ├── GeoIpRange$.html │ │ │ │ ├── GeoIpRange.html │ │ │ │ ├── ReplaceField$.html │ │ │ │ ├── ReplaceField.html │ │ │ │ ├── Signature$.html │ │ │ │ ├── Signature.html │ │ │ │ ├── TokenizeRule$.html │ │ │ │ ├── TokenizeRule.html │ │ │ │ └── package.html │ │ └── utils │ │ │ ├── Args$.html │ │ │ ├── IpUtil$.html │ │ │ ├── Logger$.html │ │ │ ├── May$.html │ │ │ ├── Monoid$.html │ │ │ ├── Monoid.html │ │ │ ├── StringUtil$.html │ │ │ ├── Timer$.html │ │ │ ├── WOption$$WOptionLogOn.html │ │ │ ├── WOption$$WOptionMaker.html │ │ │ ├── WOption$.html │ │ │ ├── WOption.html │ │ │ └── package.html │ └── package.hrml │ ├── document │ ├── index.html │ ├── index.js │ ├── index │ ├── index-_.html │ ├── index-a.html │ ├── index-b.html │ ├── index-c.html │ ├── index-d.html │ ├── index-e.html │ ├── index-f.html │ ├── index-g.html │ ├── index-h.html │ ├── index-i.html │ ├── index-j.html │ ├── index-k.html │ ├── index-l.html │ ├── index-m.html │ ├── index-n.html │ ├── index-o.html │ ├── index-p.html │ ├── index-q.html │ ├── index-r.html │ ├── index-s.html │ ├── index-t.html │ ├── index-u.html │ ├── index-v.html │ ├── index-w.html │ ├── index-x.html │ └── index-z.html │ ├── lib │ ├── arrow-down.png │ ├── arrow-right.png │ ├── class.png │ ├── class_big.png │ ├── class_diagram.png │ ├── class_to_object_big.png │ ├── constructorsbg.gif │ ├── conversionbg.gif │ ├── defbg-blue.gif │ ├── defbg-green.gif │ ├── diagrams.css │ ├── diagrams.js │ ├── filter_box_left.png │ ├── filter_box_left2.gif │ ├── filter_box_right.png │ ├── filterbg.gif │ ├── filterboxbarbg.gif │ ├── filterboxbarbg.png │ ├── filterboxbg.gif │ ├── fullcommenttopbg.gif │ ├── index.css │ ├── index.js │ ├── jquery-ui.js │ ├── jquery.js │ ├── jquery.layout.js │ ├── modernizr.custom.js │ ├── navigation-li-a.png │ ├── navigation-li.png │ ├── object.png │ ├── object_big.png │ ├── object_diagram.png │ ├── object_to_class_big.png │ ├── object_to_trait_big.png │ ├── object_to_type_big.png │ ├── ownderbg2.gif │ ├── ownerbg.gif │ ├── ownerbg2.gif │ ├── package.png │ ├── package_big.png │ ├── packagesbg.gif │ ├── permalink.png │ ├── ref-index.css │ ├── remove.png │ ├── scheduler.js │ ├── selected-implicits.png │ ├── selected-right-implicits.png │ ├── selected-right.png │ ├── selected.png │ ├── selected2-right.png │ ├── selected2.png │ ├── signaturebg.gif │ ├── signaturebg2.gif │ ├── template.css │ ├── template.js │ ├── tools.tooltip.js │ ├── trait.png │ ├── trait_big.png │ ├── trait_diagram.png │ ├── trait_to_object_big.png │ ├── type.png │ ├── type_big.png │ ├── type_diagram.png │ ├── type_to_object_big.png │ ├── typebg.gif │ ├── unselected.png │ └── valuemembersbg.gif │ └── package.html ├── img ├── Adenium_400x105.png ├── figuare1-1-1-1.png ├── figuare1-1-1.png ├── figuare1-1.png ├── figuare1.png ├── figure_1.png ├── figure_2.png └── figure_3.png ├── pom.xml ├── src └── main │ ├── main.iml │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── adenium │ ├── app │ ├── config │ │ ├── Conf.scala │ │ ├── ConfZkPaths.scala │ │ └── Opts.scala │ ├── framework │ │ ├── AdeniumBroadcast.scala │ │ ├── AdeniumBroadcastHelper.scala │ │ ├── AdeniumContext.scala │ │ ├── AdeniumControl.scala │ │ └── AdeniumMessage.scala │ └── logNormalizer │ │ ├── LNBroadcast.scala │ │ └── LogNormalizer.scala │ ├── common │ ├── Field.scala │ ├── FieldFormat.scala │ ├── FieldFormatter.scala │ ├── FieldGetter.scala │ ├── FieldKey.scala │ ├── FieldOrder.scala │ ├── FieldType.scala │ ├── KeyID.scala │ ├── Keys.scala │ ├── Normalized.scala │ ├── Parsed.scala │ ├── VariableKeys.scala │ └── package.scala │ ├── externals │ ├── kafka │ │ ├── KfConsumer.scala │ │ ├── KfPaths.scala │ │ ├── KfProducer.scala │ │ └── KfQueue.scala │ ├── spark │ │ └── SparkUtil.scala │ └── zookeeper │ │ ├── ZkClient.scala │ │ ├── ZkUtil.scala │ │ └── ZkWatcher.scala │ ├── parser │ ├── Parser.scala │ ├── ParserHelper.scala │ ├── ParserPref.scala │ ├── ParserTester.scala │ ├── devices │ │ ├── SOCDeviceKind.scala │ │ └── Syslog.scala │ ├── reference │ │ ├── File2ParserRef.scala │ │ ├── ParserRef.scala │ │ ├── ParserRefMaker.scala │ │ ├── ParserStructs.scala │ │ ├── Zk2ParserRef.scala │ │ └── package.scala │ └── structs │ │ ├── Agent.scala │ │ ├── ArrangeRule.scala │ │ ├── CompanyHost.scala │ │ ├── CompanyIp.scala │ │ ├── CompanyIpRange.scala │ │ ├── GeoIpRange.scala │ │ ├── ReplaceField.scala │ │ ├── Signature.scala │ │ ├── TokenizeRule.scala │ │ └── package.scala │ └── utils │ ├── Args.scala │ ├── IpUtil.scala │ ├── Logger.scala │ ├── May.scala │ ├── StringUtil.scala │ ├── Timer.scala │ ├── WOption.scala │ └── package.scala └── test └── sample ├── AdeniumParser.jar ├── RefData ├── agentInfo.ref ├── arrangeRules.ref ├── companyIpRange.ref ├── companyServerIp.ref ├── fields.ref ├── geoIpRange.ref ├── replaceFields.ref ├── signatures.ref └── tokenizeRules.ref └── Sample_logs └── WebFront.log /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 1. Go to '...' 13 | 2. Click on '....' 14 | 3. Scroll down to '....' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Desktop (please complete the following information):** 24 | - OS: [e.g. iOS] 25 | - Browser [e.g. chrome, safari] 26 | - Version [e.g. 22] 27 | 28 | **Smartphone (please complete the following information):** 29 | - Device: [e.g. iPhone6] 30 | - OS: [e.g. iOS8.1] 31 | - Browser [e.g. stock browser, safari] 32 | - Version [e.g. 22] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guide 2 | 3 | 이 가이드는 Adenium 프로젝트에 기여를 할 수 있는 다양한 방법을 설명합니다. 4 | 오픈소스 프로젝트에 기여한다는 것은 아시는 바와 같이 단지 코드를 작성하는 것에 국한하지 않습니다. 5 | 버그를 리포팅하고 코드 변경을 제안하는 것 못지않게 문서의 오탈자를 수정하거나 다양한 언어로의 번역, 디자인 요소 제안, 릴리즈 테스트 등 모든 관심과 참여를 중요하게 생각하고 있으며 Adenium 프로젝트를 이해하기 위한 질문까지도 환영합니다. 6 | 따라서 아래 내용은 장기적으로 참여하고자 하는 새로운 Contributor를 위한 Use Cases 안내입니다. 7 | 각 기여활동의 성격에 따라 Issues와 Pull requests를 통해 진행하시면 되며 8 | Adenium 오픈소스 프로젝트에 기여함으로써 **[Apache 2.0](https://github.com/SOCLabs/Adenium/blob/master/LICENSE)** 라이선스에 따라 사용 허가 된다는것에 동의하게 됩니다. 감사합니다. 9 | 10 | > **[Issues](https://github.com/SOCLabs/Adenium/issues)** - 버그리포트와 새로운 아이디어 제안
11 | > **[Pull requests](https://github.com/SOCLabs/Adenium/pulls)** - 포크(fork)하여 수정한 저장소의 브랜치를 원래의 저장소에 통합하도록 요청 12 | ---- 13 | - [버그 리포트](#버그-리포트) 14 | - [기능 개선](#기능-개선) 15 | - [문서 개선](#문서-개선) 16 | - [디자인 작업](#디자인-작업) 17 | - [기타](#기타) 18 | 19 | ## 버그 리포트 20 | 21 | 버그를 발견했을때 [Issues](https://github.com/SOCLabs/Adenium/issues) 메뉴를 통해 명확한 제목과 설명을 통해 버그 리포팅을 할 수 있습니다. 22 | 가능한한 많은 관련 정보와 재현방법을 보여준다면 동일한 문제를 가진 다른 사람들이 문제를 해결할때 큰 도움이 됩니다. 23 | 24 | ## 기능 개선 25 | 26 | 새로운 기능을 추가하거나 기존에 올라간 이슈를 처리하기 위해 코드작성을 할 때는 작업 시작을 다른 사람이 알 수 있도록 의견을 달아주시기 바랍니다. 때로는 동일한 작업을 다른 사람이 시작할 수도 있고 이미 다른곳에서 해당 이슈를 해결해 버린 상태 일 수도 있기 때문입니다. 27 | 28 | 29 | ## 문서 개선 30 | 문서 개선은 새로운 메뉴얼을 작성하거나 내용을 추가하여 개선하는 것부터 소스코드를 포함 각종 문서에 존재할 수 있는 오타를 수정하는 일도 컨트리뷰션 활동에 속합니다. 31 | 필요한 문서를 번역하거나 튜토리얼과 같은 가이드 문서를 작성하셨다면 [Pull request](https://github.com/SOCLabs/Adenium/pulls)로 요청해 주시기 바랍니다. 32 | 33 | - **각종 문서 신규작성** 34 | - **오탈자 수정** 35 | - **문서 요청/제안** 36 | - **번역** 37 | 38 | ## 디자인 작업 39 | 40 | 디자인 요소에 대한 기여도 환영합니다. Adenium 프로젝트는 다양성을 위해 공개범위에 UI를 포함하지 않았습니다. UI 디자인 뿐 아니라 특정 이미지 또는 로고, 배너 까지도 언제든 제안해 주시는 모든 이슈를 환영합니다. 41 | 42 | ## 기타 43 | 44 | 코드의 주석위치, 중복된 이슈 정리, 리펙토링, 의견 제시 등 위에 언급하지 않았거나 미처 정의하지 않은 유형이라도 프로젝트에 도움이 되는 그 어떤 사소한 이슈라도 컨트리뷰션 대상이 될 수 있습니다. 45 | -------------------------------------------------------------------------------- /bin/adenium-2.1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/bin/adenium-2.1.0.jar -------------------------------------------------------------------------------- /docs/scaladocs/com/adenium/app/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | app - adenium 2.2.3-a API - com.adenium.app 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 27 | 28 | 29 | 30 |
31 | 32 |

com.adenium

33 |

app

34 | 35 | 36 | 37 | 38 |
39 | 40 |

41 | 42 | 43 | package 44 | 45 | 46 | app 47 | 48 |

49 | 50 |
51 | 52 | 53 |
54 |
55 | 56 | 57 |
58 | Visibility 59 |
  1. Public
  2. All
60 |
61 |
62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 | 70 | 71 |
72 |

Value Members

73 |
  1. 74 | 75 | 76 |

    77 | 78 | 79 | package 80 | 81 | 82 | config 83 | 84 |

    85 | 86 | 87 | 88 | 89 | 90 |
  2. 91 | 92 | 93 |

    94 | 95 | 96 | package 97 | 98 | 99 | framework 100 | 101 |

    102 | 103 | 104 | 105 | 106 | 107 |
  3. 108 | 109 | 110 |

    111 | 112 | 113 | package 114 | 115 | 116 | logNormalizer 117 | 118 |

    119 | 120 | 121 | 122 | 123 | 124 |
125 |
126 | 127 | 128 | 129 | 130 |
131 | 132 |
133 | 134 | 135 |
136 | 137 |
138 |
139 |

Ungrouped

140 | 141 |
142 |
143 | 144 |
145 | 146 |
147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /docs/scaladocs/com/adenium/externals/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | externals - adenium 2.2.3-a API - com.adenium.externals 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 27 | 28 | 29 | 30 |
31 | 32 |

com.adenium

33 |

externals

34 | 35 | 36 | 37 | 38 |
39 | 40 |

41 | 42 | 43 | package 44 | 45 | 46 | externals 47 | 48 |

49 | 50 |
51 | 52 | 53 |
54 |
55 | 56 | 57 |
58 | Visibility 59 |
  1. Public
  2. All
60 |
61 |
62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 | 70 | 71 |
72 |

Value Members

73 |
  1. 74 | 75 | 76 |

    77 | 78 | 79 | package 80 | 81 | 82 | kafka 83 | 84 |

    85 | 86 | 87 | 88 | 89 | 90 |
  2. 91 | 92 | 93 |

    94 | 95 | 96 | package 97 | 98 | 99 | spark 100 | 101 |

    102 | 103 | 104 | 105 | 106 | 107 |
  3. 108 | 109 | 110 |

    111 | 112 | 113 | package 114 | 115 | 116 | zookeeper 117 | 118 |

    119 | 120 | 121 | 122 | 123 | 124 |
125 |
126 | 127 | 128 | 129 | 130 |
131 | 132 |
133 | 134 | 135 |
136 | 137 |
138 |
139 |

Ungrouped

140 | 141 |
142 |
143 | 144 |
145 | 146 |
147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /docs/scaladocs/com/adenium/externals/spark/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | spark - adenium 2.2.3-a API - com.adenium.externals.spark 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 27 | 28 | 29 | 30 |
31 | 32 |

com.adenium.externals

33 |

spark

34 | 35 | 36 | 37 | 38 |
39 | 40 |

41 | 42 | 43 | package 44 | 45 | 46 | spark 47 | 48 |

49 | 50 |
51 | 52 | 53 |
54 |
55 | 56 | 57 |
58 | Visibility 59 |
  1. Public
  2. All
60 |
61 |
62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 | 70 | 71 |
72 |

Value Members

73 |
  1. 74 | 75 | 76 |

    77 | 78 | 79 | object 80 | 81 | 82 | SparkUtil 83 | 84 |

    85 | 86 | 87 | 88 | 89 |

    Spark util wrapped class 90 |

    91 |
92 |
93 | 94 | 95 | 96 | 97 |
98 | 99 |
100 | 101 | 102 |
103 | 104 |
105 |
106 |

Ungrouped

107 | 108 |
109 |
110 | 111 |
112 | 113 |
114 | 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /docs/scaladocs/com/package.hrml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | com - adenium 2.2.3-a API - com 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 27 | 28 | 29 | 30 |
31 | 32 | 33 |

com

34 | 35 | 36 | 37 | 38 |
39 | 40 |

41 | 42 | 43 | package 44 | 45 | 46 | com 47 | 48 |

49 | 50 |
51 | 52 | 53 |
54 |
55 | 56 | 57 |
58 | Visibility 59 |
  1. Public
  2. All
60 |
61 |
62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 | 70 | 71 |
72 |

Value Members

73 |
  1. 74 | 75 | 76 |

    77 | 78 | 79 | package 80 | 81 | 82 | adenium 83 | 84 |

    85 | 86 | 87 | 88 | 89 | 90 |
91 |
92 | 93 | 94 | 95 | 96 |
97 | 98 |
99 | 100 | 101 |
102 | 103 |
104 |
105 |

Ungrouped

106 | 107 |
108 |
109 | 110 |
111 | 112 |
113 | 114 | 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /docs/scaladocs/document: -------------------------------------------------------------------------------- 1 | apidocument 2 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-_.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
+>
15 |
WOptionMaker
16 |
17 |
<+
18 |
ParserRef
19 |
20 |
~>
21 |
WOptionMaker
22 |
23 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-b.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
baseSleepTimeMs
15 |
Retry
16 |
17 |
bc
18 |
Ref
19 |
20 |
body
21 |
RFC3164Msg RFC5424Msg SyslogMsg
22 |
23 |
broadcast
24 |
LNBroadcast
25 |
26 |
broadcaster
27 |
AdeniumBroadcast
28 |
29 |
brokerIds
30 |
KfPaths
31 |
32 |
brokerTopics
33 |
KfPaths
34 |
35 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-e.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
ENDATTACKTIME
15 |
KeyID
16 |
17 |
Encodings
18 |
Syslog
19 |
20 |
eid
21 |
GeoIpRange
22 |
23 |
eip
24 |
CompanyIpRange GeoIpRange
25 |
26 |
empty
27 |
Ref LNBroadcast Monoid
28 |
29 |
encoding
30 |
logEncoding
31 |
32 |
execute
33 |
Parser
34 |
35 |
externals
36 |
adenium
37 |
38 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-h.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
HEADER
15 |
KeyID
16 |
17 |
Header3164
18 |
Syslog
19 |
20 |
Header5424
21 |
Syslog
22 |
23 |
hashmap
24 |
FieldOrder
25 |
26 |
header
27 |
RFC3164Msg RFC5424Msg SyslogMsg
28 |
29 |
helper
30 |
AdeniumControl
31 |
32 |
host
33 |
Parsed Agent
34 |
35 |
host2ip
36 |
ConfZkPaths
37 |
38 |
hostname
39 |
Header3164 Header5424 SyslogHeader SOCServer
40 |
41 |
hostname2Ip
42 |
CompanyHost
43 |
44 |
hour
45 |
Header3164
46 |
47 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-i.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
Implicits
15 |
ParserPref
16 |
17 |
IpUtil
18 |
utils
19 |
20 |
id
21 |
TokenizeRule
22 |
23 |
ignoreVendor
24 |
ReplaceField
25 |
26 |
initialize
27 |
ParserRef ParserRefMaker
28 |
29 |
initializeBroadcast
30 |
AdeniumBroadcastHelper AdeniumControl LNBroadcastHelper
31 |
32 |
instr
33 |
ReplaceField
34 |
35 |
invalidIps
36 |
IpUtil
37 |
38 |
ip
39 |
CompanyHost
40 |
41 |
ip2Long
42 |
IpUtil
43 |
44 |
ip2PublicLong
45 |
IpUtil
46 |
47 |
ipLong
48 |
Agent
49 |
50 |
isCompanyIp
51 |
CompanyIp CompanyIpRange
52 |
53 |
isDefined
54 |
Ref WOption
55 |
56 |
isEmpty
57 |
WOption
58 |
59 |
isEqualKey
60 |
Field
61 |
62 |
isInvalidIp
63 |
IpUtil
64 |
65 |
isSpecialIp
66 |
IpUtil
67 |
68 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-j.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
JSONEscape
15 |
StringUtil
16 |
17 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-k.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
KeyID
15 |
common
16 |
17 |
Keys
18 |
common
19 |
20 |
KfConsumer
21 |
kafka
22 |
23 |
KfPaths
24 |
kafka
25 |
26 |
KfProducer
27 |
kafka
28 |
29 |
KfQueue
30 |
kafka
31 |
32 |
kafka
33 |
externals
34 |
35 |
key
36 |
Field GeoIpRange
37 |
38 |
keyID
39 |
FieldKey
40 |
41 |
keys
42 |
VariableKeys
43 |
44 |
keywords
45 |
Opts
46 |
47 |
kfQueue
48 |
LNBroadcast
49 |
50 |
kf_needRestore
51 |
Conf
52 |
53 |
kf_needSave
54 |
Conf
55 |
56 |
kf_param
57 |
Conf
58 |
59 |
kf_topicerr
60 |
Conf
61 |
62 |
kf_topicout
63 |
Conf
64 |
65 |
kfostr
66 |
Conf Opts
67 |
68 |
kfstr
69 |
Conf Opts
70 |
71 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-l.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
LNBroadcast
15 |
logNormalizer
16 |
17 |
LNBroadcastHelper
18 |
logNormalizer
19 |
20 |
LOGTIME
21 |
KeyID
22 |
23 |
ListMonoid
24 |
Monoid
25 |
26 |
LogNormalizer
27 |
logNormalizer
28 |
29 |
LogNormalizerHelper
30 |
logNormalizer
31 |
32 |
Logger
33 |
utils
34 |
35 |
length
36 |
VariableKeys
37 |
38 |
lift
39 |
May StringUtil
40 |
41 |
locationString
42 |
CompanyIpRange
43 |
44 |
log
45 |
Parsed Accumulate WOption WOptionLogOn
46 |
47 |
logDebug
48 |
Logger
49 |
50 |
logEncoding
51 |
reference
52 |
53 |
logInfo
54 |
Logger
55 |
56 |
logNormalizer
57 |
app
58 |
59 |
logOn
60 |
ParserTester
61 |
62 |
logStatus
63 |
ParserRef
64 |
65 |
logStr
66 |
Accumulate
67 |
68 |
logTime
69 |
FieldGetter
70 |
71 |
logWarning
72 |
Logger
73 |
74 |
logencode
75 |
ConfZkPaths
76 |
77 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-n.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
Normalized
15 |
common
16 |
17 |
name
18 |
FieldKey CompanyHost
19 |
20 |
nation
21 |
GeoIpRange
22 |
23 |
noLog
24 |
WOption
25 |
26 |
nonEmpty
27 |
WOption
28 |
29 |
normal
30 |
SOCDeviceKind
31 |
32 |
normalize
33 |
LogNormalizerHelper
34 |
35 |
normalizer
36 |
ConfZkPaths
37 |
38 |
numeric
39 |
FieldFormatter
40 |
41 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-o.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
Opts
15 |
config
16 |
17 |
offsets
18 |
ConfZkPaths
19 |
20 |
on
21 |
WOptionLogOn
22 |
23 |
onZkChange
24 |
ZkWatcher
25 |
26 |
option
27 |
WOption
28 |
29 |
opts
30 |
Conf ParserTester
31 |
32 |
orElse
33 |
WOption
34 |
35 |
order
36 |
FieldOrder
37 |
38 |
ordered
39 |
Normalized
40 |
41 |
outstr
42 |
ReplaceField
43 |
44 |
overwrite
45 |
ParserHelper
46 |
47 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-q.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
queue
15 |
AdeniumControl
16 |
17 |
queue2Broadcast
18 |
AdeniumControl
19 |
20 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-u.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
URI
15 |
KeyID
16 |
17 |
URIPARAMS
18 |
KeyID
19 |
20 |
URL
21 |
KeyID
22 |
23 |
UnitTimer
24 |
Timer
25 |
26 |
unescapeString
27 |
StringUtil
28 |
29 |
unescapeUnicode
30 |
StringUtil
31 |
32 |
unit
33 |
WOption
34 |
35 |
using
36 |
May
37 |
38 |
utils
39 |
adenium
40 |
41 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-v.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
VENDOR
15 |
KeyID
16 |
17 |
VariableKeys
18 |
common
19 |
20 |
value
21 |
Ref Field
22 |
23 |
valueWithId
24 |
Field
25 |
26 |
varfields
27 |
ConfZkPaths
28 |
29 |
vendor
30 |
ReplaceField Signature
31 |
32 |
vendorId
33 |
Agent
34 |
35 |
vendorName
36 |
Agent
37 |
38 |
version
39 |
Header5424
40 |
41 |
versionHeader
42 |
Normalized ParserTester
43 |
44 |
versionRoot
45 |
ConfZkPaths
46 |
47 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-w.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
WOption
15 |
utils
16 |
17 |
WOptionLogOn
18 |
WOption
19 |
20 |
WOptionMaker
21 |
WOption
22 |
23 |
warn
24 |
May
25 |
26 |
watch
27 |
ConfZkPaths
28 |
29 |
writeLog
30 |
Opts
31 |
32 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-x.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
XMLEscape
15 |
StringUtil
16 |
17 | -------------------------------------------------------------------------------- /docs/scaladocs/index/index-z.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | adenium 2.2.3-a API 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
Zk2ParserRef
15 |
reference
16 |
17 |
ZkClient
18 |
zookeeper
19 |
20 |
ZkUtil
21 |
zookeeper
22 |
23 |
ZkWatcher
24 |
zookeeper
25 |
26 |
zkClient
27 |
AdeniumContext
28 |
29 |
zkCurator
30 |
LNBroadcast
31 |
32 |
zk_nodebyte
33 |
Conf
34 |
35 |
zk_sepItem
36 |
Conf
37 |
38 |
zk_sepLine
39 |
Conf
40 |
41 |
zk_sepLiveDetectItem
42 |
Conf
43 |
44 |
zkp
45 |
Conf
46 |
47 |
zkstr
48 |
Conf Opts
49 |
50 |
zookeeper
51 |
externals
52 |
53 | -------------------------------------------------------------------------------- /docs/scaladocs/lib/arrow-down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/arrow-down.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/arrow-right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/arrow-right.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/class.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/class.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/class_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/class_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/class_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/class_diagram.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/class_to_object_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/class_to_object_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/constructorsbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/constructorsbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/conversionbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/conversionbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/defbg-blue.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/defbg-blue.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/defbg-green.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/defbg-green.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/diagrams.css: -------------------------------------------------------------------------------- 1 | .diagram-container 2 | { 3 | display: none; 4 | } 5 | 6 | .diagram 7 | { 8 | overflow: hidden; 9 | padding-top:15px; 10 | } 11 | 12 | .diagram svg 13 | { 14 | display: block; 15 | position: absolute; 16 | visibility: hidden; 17 | margin: auto; 18 | } 19 | 20 | .diagram-help 21 | { 22 | float:right; 23 | display:none; 24 | } 25 | 26 | .magnifying 27 | { 28 | cursor: -webkit-zoom-in ! important; 29 | cursor: -moz-zoom-in ! important; 30 | cursor: pointer; 31 | } 32 | 33 | #close-link 34 | { 35 | position: absolute; 36 | z-index: 100; 37 | font-family: Arial, sans-serif; 38 | font-size: 10pt; 39 | text-decoration: underline; 40 | color: #315479; 41 | } 42 | 43 | #close:hover 44 | { 45 | text-decoration: none; 46 | } 47 | 48 | svg a 49 | { 50 | cursor:pointer; 51 | } 52 | 53 | svg text 54 | { 55 | font-size: 10px; 56 | } 57 | 58 | /* try to move the node text 1px in order to be vertically 59 | centered (does not work in all browsers) */ 60 | svg .node text 61 | { 62 | transform: translate(0px,1px); 63 | -ms-transform: translate(0px,1px); 64 | -webkit-transform: translate(0px,1px); 65 | -o-transform: translate(0px,1px); 66 | -moz-transform: translate(0px,1px); 67 | } 68 | 69 | /* hover effect for edges */ 70 | 71 | svg .edge.over text, 72 | svg .edge.implicit-incoming.over polygon, 73 | svg .edge.implicit-outgoing.over polygon 74 | { 75 | fill: #202020; 76 | } 77 | 78 | svg .edge.over path, 79 | svg .edge.over polygon 80 | { 81 | stroke: #202020; 82 | } 83 | 84 | /* hover effect for nodes in class diagrams */ 85 | 86 | svg.class-diagram .node 87 | { 88 | opacity: 0.75; 89 | } 90 | 91 | svg.class-diagram .node.this 92 | { 93 | opacity: 1.0; 94 | } 95 | 96 | svg.class-diagram .node.over 97 | { 98 | opacity: 1.0; 99 | } 100 | 101 | svg .node.over polygon 102 | { 103 | stroke: #202020; 104 | } 105 | 106 | /* hover effect for nodes in package diagrams */ 107 | 108 | svg.package-diagram .node.class.over polygon, 109 | svg.class-diagram .node.this.class.over polygon 110 | { 111 | fill: #098552; 112 | fill: #04663e; 113 | } 114 | 115 | svg.package-diagram .node.trait.over polygon, 116 | svg.class-diagram .node.this.trait.over polygon 117 | { 118 | fill: #3c7b9b; 119 | fill: #235d7b; 120 | } 121 | 122 | svg.package-diagram .node.type.over polygon, 123 | svg.class-diagram .node.this.type.over polygon 124 | { 125 | fill: #098552; 126 | fill: #04663e; 127 | } 128 | 129 | 130 | svg.package-diagram .node.object.over polygon 131 | { 132 | fill: #183377; 133 | } 134 | 135 | svg.package-diagram .node.outside.over polygon 136 | { 137 | fill: #d4d4d4; 138 | } 139 | 140 | svg.package-diagram .node.default.over polygon 141 | { 142 | fill: #d4d4d4; 143 | } 144 | -------------------------------------------------------------------------------- /docs/scaladocs/lib/filter_box_left.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filter_box_left.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/filter_box_left2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filter_box_left2.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/filter_box_right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filter_box_right.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/filterbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filterbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/filterboxbarbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filterboxbarbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/filterboxbarbg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filterboxbarbg.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/filterboxbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/filterboxbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/fullcommenttopbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/fullcommenttopbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/modernizr.custom.js: -------------------------------------------------------------------------------- 1 | /* Modernizr 2.5.3 (Custom Build) | MIT & BSD 2 | * Build: http://www.modernizr.com/download/#-inlinesvg 3 | */ 4 | ;window.Modernizr=function(a,b,c){function u(a){i.cssText=a}function v(a,b){return u(prefixes.join(a+";")+(b||""))}function w(a,b){return typeof a===b}function x(a,b){return!!~(""+a).indexOf(b)}function y(a,b,d){for(var e in a){var f=b[a[e]];if(f!==c)return d===!1?a[e]:w(f,"function")?f.bind(d||b):f}return!1}var d="2.5.3",e={},f=b.documentElement,g="modernizr",h=b.createElement(g),i=h.style,j,k={}.toString,l={svg:"http://www.w3.org/2000/svg"},m={},n={},o={},p=[],q=p.slice,r,s={}.hasOwnProperty,t;!w(s,"undefined")&&!w(s.call,"undefined")?t=function(a,b){return s.call(a,b)}:t=function(a,b){return b in a&&w(a.constructor.prototype[b],"undefined")},Function.prototype.bind||(Function.prototype.bind=function(b){var c=this;if(typeof c!="function")throw new TypeError;var d=q.call(arguments,1),e=function(){if(this instanceof e){var a=function(){};a.prototype=c.prototype;var f=new a,g=c.apply(f,d.concat(q.call(arguments)));return Object(g)===g?g:f}return c.apply(b,d.concat(q.call(arguments)))};return e}),m.inlinesvg=function(){var a=b.createElement("div");return a.innerHTML="",(a.firstChild&&a.firstChild.namespaceURI)==l.svg};for(var z in m)t(m,z)&&(r=z.toLowerCase(),e[r]=m[z](),p.push((e[r]?"":"no-")+r));return u(""),h=j=null,e._version=d,e}(this,this.document); -------------------------------------------------------------------------------- /docs/scaladocs/lib/navigation-li-a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/navigation-li-a.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/navigation-li.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/navigation-li.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/object.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/object.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/object_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/object_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/object_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/object_diagram.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/object_to_class_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/object_to_class_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/object_to_trait_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/object_to_trait_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/object_to_type_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/object_to_type_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/ownderbg2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/ownderbg2.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/ownerbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/ownerbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/ownerbg2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/ownerbg2.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/package.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/package.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/package_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/package_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/packagesbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/packagesbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/permalink.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/permalink.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/ref-index.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-size: 10pt; 3 | font-family: Arial, sans-serif; 4 | } 5 | 6 | a { 7 | color:#315479; 8 | } 9 | 10 | .letters { 11 | width:100%; 12 | text-align:center; 13 | margin:0.6em; 14 | padding:0.1em; 15 | border-bottom:1px solid gray; 16 | } 17 | 18 | .entry { 19 | border-bottom: 1px solid lightgray; 20 | padding: 5px 0 8px; 21 | } 22 | 23 | .name { 24 | /* background-color:#E5E5E5; */ 25 | } 26 | 27 | .occurrences { 28 | margin-left: 1em; 29 | margin-top: 5px; 30 | } -------------------------------------------------------------------------------- /docs/scaladocs/lib/remove.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/remove.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/scheduler.js: -------------------------------------------------------------------------------- 1 | // © 2010 EPFL/LAMP 2 | // code by Gilles Dubochet 3 | 4 | function Scheduler() { 5 | var scheduler = this; 6 | var resolution = 0; 7 | this.timeout = undefined; 8 | this.queues = new Array(0); // an array of work pacakges indexed by index in the labels table. 9 | this.labels = new Array(0); // an indexed array of labels indexed by priority. This should be short. 10 | this.label = function(name, priority) { 11 | this.name = name; 12 | this.priority = priority; 13 | } 14 | this.work = function(fn, self, args) { 15 | this.fn = fn; 16 | this.self = self; 17 | this.args = args; 18 | } 19 | this.addLabel = function(name, priority) { 20 | var idx = 0; 21 | while (idx < scheduler.queues.length && scheduler.labels[idx].priority <= priority) { idx = idx + 1; } 22 | scheduler.labels.splice(idx, 0, new scheduler.label(name, priority)); 23 | scheduler.queues.splice(idx, 0, new Array(0)); 24 | } 25 | this.clearLabel = function(name) { 26 | var idx = 0; 27 | while (idx < scheduler.queues.length && scheduler.labels[idx].name != name) { idx = idx + 1; } 28 | if (idx < scheduler.queues.length && scheduler.labels[i].name == name) { 29 | scheduler.labels.splice(idx, 1); 30 | scheduler.queues.splice(idx, 1); 31 | } 32 | } 33 | this.nextWork = function() { 34 | var fn = undefined; 35 | var idx = 0; 36 | while (idx < scheduler.queues.length && scheduler.queues[idx].length == 0) { idx = idx + 1; } 37 | if (idx < scheduler.queues.length && scheduler.queues[idx].length > 0) { 38 | var fn = scheduler.queues[idx].shift(); 39 | } 40 | return fn; 41 | } 42 | this.add = function(labelName, fn, self, args) { 43 | var doWork = function() { 44 | scheduler.timeout = setTimeout(function() { 45 | var work = scheduler.nextWork(); 46 | if (work != undefined) { 47 | if (work.args == undefined) { work.args = new Array(0); } 48 | work.fn.apply(work.self, work.args); 49 | doWork(); 50 | } 51 | else { 52 | scheduler.timeout = undefined; 53 | } 54 | }, resolution); 55 | } 56 | var idx = 0; 57 | while (idx < scheduler.labels.length && scheduler.labels[idx].name != labelName) { idx = idx + 1; } 58 | if (idx < scheduler.queues.length && scheduler.labels[idx].name == labelName) { 59 | scheduler.queues[idx].push(new scheduler.work(fn, self, args)); 60 | if (scheduler.timeout == undefined) doWork(); 61 | } 62 | else throw("queue for add is non existant"); 63 | } 64 | this.clear = function(labelName) { 65 | var idx = 0; 66 | while (idx < scheduler.labels.length && scheduler.labels[idx].name != labelName) { idx = idx + 1; } 67 | if (idx < scheduler.queues.length && scheduler.labels[idx].name == labelName) { 68 | scheduler.queues[idx] = new Array(); 69 | } 70 | } 71 | }; 72 | -------------------------------------------------------------------------------- /docs/scaladocs/lib/selected-implicits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/selected-implicits.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/selected-right-implicits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/selected-right-implicits.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/selected-right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/selected-right.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/selected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/selected.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/selected2-right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/selected2-right.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/selected2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/selected2.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/signaturebg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/signaturebg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/signaturebg2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/signaturebg2.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/tools.tooltip.js: -------------------------------------------------------------------------------- 1 | /* 2 | * tools.tooltip 1.1.3 - Tooltips done right. 3 | * 4 | * Copyright (c) 2009 Tero Piirainen 5 | * http://flowplayer.org/tools/tooltip.html 6 | * 7 | * Dual licensed under MIT and GPL 2+ licenses 8 | * http://www.opensource.org/licenses 9 | * 10 | * Launch : November 2008 11 | * Date: ${date} 12 | * Revision: ${revision} 13 | */ 14 | (function(c){var d=[];c.tools=c.tools||{};c.tools.tooltip={version:"1.1.3",conf:{effect:"toggle",fadeOutSpeed:"fast",tip:null,predelay:0,delay:30,opacity:1,lazy:undefined,position:["top","center"],offset:[0,0],cancelDefault:true,relative:false,oneInstance:true,events:{def:"mouseover,mouseout",input:"focus,blur",widget:"focus mouseover,blur mouseout",tooltip:"mouseover,mouseout"},api:false},addEffect:function(e,g,f){b[e]=[g,f]}};var b={toggle:[function(e){var f=this.getConf(),g=this.getTip(),h=f.opacity;if(h<1){g.css({opacity:h})}g.show();e.call()},function(e){this.getTip().hide();e.call()}],fade:[function(e){this.getTip().fadeIn(this.getConf().fadeInSpeed,e)},function(e){this.getTip().fadeOut(this.getConf().fadeOutSpeed,e)}]};function a(f,g){var p=this,k=c(this);f.data("tooltip",p);var l=f.next();if(g.tip){l=c(g.tip);if(l.length>1){l=f.nextAll(g.tip).eq(0);if(!l.length){l=f.parent().nextAll(g.tip).eq(0)}}}function o(u){var t=g.relative?f.position().top:f.offset().top,s=g.relative?f.position().left:f.offset().left,v=g.position[0];t-=l.outerHeight()-g.offset[0];s+=f.outerWidth()+g.offset[1];var q=l.outerHeight()+f.outerHeight();if(v=="center"){t+=q/2}if(v=="bottom"){t+=q}v=g.position[1];var r=l.outerWidth()+f.outerWidth();if(v=="center"){s-=r/2}if(v=="left"){s-=r}return{top:t,left:s}}var i=f.is(":input"),e=i&&f.is(":checkbox, :radio, select, :button"),h=f.attr("type"),n=g.events[h]||g.events[i?(e?"widget":"input"):"def"];n=n.split(/,\s*/);if(n.length!=2){throw"Tooltip: bad events configuration for "+h}f.bind(n[0],function(r){if(g.oneInstance){c.each(d,function(){this.hide()})}var q=l.data("trigger");if(q&&q[0]!=this){l.hide().stop(true,true)}r.target=this;p.show(r);n=g.events.tooltip.split(/,\s*/);l.bind(n[0],function(){p.show(r)});if(n[1]){l.bind(n[1],function(){p.hide(r)})}});f.bind(n[1],function(q){p.hide(q)});if(!c.browser.msie&&!i&&!g.predelay){f.mousemove(function(){if(!p.isShown()){f.triggerHandler("mouseover")}})}if(g.opacity<1){l.css("opacity",g.opacity)}var m=0,j=f.attr("title");if(j&&g.cancelDefault){f.removeAttr("title");f.data("title",j)}c.extend(p,{show:function(r){if(r){f=c(r.target)}clearTimeout(l.data("timer"));if(l.is(":animated")||l.is(":visible")){return p}function q(){l.data("trigger",f);var t=o(r);if(g.tip&&j){l.html(f.data("title"))}r=r||c.Event();r.type="onBeforeShow";k.trigger(r,[t]);if(r.isDefaultPrevented()){return p}t=o(r);l.css({position:"absolute",top:t.top,left:t.left});var s=b[g.effect];if(!s){throw'Nonexistent effect "'+g.effect+'"'}s[0].call(p,function(){r.type="onShow";k.trigger(r)})}if(g.predelay){clearTimeout(m);m=setTimeout(q,g.predelay)}else{q()}return p},hide:function(r){clearTimeout(l.data("timer"));clearTimeout(m);if(!l.is(":visible")){return}function q(){r=r||c.Event();r.type="onBeforeHide";k.trigger(r);if(r.isDefaultPrevented()){return}b[g.effect][1].call(p,function(){r.type="onHide";k.trigger(r)})}if(g.delay&&r){l.data("timer",setTimeout(q,g.delay))}else{q()}return p},isShown:function(){return l.is(":visible, :animated")},getConf:function(){return g},getTip:function(){return l},getTrigger:function(){return f},bind:function(q,r){k.bind(q,r);return p},onHide:function(q){return this.bind("onHide",q)},onBeforeShow:function(q){return this.bind("onBeforeShow",q)},onShow:function(q){return this.bind("onShow",q)},onBeforeHide:function(q){return this.bind("onBeforeHide",q)},unbind:function(q){k.unbind(q);return p}});c.each(g,function(q,r){if(c.isFunction(r)){p.bind(q,r)}})}c.prototype.tooltip=function(e){var f=this.eq(typeof e=="number"?e:0).data("tooltip");if(f){return f}var g=c.extend(true,{},c.tools.tooltip.conf);if(c.isFunction(e)){e={onBeforeShow:e}}else{if(typeof e=="string"){e={tip:e}}}e=c.extend(true,g,e);if(typeof e.position=="string"){e.position=e.position.split(/,?\s/)}if(e.lazy!==false&&(e.lazy===true||this.length>20)){this.one("mouseover",function(h){f=new a(c(this),e);f.show(h);d.push(f)})}else{this.each(function(){f=new a(c(this),e);d.push(f)})}return e.api?f:this}})(jQuery); -------------------------------------------------------------------------------- /docs/scaladocs/lib/trait.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/trait.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/trait_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/trait_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/trait_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/trait_diagram.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/trait_to_object_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/trait_to_object_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/type.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/type.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/type_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/type_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/type_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/type_diagram.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/type_to_object_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/type_to_object_big.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/typebg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/typebg.gif -------------------------------------------------------------------------------- /docs/scaladocs/lib/unselected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/unselected.png -------------------------------------------------------------------------------- /docs/scaladocs/lib/valuemembersbg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/docs/scaladocs/lib/valuemembersbg.gif -------------------------------------------------------------------------------- /docs/scaladocs/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | root - adenium 2.2.3-a API - _root_ 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 27 | 28 | 29 | 30 |
31 | 32 | 33 |

root package

34 | 35 | 36 | 37 | 38 |
39 | 40 |

41 | 42 | 43 | package 44 | 45 | 46 | root 47 | 48 |

49 | 50 |
51 | 52 | 53 |
54 |
55 | 56 | 57 |
58 | Visibility 59 |
  1. Public
  2. All
60 |
61 |
62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 | 70 | 71 |
72 |

Value Members

73 |
  1. 74 | 75 | 76 |

    77 | 78 | 79 | package 80 | 81 | 82 | com 83 | 84 |

    85 | 86 | 87 | 88 | 89 | 90 |
91 |
92 | 93 | 94 | 95 | 96 |
97 | 98 |
99 | 100 | 101 |
102 | 103 |
104 |
105 |

Ungrouped

106 | 107 |
108 |
109 | 110 |
111 | 112 |
113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /img/Adenium_400x105.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/Adenium_400x105.png -------------------------------------------------------------------------------- /img/figuare1-1-1-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figuare1-1-1-1.png -------------------------------------------------------------------------------- /img/figuare1-1-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figuare1-1-1.png -------------------------------------------------------------------------------- /img/figuare1-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figuare1-1.png -------------------------------------------------------------------------------- /img/figuare1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figuare1.png -------------------------------------------------------------------------------- /img/figure_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figure_1.png -------------------------------------------------------------------------------- /img/figure_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figure_2.png -------------------------------------------------------------------------------- /img/figure_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/img/figure_3.png -------------------------------------------------------------------------------- /src/main/main.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Log4j Setting file 2 | log4j.rootLogger=DEBUG, console, logfile 3 | 4 | # Console log 5 | log4j.appender.console=org.apache.log4j.ConsoleAppender 6 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 7 | 8 | # Daily file log 9 | log4j.appender.logfile=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.logfile.File=D:/Data/Log/data.log 11 | log4j.appender.logfile.DatePattern='.'yyyy-MM-dd 12 | log4j.appender.logfile.layout=org.apache.log4j.PatternLayout 13 | log4j.appender.logfile.layout.ConversionPattern=[%d{HH:mm:ss}][%-5p] %m%n 14 | 15 | 16 | 17 | # log level and appender 18 | #log4j.logger.name.of.the.package.one=DEBUG, console 19 | log4j.logger.org.apache.hive.jdbc.HiveQueryResultSet=ERROR 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/config/Conf.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.config 2 | 3 | import com.adenium.externals.kafka.KfConsumer 4 | import com.adenium.externals.zookeeper.ZkClient 5 | import com.adenium.utils.May.{maybeWarn, maybe} 6 | import org.apache.curator.framework.CuratorFramework 7 | 8 | /** 9 | * Contains configuration information for the behavior of the Adenium framework. 10 | * @param zkstr Zookeeper node 11 | * @param kfstr Kafka brokers 12 | * @param topic Kafka Topics 13 | * @param opts Options 14 | */ 15 | case class Conf( zkstr: String, 16 | kfstr: String, 17 | topic: String, 18 | opts: Opts ) 19 | { 20 | 21 | 22 | // kafka output broker : for migration 23 | lazy val kfostr: String = opts.kfostr getOrElse kfstr 24 | 25 | ////////////////////////////////////// 26 | lazy val zkp: ConfZkPaths = ConfZkPaths ( opts.app) 27 | 28 | ////////////////////////////////////// 29 | lazy val sp_master: String = opts.get( "sp:master") 30 | lazy val sp_app: String = opts.get( "sp:app") 31 | lazy val sp_batchDuration: String = opts.get( "sp:duration") 32 | lazy val sp_bulksize: Int = opts.getOption("sp:bulk").flatMap(v => maybe( v.toInt)).getOrElse( 512 ) 33 | 34 | ////////////////////////////////////// 35 | lazy val ctrl_topic: Option[String] = opts.getOption( "kf:ctrl") 36 | lazy val ctrl_zknode: String = zkp.watch 37 | 38 | ////////////////////////////////////// 39 | lazy val kf_param: Map[ String, String ] = Map[ String, String ]( "metadata.broker.list" -> kfstr ) 40 | lazy val kf_needRestore: Option[Conf] = opts.getOption( "kf:restore").map(_ => this) 41 | lazy val kf_needSave: Option[Conf] = Some(this) //opts.getOption( "kf:save").map(_ => this) 42 | lazy val kf_topicout: String = opts.get("kf:out_topic") 43 | lazy val kf_topicerr: Option[String] = opts.getOption("kf:err_topic") 44 | 45 | } 46 | 47 | object Conf { 48 | 49 | def apply( opts: Opts): Conf = { 50 | opts.writeLog("====================== Conf log ======================") 51 | Conf( opts.zkstr, opts.kfstr, opts.topic, opts) 52 | } 53 | 54 | def apply( args: Array[String]): Conf = Conf.apply( Opts(args)) 55 | 56 | /** 57 | * Monitors adenium control messages entered in the specified kafka topic. 58 | * @param conf Adenium config 59 | * @return 60 | */ 61 | def ctrlReader( conf: Conf) 62 | : Option[ KfConsumer] = 63 | conf.ctrl_topic.flatMap { topic => 64 | maybeWarn { KfConsumer( topic, conf.sp_app, conf.zkstr, readFromStartOfStream = false ) } 65 | } 66 | 67 | /** 68 | * Monitors the adenium control messages entered in the watche node. 69 | * @param conf 70 | * @return 71 | */ 72 | def ctrlWatcher( conf: Conf) 73 | : Option[(CuratorFramework, String)] 74 | = ZkClient(conf.zkstr).curator.map ( _ -> conf.ctrl_zknode ) 75 | 76 | /// don't change 77 | //////////////////////////////////////////////////////////////////// 78 | val zk_nodebyte = 524255 // 512 KByte 79 | val zk_sepLine ="\n" 80 | val zk_sepItem ="\t" 81 | val zk_sepLiveDetectItem = "\t:\t" 82 | // val zk_sepLiveDetectItem = "," 83 | } 84 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/config/ConfZkPaths.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.config 2 | 3 | /** 4 | * Defines the path of the ZooKeeper node where the control messages and reference information of the Adenium framework are stored. 5 | * @param appName application name 6 | * @param versionRoot 7 | */ 8 | case class ConfZkPaths( appName: String, versionRoot: String= "/Adenium/2.0") { 9 | 10 | /** 11 | * Zookeeper Properties structure 12 | * 13 | * VersionRoot 14 | * |-- app 15 | * | |-- watch 16 | * | |-- offsets 17 | * | |-- common // added 10.17, g3nie 18 | * | | |-- var_fields // moved from ( normalizer -- parser_ref -- fields ) 19 | * | |-- normalizer 20 | * | | |-- parser_log 21 | * | | |-- parser_ref 22 | * | | | |-- geo_ip_range 23 | * | | | |-- company_ip_range 24 | * | | | |-- signatures 25 | * | | | |-- agents 26 | * | | | |-- tokenize_rules 27 | * | | | |-- arrange_rules 28 | * | | | |-- fields <----- may need to be located in other path... 29 | * | | | |-- replace_fields 30 | * | | | |-- company_ips 31 | */ 32 | 33 | val app: String = versionRoot + "/" + "app" 34 | 35 | val watch: String = app + "/" + "watch" + "/" + appName 36 | val offsets: String = app + "/" + "offsets" + "/" + appName 37 | val common: String = app + "/" + "common" 38 | val varfields: String = app + "/" + "var_fields" // added 10.17 39 | 40 | val normalizer: String = app + "/" + "normalizer" 41 | val parserlog: String = normalizer + "/" + "parser_log" 42 | val parser: String = normalizer + "/" + "parser_ref" 43 | val geoip: String = parser + "/" + "geo_ip_range" 44 | val cipr: String = parser + "/" + "company_ip_range" 45 | val sig: String = parser + "/" + "signatures" 46 | val agn: String = parser + "/" + "agents" 47 | val tokrul: String = parser + "/" + "tokenize_rules" 48 | val arrrul: String = parser + "/" + "arrange_rules" 49 | val fields: String = parser + "/" + "fields" // <-- deprecating 10.17, g3nie 50 | val repfld: String = parser + "/" + "replace_fields" 51 | val cips: String = parser + "/" + "company_ips" 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/config/Opts.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.config 2 | 3 | import com.adenium.utils.Args.args2map 4 | import com.adenium.utils.Logger 5 | 6 | /** 7 | * Control the execution parameters and options of the Adenium framework. 8 | * @param args arguments 9 | * @param keywords Predefined Option key 10 | */ 11 | case class Opts( args: Array[String], keywords: String = Opts.keywords ) { 12 | 13 | private lazy val optMap: Map[String, Array[String ] ] = args2map( args, keywords ) 14 | 15 | private lazy val log: String = optMap 16 | .map( v => v._1 -> v._2.mkString("\t")) 17 | .mkString( "\n=== Opts ( option -> value ) ===\n", "\n", "\n=============================" ) 18 | 19 | def getOption( opt: String): Option[String] = optMap.get( opt).map(_.mkString(",") ) 20 | 21 | ////////////////////////////////////////////////// 22 | lazy val zkstr: String = get( "zk:conn" ) 23 | lazy val kfstr: String = get( "kf:broker") 24 | lazy val kfostr: Option[String] = getOption( "kf:outbroker") 25 | lazy val topic: String = get( "kf:topic") 26 | lazy val app: String= get( "sp:app") 27 | 28 | ////////////////////////////////////////////////// 29 | 30 | /** 31 | * Outputs the received options. 32 | * @param head 33 | */ 34 | def writeLog(head: String): Unit = Logger.logInfo( head + log) 35 | 36 | /** Gets the value of the option. 37 | * 38 | * @param opt option key 39 | * @param default default value 40 | * @return option value 41 | */ 42 | def get( opt: String, default: Option[String] = None ): String = 43 | optMap.get( opt).map( _.mkString(",") ) 44 | .getOrElse { 45 | default 46 | .getOrElse { 47 | throw new Exception(s"""\n$opt in not set. (${args.mkString("\t")})\n($optMap)\n""") 48 | "check_options" 49 | } 50 | } 51 | } 52 | 53 | /** 54 | * Predefined Option key 55 | */ 56 | object Opts { 57 | 58 | val keywords: String = 59 | "sp:master|sp:app|sp:duration|sp:bulk|" + 60 | "zk:conn|zk:ctrl|" + 61 | "kf:broker|kf:outbroker|kf:topic|kf:ctrl|kf:restore|kf:save|kf:out_topic|kf:err_topic|" 62 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/framework/AdeniumBroadcast.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.framework 2 | 3 | import com.adenium.utils.May.maybeWarn 4 | import org.apache.spark.SparkContext 5 | import org.apache.spark.broadcast.Broadcast 6 | 7 | import scala.reflect.ClassTag 8 | 9 | /** 10 | * Defines and handles "spark broadcast variables". 11 | */ 12 | object AdeniumBroadcast { 13 | 14 | case class Ref[A]( value: Option[A], bc: Option[Broadcast[A]]) { 15 | def status: (Boolean, Boolean) = ( value.isDefined, bc.isDefined ) 16 | def isDefined: Boolean= value.isDefined 17 | } 18 | 19 | object Ref { 20 | def empty[A]: Ref[A] = Ref[A]( None, None) 21 | def apply[A]( a: => A): Ref[A] = Ref( maybeWarn(a), None) 22 | } 23 | 24 | trait Caster { 25 | def apply[A:ClassTag]( neo: Ref[A], old: Ref[A]): Ref[A] 26 | } 27 | 28 | /** 29 | * Update "Spark broadcast Variables". 30 | * You can set blocking options that control the placement of workers during the update of broadcast variables. 31 | * true: The batch operation is aborted while the broadcast variable is being updated. 32 | * false: The batch job is asynchronous while the broadcast variable is being updated. 33 | * 34 | * @param blocking true/false 35 | * @param sparkContext 36 | * @return Caster 37 | */ 38 | def broadcaster( blocking: Boolean, sparkContext: SparkContext) 39 | : Caster = new Caster { 40 | 41 | override def apply[A:ClassTag](n: Ref[A], o: Ref[A]) 42 | : Ref[A] = update[A](blocking, sparkContext)(n, o) 43 | 44 | } 45 | 46 | private def update[A:ClassTag]( blocking: Boolean, sparkContext: SparkContext) 47 | ( neo: Ref[A], old: Ref[A]) : Ref[A] = { 48 | 49 | neo.value.map { up => 50 | 51 | old.bc.foreach(_.unpersist(blocking)) 52 | val b = maybeWarn( sparkContext.broadcast(up)) 53 | val r = Ref( neo.value, b) 54 | r 55 | 56 | } getOrElse old 57 | } 58 | 59 | 60 | private def getBroadcastedRef[A](broadcasted: Ref[A] ) 61 | : Option[A] = maybeWarn( broadcasted.bc.map(_.value) ).flatten 62 | 63 | /** 64 | * Get the value from the broadcasted variable. 65 | * 66 | * @param current Spark Broadcast value 67 | * @param f Casting function 68 | * @tparam A 69 | * @tparam B 70 | * @return 71 | */ 72 | def getBroadcast[A,B](current: Option[A])(f: A => Ref[B]) 73 | :Option[B] = current.flatMap( itm => AdeniumBroadcast.getBroadcastedRef( f(itm))) 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/framework/AdeniumBroadcastHelper.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.framework 2 | 3 | import org.apache.spark.SparkContext 4 | 5 | trait AdeniumBroadcastHelper[T] { 6 | 7 | /** 8 | * Initializes the broadcast variable. 9 | * @param sparkContext 10 | * @return 11 | */ 12 | def initializeBroadcast( sparkContext: SparkContext): T 13 | 14 | /** 15 | * Update the broadcast variable. 16 | * 17 | * @param current 18 | * @param update 19 | * @param blocking 20 | * @param sparkContext 21 | * @return 22 | */ 23 | def reference2Broadcast( current:T, update: T, blocking: Boolean)(sparkContext: SparkContext): T 24 | 25 | /** 26 | * Handles Adenium control commands. 27 | * Updates the broadcast variable according to the defined message. 28 | * @param message 29 | * @return 30 | */ 31 | def command2Reference( message: String ) : Option[T] 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/framework/AdeniumContext.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.framework 2 | 3 | import _root_.kafka.message.MessageAndMetadata 4 | import _root_.kafka.serializer.StringDecoder 5 | import com.adenium.app.config.Conf 6 | import com.adenium.externals.spark.SparkUtil.Accumulate 7 | import com.adenium.externals.zookeeper.ZkClient 8 | import com.adenium.utils.Logger 9 | import com.adenium.utils.May._ 10 | import kafka.common.TopicAndPartition 11 | import org.apache.curator.framework.CuratorFramework 12 | import org.apache.spark.streaming.dstream.InputDStream 13 | import org.apache.spark.streaming.kafka.{KafkaUtils, OffsetRange} 14 | import org.apache.spark.streaming.{Milliseconds, StreamingContext} 15 | import org.apache.spark.{SparkConf, SparkContext} 16 | 17 | /** 18 | * The Adenium context includes "spark context", "spark direct stream", "engine configuration". 19 | * AdenuimContext creates spark context, spark direct stream, and zookeeper client. 20 | * spark direct stream can be created from a specific offset of kafka topic and provides a method for it. 21 | * 22 | */ 23 | 24 | case class AdeniumContext( sparkContext: SparkContext, 25 | streamContext: StreamingContext, 26 | zkClient: ZkClient, 27 | conf: Conf ) { 28 | 29 | /** 30 | * 31 | * @param acs 32 | * @return 33 | */ 34 | def createAccumulators( acs: Seq[String]) 35 | : Accumulate = { 36 | Logger.logWarning( s"[ AdeniumContext ] createAccumulators : $acs") 37 | Accumulate( acs map sparkContext.longAccumulator) 38 | } 39 | 40 | /** 41 | * Save Kafka's current offset to the zookeeper. 42 | * @param offsets 43 | * @return 44 | */ 45 | def saveOffsets( offsets: => Array[OffsetRange]): Option[String] 46 | = { 47 | 48 | conf.kf_needSave 49 | .flatMap { conf => 50 | 51 | lazy val off = offsets 52 | 53 | Logger.logWarning( s"[ AdeniumContext ] saveOffsets to ${conf.zkp.offsets}\t" + 54 | s"""${off.map( _.toString() ).mkString("\n")}""" ) 55 | 56 | this.zkClient.setOffsets( conf.zkp.offsets, conf.topic, off) 57 | } 58 | } 59 | 60 | /** 61 | * 62 | * @param path 63 | * @return 64 | */ 65 | def getConfiguration( path: String): Option[String] = { 66 | zkClient.getZkString( path) 67 | } 68 | } 69 | 70 | object AdeniumContext { 71 | 72 | /** 73 | * AdeniumContext constructor 74 | * 75 | * @param conf : engine configuration [[com.adenium.app.config.Conf]] 76 | * @param registerClasses : Function to register class to be serialized 77 | * @return 78 | */ 79 | def apply(conf: Conf, registerClasses: SparkConf => SparkConf) 80 | : AdeniumContext = { 81 | 82 | Logger.logInfo( "[ AdeniumContext ] spark (master, app, batchDuration) =" + 83 | s" ${(conf.sp_master, conf.sp_app, conf.sp_batchDuration)}" ) 84 | 85 | val sparkConf = new SparkConf() 86 | .setMaster(conf.sp_master) 87 | .setAppName(conf.sp_app) 88 | 89 | registerClasses( sparkConf) 90 | 91 | val context = new SparkContext( sparkConf) 92 | 93 | val streamcontext = new StreamingContext( context, Milliseconds( conf.sp_batchDuration.toLong)) 94 | 95 | val zkCurator= ZkClient( conf.zkstr) 96 | 97 | AdeniumContext(context, streamcontext, zkCurator, conf) 98 | } 99 | 100 | 101 | /** 102 | * create an InputDStream using a streamingContext defined in AdeniumContext 103 | * 104 | * @param context : AdeniumContext [[com.adenium.app.framework.AdeniumContext]] 105 | * @param conf : configurations [[com.adenium.app.config.Conf]] 106 | * @return 107 | */ 108 | def createDirectStream(context: AdeniumContext, conf: Conf ) 109 | : InputDStream[(String, String)] = { 110 | 111 | val sc = context.streamContext 112 | val zk = context.zkClient 113 | 114 | /** 115 | * If the first parameter of readOffsetsIfNeeded is true, reads the last offset stored in the zookeeper. 116 | * This offset is used to create an InputDStream. If there is no offset, 117 | * InputDStream is created using the current offsets. 118 | */ 119 | val offsets = AdeniumContext.readOffsetsIfNeeded(conf, zk) 120 | 121 | offsets.flatMap{ from => 122 | 123 | maybeInfo { 124 | val handler = (msg: MessageAndMetadata[String, String]) => (msg.key(), msg.message()) 125 | 126 | Logger.logInfo("[ createDStream ] ( broker, restore) : " + (conf.kfstr, from)) 127 | KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder, (String, String)]( 128 | sc, conf.kf_param, from, handler) 129 | 130 | }("[ createDStream ] can't restore from zk's offsets.. so, i'll try from current") 131 | 132 | } getOrElse { 133 | 134 | Logger.logInfo("[ createDStream ] (broker, current) : " + (conf.kfstr, conf.topic)) 135 | KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder]( 136 | sc, conf.kf_param, conf.topic.split(",").toSet) 137 | } 138 | } 139 | 140 | /** 141 | * Read the last offset stored in the zookeeper. 142 | * @param conf : configuration 143 | * @param zkCurator : zookeeper client 144 | * @return 145 | */ 146 | private def readOffsetsIfNeeded( conf: Conf, zkCurator: ZkClient) 147 | : Option[Map[TopicAndPartition, Long ] ] = { 148 | 149 | conf.kf_needRestore 150 | .flatMap { conf => 151 | zkCurator.getOffsets( conf.zkp.offsets, conf.topic) 152 | } 153 | } 154 | } 155 | 156 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/framework/AdeniumControl.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.framework 2 | 3 | import java.util.concurrent.ConcurrentLinkedQueue 4 | 5 | import com.adenium.app.config.Conf 6 | import com.adenium.externals.kafka.KfConsumer 7 | import com.adenium.externals.zookeeper.ZkWatcher 8 | import com.adenium.utils.Logger 9 | import com.adenium.utils.May._ 10 | import org.apache.curator.framework.CuratorFramework 11 | import org.apache.spark.SparkContext 12 | 13 | import scala.concurrent.Future 14 | 15 | /** 16 | * It monitors the control commands of Adenium Framework and performs control actions according to messages. 17 | */ 18 | 19 | case class AdeniumControl[A](queue: ConcurrentLinkedQueue[A], helper: AdeniumBroadcastHelper[A] ) { 20 | 21 | private var currentValue : A = _ 22 | 23 | private def push(msg: A): Boolean = queue.offer(msg) 24 | private def pop(): Option[A] = { 25 | val ret = Option( queue.poll()) 26 | ret.foreach ( r => state( s" AdeniumControl == pop message: $r") ) 27 | ret 28 | } 29 | 30 | /** 31 | * Execute AdeniumControl. 32 | * Call the initializeBroadcast method to initialize the broadcast variable and execute the Control Message loop. 33 | * 34 | * @param sparkContext 35 | * @param conf 36 | * @return 37 | */ 38 | def start( sparkContext: SparkContext, conf: Conf): AdeniumControl[A] = { 39 | 40 | state("[ AdeniumControl ] == start ......... ") 41 | 42 | maybeWarn2 { 43 | 44 | initializeBroadcast( sparkContext) 45 | 46 | Conf.ctrlReader( conf) 47 | .foreach ( AdeniumControl.startKfMsgLoop( this) ) 48 | 49 | Conf.ctrlWatcher( conf) 50 | .foreach { case (curator, path) => AdeniumControl.startZkMsgLoop(this)(curator, path) } 51 | 52 | }("[ AdeniumControl ] == is not successfully started.... ") 53 | 54 | this 55 | } 56 | 57 | /** 58 | * initialize the broadcast variable 59 | * @param sparkContext 60 | * @return 61 | */ 62 | def initializeBroadcast(sparkContext: SparkContext): A = { 63 | 64 | state("[ AdeniumControl - == initializeBroadcast ] ......... ") 65 | 66 | currentValue = helper.initializeBroadcast( sparkContext) 67 | state(s"[ AdeniumControl - == initializeBroadcast ] status : $currentValue ") 68 | 69 | currentValue 70 | } 71 | 72 | /** 73 | * Broadcasts the updated data to the Queue. 74 | * 75 | * @param sparkContext 76 | * @param blocking 77 | * @return 78 | */ 79 | def queue2Broadcast(sparkContext: => SparkContext, blocking: Boolean = true) 80 | : Option[A] = { 81 | 82 | pop().map { ref => 83 | 84 | state(s"[ AdeniumControl - message2Broadcast ] == message poped1 = $ref ......... ") 85 | 86 | val ret = helper.reference2Broadcast(currentValue, ref, blocking )(sparkContext) 87 | state(s"[ AdeniumControl - message2Broadcast ] == message poped2 = $ret ......... ") 88 | 89 | ret 90 | 91 | }.foreach ( currentValue = _ ) 92 | 93 | state(s"[ AdeniumControl - message2Broadcast ] == message poped3 ......... $currentValue") 94 | Some( currentValue ) 95 | 96 | } 97 | 98 | } 99 | 100 | object AdeniumControl { 101 | 102 | import scala.concurrent.ExecutionContext.Implicits.global 103 | 104 | /** 105 | * AdeniumControl implements a supervisor that monitors the Adenium control messages and broadcasts according to the message. 106 | * The type parameter A of the constructor is an object that defines a broadcast variable wrapped by Ref [[com.adenium.app.framework.AdeniumBroadcast.Ref]]. 107 | * AdeniumControl consists of a Queue that stores this object and a Helper that processes 108 | * the Queue according to the methods defined in Helper and updates and broadcasts the Queue. 109 | * 110 | * @param helper 111 | * @tparam A 112 | * @return 113 | */ 114 | def apply[A]( helper : AdeniumBroadcastHelper[A]): AdeniumControl[A] 115 | = { 116 | 117 | state( "[ AdeniumControl ] create queue for receiving admin commands..") 118 | new AdeniumControl[A]( new ConcurrentLinkedQueue[A](), helper ) 119 | 120 | } 121 | 122 | private def command2queue[A]( msg: String, queue: AdeniumControl[A]) 123 | : Boolean = { 124 | 125 | val ret = queue 126 | .helper 127 | .command2Reference( msg) 128 | .map( queue.push ) 129 | 130 | ret getOrElse false 131 | } 132 | 133 | /** 134 | * Executes the Kafka message loop. 135 | * The message loop is a Watcher thread that monitors the AdeniumControl message. 136 | * Performs the defined action according to the entered message. 137 | * 138 | * @param reader 139 | * @param queue 140 | * @tparam A 141 | */ 142 | def startKfMsgLoop[A]( queue: AdeniumControl[A])(reader: KfConsumer) 143 | : Unit = Future { 144 | 145 | state(s" [ AdeniumControl - startKfMsgLoop] == startKfMsgLoop started : $reader") 146 | 147 | reader.readTuple { case ( _, value) => 148 | 149 | val msg = new String(value) 150 | state ( s"[ AdeniumControl ] kafka message : $msg ]" ) 151 | 152 | val ret = command2queue( msg, queue) 153 | state ( s"[ AdeniumControl ] kafka message pushed: ${(msg, ret)}" ) 154 | } 155 | 156 | } onComplete { 157 | _ => Logger.logWarning( "[ AdeniumControl ] startKfMsgLoop : Stopped ]" ) 158 | } 159 | 160 | /** 161 | * Executes the Zookeeper message loop. 162 | * The message loop is a Watcher thread that monitors the AdeniumControl message. 163 | * Performs the defined action according to the entered message. 164 | * 165 | * @param queue 166 | * @param cur 167 | * @param path 168 | * @tparam A 169 | */ 170 | def startZkMsgLoop[A](queue: AdeniumControl[A])(cur: CuratorFramework, path: String) 171 | : Unit = Future { 172 | 173 | state(s" [ AdeniumControl - startZkMsgLoop] == startKfMsgLoop started : ${(cur, path)}") 174 | 175 | ZkWatcher.onZkChange(cur, path) { case ( msg, _) => 176 | state ( "[ AdeniumControl ] == Zookeeper message : " + msg ) 177 | 178 | val ret = command2queue( msg, queue) 179 | state ( "[ AdeniumControl ] == Zookeeper message pushed: " + (msg, ret)) 180 | } 181 | } onComplete { 182 | _ => Logger.logWarning( "[ AdeniumControl ] startZkMsgLoop : Stopped ]" ) 183 | } 184 | 185 | } 186 | 187 | 188 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/framework/AdeniumMessage.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.framework 2 | 3 | import com.adenium.utils.Args.args2map 4 | import com.adenium.utils.StringUtil 5 | 6 | /** 7 | * Defines an AdeniumControl message. 8 | * 9 | * @param message 10 | * @param command 11 | */ 12 | case class AdeniumMessage( message: String, command: String) { 13 | 14 | private lazy val map: Option[Map[String, Array[String]]] = StringUtil.lift(message) 15 | .map( _.split(" ")) 16 | .map( args2map ( _, command)) 17 | 18 | /** 19 | * 20 | * @param cmd 21 | * @return 22 | */ 23 | def getOpt( cmd: String): Option[Array[String]] = map.flatMap( _.get( cmd) ) 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/app/logNormalizer/LNBroadcast.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.app.logNormalizer 2 | 3 | import com.adenium.app.config.Conf 4 | import com.adenium.app.framework.AdeniumBroadcast.{Caster, Ref, broadcaster} 5 | import com.adenium.app.framework.{AdeniumBroadcastHelper, AdeniumMessage} 6 | import com.adenium.externals.kafka.KfQueue 7 | import com.adenium.externals.zookeeper.ZkClient 8 | import com.adenium.parser.reference.{ParserRef, ParserRefMaker} 9 | import com.adenium.utils.Logger 10 | import com.adenium.utils.May.memo 11 | import org.apache.curator.framework.CuratorFramework 12 | import org.apache.spark.{SparkConf, SparkContext} 13 | 14 | /** 15 | * Defines broadcast variables used for normalization. 16 | * The zkcurator, kafka queue, and normalization reference data is broadcast. 17 | * Broadcast variables are wrapped in the Ref class.[[com.adenium.app.framework.AdeniumBroadcast.Ref]] 18 | * 19 | * @param zkcurator : zookeeper client 20 | * @param kfQueue : kafka producer 21 | * @param parserRef : Normalization reference data 22 | */ 23 | 24 | case class LNBroadcast( zkCurator: Ref[ZkClient] = Ref.empty, 25 | kfQueue: Ref[KfQueue] = Ref.empty, 26 | parserRef: Ref[ParserRef] = Ref.empty) { 27 | def status: String = s"LNBroadcast( zk, kf, pr) = ${(zkCurator.status, kfQueue.status, parserRef.status)}" 28 | } 29 | 30 | object LNBroadcast { 31 | 32 | /** 33 | * A dummy broadcast object that represents the initial creation state. 34 | * The broadcast target variable was not initialized on this object. 35 | */ 36 | val empty = new LNBroadcast() 37 | 38 | 39 | /** 40 | * This method serializes the class. 41 | * 42 | * @param sparkConf : sparkConf 43 | * @return 44 | */ 45 | def registerClasses(sparkConf: SparkConf): SparkConf = sparkConf 46 | .set( "spark.serializer", "org.apache.spark.serializer.KryoSerializer" ) 47 | .registerKryoClasses( Array ( 48 | classOf[ParserRef], 49 | classOf[ZkClient], 50 | classOf[KfQueue] ) 51 | ) 52 | 53 | 54 | /** 55 | * Creates a new (neo) LNBroadcast object. 56 | * @param zkCurator 57 | * @param kfQueue 58 | * @param parserRef 59 | * @return 60 | */ 61 | def from( zkCurator: Option[Ref[ZkClient]] = None, 62 | kfQueue: Option[Ref[KfQueue]] = None, 63 | parserRef: Option[Ref[ParserRef]] = None): Option[LNBroadcast] = { 64 | 65 | if ( zkCurator.isDefined || kfQueue.isDefined || parserRef.isDefined ) 66 | Some ( LNBroadcast( 67 | zkCurator.getOrElse( Ref.empty), 68 | kfQueue.getOrElse( Ref.empty), 69 | parserRef.getOrElse( Ref.empty)) ) 70 | else 71 | None 72 | } 73 | 74 | /** 75 | * Update the "spark broadcast variable" using the caster function "f". 76 | * 77 | * @param old : Variable that is currently broadcast to the sparkContext. 78 | * @param neo : The variable to be newly broadcast to the sparkContext. 79 | * @param f : Methods for broadcasting LNBroadcasts to sparkContext. 80 | * @return : Returns an LNBroadcast object containing the newly broadcasted variable. 81 | */ 82 | def broadcast( old: LNBroadcast, neo: LNBroadcast, f: Caster): LNBroadcast = { 83 | 84 | LNBroadcast( 85 | f( neo.zkCurator, old.zkCurator), 86 | f( neo.kfQueue, old.kfQueue), 87 | f( neo.parserRef, old.parserRef) 88 | ) 89 | } 90 | } 91 | 92 | ////////////////////////////////////////////////////////////////////////////////////// 93 | case class LNBroadcastHelper( conf: Conf)( implicit parserRefMaker: ParserRefMaker) 94 | extends AdeniumBroadcastHelper[LNBroadcast] { 95 | 96 | /** 97 | * curator variables are lazy variables. 98 | * Because of The actual connection is created when you call "zkClient.curator". 99 | * This action is to manage the connection like a singleton. 100 | */ 101 | lazy val curator: Option[CuratorFramework] = ZkClient( conf.zkstr).curator 102 | 103 | //////// private ref loader 104 | private def pa = Ref( ParserRef.initialize( parserRefMaker)) 105 | private def zk ( str: String) = Ref( ZkClient( str)) 106 | private def kf ( str: String) = Ref( KfQueue( str)) 107 | 108 | override def initializeBroadcast(sc: SparkContext): LNBroadcast = { 109 | 110 | val neo = LNBroadcast( zk(conf.zkstr), kf(conf.kfostr), pa) 111 | 112 | Logger.logWarning( s" LNBroadcastHelper : initializeBroadcast : ${neo.status}") 113 | 114 | reference2Broadcast( LNBroadcast.empty, neo, blocking = true)( sc) 115 | } 116 | 117 | override def command2Reference( message: String): Option[LNBroadcast] = { 118 | 119 | val m = AdeniumMessage(message, LNBroadcastHelper.commands) 120 | 121 | LNBroadcast.from( 122 | m.getOpt( "update:zkstr").map( _.head).map(zk), 123 | m.getOpt( "update:kfstr").map( _.head).map(kf), 124 | m.getOpt( "update:parser").map(_ => pa) 125 | ) 126 | } 127 | 128 | override def reference2Broadcast( old: LNBroadcast, 129 | neo: LNBroadcast, blocking: Boolean)(sparkContext: SparkContext) 130 | : LNBroadcast = 131 | { 132 | val ret = LNBroadcast.broadcast( old, neo, broadcaster( blocking, sparkContext)) 133 | memo( s"LNBroadcastHelper : reference2Broadcast: ${ret.status}") 134 | ret 135 | } 136 | } 137 | 138 | 139 | object LNBroadcastHelper { 140 | 141 | val commands : String = "update:zkstr|update:kfstr|update:parser|save:parser" 142 | 143 | } 144 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/Field.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | /** Normalization field 4 | * 5 | * A normalized event consists of a Field class representing the value of each item. 6 | Normalization events are displayed as FieldGetter, Normalized, Parsed, etc. 7 | depending on the display format, the presence of auxiliary information, and the difference in functionality provided. 8 | * 9 | - Normalized : [[com.adenium.common.Normalized]]
10 | - Parsed : [[com.adenium.common.Parsed]] 11 | * 12 | * The field consists of a pair of field key and actual item value. 13 | * 14 | * @constructor 15 | * @param key Predefined Field Id [[com.adenium.common.FieldKey]] 16 | * @param value Field value */ 17 | case class Field ( key: FieldKey, 18 | value: Option[String] ) { 19 | 20 | /** 21 | * The value represented by a string that is a pair of the unique number (mid) and value of the field. 22 | */ 23 | val valueWithId: Option[String] = value.map ( key.fid.toString + Field.TAB + _ ) 24 | /** 25 | * check if current field's key is equal to given field's. 26 | */ 27 | def isEqualKey( fld: Field ): Boolean = key.fid == fld.key.fid 28 | } 29 | 30 | object Field { 31 | 32 | /** 33 | * default field key-value separate string 34 | */ 35 | val TAB = "\t" 36 | 37 | def apply( key: FieldKey, value: String ) : Field = Field( key, Some(value)) 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/FieldFormat.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | /** 4 | * Fields Formats cover all JSON format. 5 | * maybe later, FString will be split to sub-classes, ie. FIPString, FDNSString ... 6 | * Please refer to the FieldType description. 7 | * [[com.adenium.common.FieldType]] 8 | */ 9 | sealed trait FieldFormat[T] { 10 | def get : T 11 | def string : String = get match { 12 | case str : String => str 13 | case etc => etc.toString 14 | } 15 | } 16 | 17 | object FNull extends FieldFormat[Null] { override def get: Null = null } 18 | 19 | /** Adenium field type [[com.adenium.common.FieldType]] */ 20 | final case class FString( get: String) extends FieldFormat[String] 21 | /** Adenium field type [[com.adenium.common.FieldType]] */ 22 | final case class FDateString( get: String) extends FieldFormat[String] 23 | /** Adenium field type [[com.adenium.common.FieldType]] */ 24 | final case class FDateMillis( get: String) extends FieldFormat[String] 25 | /** Adenium field type [[com.adenium.common.FieldType]] */ 26 | final case class FNumeric( get: Long) extends FieldFormat[Long] 27 | /** Adenium field type [[com.adenium.common.FieldType]] */ 28 | final case class FBoolean( get: Boolean) extends FieldFormat[Boolean] 29 | /** Adenium field type [[com.adenium.common.FieldType]] */ 30 | final case class FDouble( get: Double) extends FieldFormat[Double] 31 | 32 | 33 | object FieldFormat { 34 | 35 | type Formats = FieldFormat[_ >: String with Long with Boolean with Double with Null] 36 | } 37 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/FieldFormatter.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | import java.time.Instant 4 | 5 | import com.adenium.common.FieldFormat.Formats 6 | import com.adenium.common.FieldType._ 7 | import com.adenium.utils.May._ 8 | import com.adenium.utils.StringUtil 9 | 10 | case class FieldFormatter( fString: Option[ String => Formats], 11 | fNumeric: Option[ String => Formats], 12 | fDateString: Option[ String => Formats], 13 | fDateMillis: Option[ String => Formats], 14 | fBoolean: Option[ String => Formats] = None, // not used yet. 15 | fDouble: Option[ String => Formats] = None // not used yet. 16 | ) 17 | { 18 | 19 | def converter(k: FieldKey): String => Formats = { 20 | 21 | val f = k.fieldType match { 22 | case TString => fString 23 | case TNumeric => fNumeric 24 | case TDateString => fDateString 25 | case TDateMillis => fDateMillis 26 | case TBoolean => fBoolean 27 | case TDouble => fDouble 28 | case _ => None 29 | } 30 | 31 | // note: i'll use FString as formatter, if formatter is not defined... 32 | f.getOrElse(FString) 33 | } 34 | 35 | def format( vf: VariableKeys) 36 | ( kv: (Int, String)): Option[(String, Formats)] = { 37 | 38 | ///////////////////////////// 39 | val fid = kv._1 40 | val str = kv._2 41 | 42 | vf.getKey(fid).map { k => 43 | 44 | /** 45 | * assert 46 | * 1. k must contain 'fieldType' and 'fieldName' 47 | * 2. this(FieldFormattter) must provide fieldFormatting Function (converter) 48 | * 3. converter may cause an exception, 49 | * 50 | */ 51 | k.name -> { 52 | maybeWarn( converter(k)(str), "[ format ] Major : covnert throw an exception") 53 | .getOrElse( FString(str) ) 54 | } 55 | }.orElse { 56 | /** 57 | * key not found in vf.. maybe conf. has some problems. 58 | * hmmm... should i 59 | */ 60 | // todo 61 | // warn( Some( vf.defaultName( fid) -> FString(str)) )("[ format ] Major : fieldKey not found..") 62 | None 63 | } 64 | } 65 | } 66 | 67 | object FieldFormatter { 68 | 69 | ////////////////////////////////////////////////////////////////// 70 | // formatter 71 | ////////////////////////////////////////////////////////////////// 72 | def strDefault(str: String) = FString(str) 73 | def strJson(str: String) = FString( StringUtil.JSONEscape(str) ) 74 | def strXml(str: String) = FString( StringUtil.XMLEscape(str)) 75 | 76 | def numeric( str: String) = FNumeric( maybe(str.toLong).getOrElse( 0L) ) // todo: toInt or toLong ?? 77 | def dateCEP ( str: String) = FDateString( DateFieldFormatter.cepDt( str)) 78 | def millHD( str: String) = FDateMillis( DateFieldFormatter.mil2DateHd( str)) 79 | 80 | object DateFieldFormatter { 81 | 82 | import java.time.format.DateTimeFormatter 83 | import java.time.{Instant, ZoneId} 84 | import java.util.Locale 85 | 86 | def dateTimeFormatter(str: String): DateTimeFormatter = { 87 | DateTimeFormatter.ofPattern( str) 88 | .withZone( ZoneId.systemDefault() ) 89 | .withLocale( Locale.ENGLISH ) 90 | } 91 | 92 | def millis2DateString(millis: Long, dateTimeFormatter: DateTimeFormatter): String = { 93 | maybeInfo ( 94 | dateTimeFormatter.format( Instant.ofEpochMilli( millis)) 95 | )("fail so i'll use current.. ").getOrElse( 96 | dateTimeFormatter.format( Instant.now()) 97 | ) 98 | } 99 | 100 | private val es_dateformat = dateTimeFormatter( "yyyyMMdd'T'HHmmssZ" ) 101 | private val dr_dateformat = dateTimeFormatter( "yyyy-MM-dd HH:mm:ss z") 102 | private val dr_dateformat2 = dateTimeFormatter( "yyyy-MM-dd HH:mm:ss") 103 | private val hd_dateformat = dateTimeFormatter( "yyyy-MM-dd HH:mm:ss z") 104 | 105 | def mil2DateHd( millis: String): String = millis2DateString( millis.toLong, hd_dateformat) 106 | 107 | // added to temp. cure 108 | def cepDt ( str: String) : String = { 109 | maybeInfo { 110 | val dt = str.take(19) 111 | dr_dateformat2.parse( dt ) 112 | dt + " KST" 113 | }("fail so i'll use current.. ").getOrElse( 114 | dr_dateformat2.format( Instant.now()) + " KST" 115 | ) 116 | } 117 | } 118 | 119 | object DatePath { 120 | 121 | import DateFieldFormatter.{dateTimeFormatter, millis2DateString} 122 | 123 | private val date2minBolck = dateTimeFormatter("yyyy/MM/dd/HH/mm") 124 | private val date2dateHour = dateTimeFormatter("yyyyMMddHH") 125 | private val date2datePath = dateTimeFormatter("yyyyMMdd") 126 | 127 | def minBlockPath( millis: Long): String = millis2DateString( millis, date2minBolck ).take(15) + "0" 128 | def minBlockPath( millis: String): String = DatePath.minBlockPath( millis.toLong) 129 | 130 | def dateHour( millis: Long): String = millis2DateString( millis, date2dateHour ) 131 | def dateHour( millis: String): String = DatePath.dateHour( millis.toLong ) 132 | 133 | // todo : may cause integrity problem 134 | def dateString( instant: Option[Instant] = None): String = { 135 | val date = instant getOrElse Instant.now() 136 | date2datePath.format( date) 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/FieldGetter.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | import com.adenium.common.FieldFormatter.DatePath 4 | import Keys.m_LOGTIME 5 | import Keys.m_COMPANYID 6 | import Keys.m_AGENTID 7 | import com.adenium.utils.May.warn 8 | 9 | 10 | trait FieldGetter { 11 | def get( mid: Int): Option[String] 12 | 13 | lazy val logTime: Option[String] = get( m_LOGTIME ) 14 | lazy val company: Option[String] = get( m_COMPANYID ) 15 | lazy val agentId: Option[String] = get( m_AGENTID ) 16 | lazy val companyId: Int = warn( company.map(_.toInt) )("Invalid CompanyId") getOrElse -1 17 | } 18 | 19 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/FieldKey.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | import com.adenium.common.FieldType.{ FieldType, TString} 4 | 5 | /** Field Key 6 | * 7 | * Field item descriptor 8 | * 9 | * FieldKey is a value that indicates which item a particular field corresponds to 10 | - A unique number (mid) 11 | - Name (name) 12 | - Unique ID : predefined items ( reserved ID : ex- SourceIP ) 13 | - See the description of the Formatted field value format. [[com.adenium.common.FieldType]] 14 | * 15 | * @param fid Field ID 16 | * @param name Field Name (presentation String ) 17 | * @param keyID Reserved Id 18 | * @param fieldType Field type 19 | */ 20 | case class FieldKey( fid: Int, 21 | name: String, 22 | keyID: Option[KeyID.Value]= None, 23 | fieldType: FieldType = TString) { 24 | 25 | def tuple: (Int, FieldKey) = fid -> this 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/FieldOrder.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | import com.adenium.common.Keys._ 4 | 5 | import scala.collection.immutable.HashMap 6 | 7 | /** 8 | * Defines the order of the reserved fields. 9 | * This sequence is used to map the capture order of the regular expression. 10 | * 11 | * @param order 12 | * @param hashmap 13 | * @param size 14 | */ 15 | case class FieldOrder( order : Array[ Int], 16 | hashmap: HashMap[Int, Int], 17 | size: Int) { 18 | 19 | def contains( int: Int) : Boolean = hashmap.contains(int) 20 | } 21 | 22 | object FieldOrder { 23 | 24 | def apply( order: Array[Int]): FieldOrder = { 25 | 26 | val map = HashMap( order.zipWithIndex: _*) 27 | val size = order.length 28 | 29 | new FieldOrder( order, map, size) 30 | } 31 | 32 | def makeOrElse(maybe: Option[Array[Int]], default: FieldOrder): FieldOrder = { 33 | maybe.map( FieldOrder.apply ) getOrElse default 34 | } 35 | 36 | private val oldTSVOrder: Array[ Int] = Array ( 37 | m_CATEGORY1,m_CATEGORY2,m_CATEGORY3,m_SIGNATURE ,m_SEVERITY,m_COUNT,m_REPEATCOUNT,m_SRCIP,m_SRCPORT,m_SRCMAC, 38 | m_SRCCOUNTRY,m_DESTIP,m_DESTPORT,m_DESTMAC,m_DESTCOUNTRY,m_SRCDIRECTION,m_DESTDIRECTION,m_URL,m_URI,m_URIPARAMS, 39 | m_HEADER,m_PROTOCOL,m_PAYLOAD,m_CODE,m_RCVDBYTES,m_SENTBYTES,m_MESSAGEID,m_SRCZONE,m_DESTZONE,m_SERVICE,m_DURATION, 40 | m_ACLNM,m_ACTION,m_RAWDATA,m_SENDER,m_ATTACHMENT,m_STARTATTACKTIME,m_ENDATTACKTIME,m_LOGTIME,m_SYSLOGTIME, 41 | m_SYSLOGHOST,m_AGENTID,m_AGENTIP,m_COMPANYID,m_COMPANYNM,m_COMPANYGROUPID,m_DEVICETYPE,m_DEVICEMODEL,m_VENDOR ) 42 | 43 | } 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/FieldType.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | /** Generic types in JSON 4 | - '''Number:''' a signed decimal number that may contain a fractional part and may use exponential E notation, 5 | but cannot include non-numbers such as NaN. The format makes no distinction between integer and floating-point. 6 | JavaScript uses a double-precision floating-point format for all its numeric values, but other languages implementing JSON may encode numbers differently. 7 | - '''String:''' a sequence of zero or more Unicode characters. Strings are delimited with double-quotation marks and support a backslash escaping syntax. 8 | - '''Boolean:''' either of the values true or false 9 | 10 | * Basic 11 | 1. '''Most types are converted to String''' 12 | - String, Date, DateTime... ==> "String", Right DateString format ?, 13 | - JSON itself does not specify how dates should be represented, but JavaScript does. 14 | 1. '''Numerics are converted to Long or Double''' 15 | - Long, Integer, ==> Long, 12345, Double, Float ==> Double, 16 | - Double or Float : not used in current adenium normalized result. 17 | 1. '''Booleans are converted to Boolean''' 18 | - Boolean ==> (Boolean or true), 19 | - false Boolean : not used in current adenium normalized result. 20 | 1. '''Collections''' 21 | - Map, Tuple ==> { "key" : "value", "key2" : "value2", .... } 22 | - Array, ... ==> [ "key" : "value", "key2" : "value2", .... } 23 | * 24 | * 25 | * Data Type of Normalization field 26 | * 27 | * further thinking... 28 | - typeStringEmail, 29 | - typeStringAddress, 30 | - typeStringIP, 31 | - typeStringPort, 32 | - typeStringURL 33 | */ 34 | 35 | object FieldType extends Enumeration { 36 | 37 | type FieldType = Value 38 | 39 | val TString: FieldType.Value = Value 40 | val TNumeric: FieldType.Value = Value 41 | val TDateString: FieldType.Value = Value 42 | val TDateMillis: FieldType.Value = Value 43 | val TBoolean: FieldType.Value = Value 44 | val TDouble: FieldType.Value = Value 45 | 46 | } 47 | 48 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/KeyID.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | /** Reserved IDs */ 4 | object KeyID extends Enumeration { 5 | 6 | type AdeniumFieldKey = Value 7 | 8 | val CATEGORY1 : KeyID.Value = Value 9 | val CATEGORY2 : KeyID.Value = Value 10 | val CATEGORY3 : KeyID.Value = Value 11 | val SIGNATURE : KeyID.Value = Value 12 | val SEVERITY : KeyID.Value = Value 13 | val COUNT : KeyID.Value = Value 14 | val REPEATCOUNT : KeyID.Value = Value 15 | val SRCIP : KeyID.Value = Value 16 | val SRCPORT : KeyID.Value = Value 17 | val SRCMAC : KeyID.Value = Value 18 | val SRCCOUNTRY : KeyID.Value = Value 19 | val DESTIP : KeyID.Value = Value 20 | val DESTPORT : KeyID.Value = Value 21 | val DESTMAC : KeyID.Value = Value 22 | val DESTCOUNTRY : KeyID.Value = Value 23 | val SRCDIRECTION : KeyID.Value = Value 24 | val DESTDIRECTION : KeyID.Value = Value 25 | val URL : KeyID.Value = Value 26 | val URI : KeyID.Value = Value 27 | val URIPARAMS : KeyID.Value = Value 28 | val HEADER : KeyID.Value = Value 29 | val PROTOCOL : KeyID.Value = Value 30 | val PAYLOAD : KeyID.Value = Value 31 | val CODE : KeyID.Value = Value 32 | val RCVDBYTES : KeyID.Value = Value 33 | val SENTBYTES : KeyID.Value = Value 34 | val MESSAGEID : KeyID.Value = Value 35 | val SRCZONE : KeyID.Value = Value 36 | val DESTZONE : KeyID.Value = Value 37 | val SERVICE : KeyID.Value = Value 38 | val DURATION : KeyID.Value = Value 39 | val ACLNM : KeyID.Value = Value 40 | val ACTION : KeyID.Value = Value 41 | val RAWDATA : KeyID.Value = Value 42 | val SENDER : KeyID.Value = Value 43 | val ATTACHMENT : KeyID.Value = Value 44 | val STARTATTACKTIME : KeyID.Value = Value 45 | val ENDATTACKTIME : KeyID.Value = Value 46 | val LOGTIME : KeyID.Value = Value 47 | val SYSLOGTIME : KeyID.Value = Value 48 | val SYSLOGHOST : KeyID.Value = Value 49 | val AGENTID : KeyID.Value = Value 50 | val AGENTIP : KeyID.Value = Value 51 | val COMPANYID : KeyID.Value = Value 52 | val COMPANYNM : KeyID.Value = Value 53 | val COMPANYGROUPID : KeyID.Value = Value 54 | val DEVICETYPE : KeyID.Value = Value 55 | val DEVICEMODEL : KeyID.Value = Value 56 | val VENDOR : KeyID.Value = Value 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/Normalized.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | import com.adenium.utils.May._ 4 | 5 | import scala.collection.immutable.HashMap 6 | 7 | /** 8 | * Normalization result 9 | * 10 | * @param arrayWithID 11 | */ 12 | case class Normalized( arrayWithID : Array[ (Int, String)]) extends FieldGetter { 13 | 14 | private lazy val hashmap: HashMap[Int, String] = HashMap( arrayWithID: _* ) 15 | override def get( mid: Int): Option[String] = hashmap.get( mid) 16 | 17 | /** 18 | * get all listed values according to listed seq. : value may be None if not exists. 19 | * @param ids 20 | * @return 21 | */ 22 | def getAll( ids: Array[Int]): Array[(Int, Option[String])] = ids.map(i => i -> get(i)) 23 | 24 | /** 25 | * get all listed values according to listed seq.: value may be omitted if not exists. 26 | * @param ids 27 | * @return 28 | */ 29 | def gets( ids: Array[Int]): Array[(Int, String)] = ids.flatMap(i => get(i).map( i -> _)) 30 | 31 | /** 32 | * get all listed values according to listed seq. : value is set to default if not exists. 33 | * @param ids 34 | * @return 35 | */ 36 | def getAllwithDefault( ids: Array[Int], default: String) 37 | : Array[(Int, String)] = ids.map(i => i -> get(i).getOrElse(default)) 38 | } 39 | 40 | object Normalized { 41 | 42 | def apply( TSV : String): Normalized = Normalized( TSV2Normalized( TSV)) 43 | 44 | def filtered( ids: Array[Int]) 45 | ( norm: Normalized): Normalized = new Normalized( norm.gets(ids)) 46 | 47 | def ordered( ids: Array[Int], default: String= "") 48 | ( norm: Normalized): Normalized = new Normalized( norm.getAllwithDefault(ids, default)) 49 | 50 | /////////////////////////////////////////////////////// 51 | // interim String format 52 | /////////////////////////////////////////////////////// 53 | val versionHeader : String = "adenium" 54 | val TAB: String = "\t" 55 | 56 | /** 57 | * The result of the normalization of TAB delimiters is made into an array of (key, value) pairs. 58 | * @param TSV 59 | * @return 60 | */ 61 | 62 | def TSV2Normalized( TSV: String ): Array[ (Int, String)] = { 63 | 64 | val arr = TSV.split( TAB) 65 | val ret = 66 | if ( arr.headOption.contains( versionHeader) ) { 67 | arr 68 | .drop(1) 69 | .grouped(2) 70 | .flatMap ( ar => maybeWarn { 71 | val idx = ar(0).toInt 72 | val value = ar(1) 73 | (idx, value) 74 | }) 75 | .toArray 76 | } 77 | else 78 | arr.zipWithIndex.map { case (v, i) => ( i, v) } 79 | ret 80 | } 81 | 82 | def parsed2TSV( parsed: Parsed): String = { 83 | val TSV: String = warn { 84 | parsed.fields.map ( _.flatMap { _.valueWithId }.mkString( TAB) ) 85 | }("[ Normalized ] : toTSV = Fields is empty.").getOrElse( "") 86 | 87 | versionHeader + TAB + TSV 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/Parsed.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | case class Parsed ( fields: Option[ Array[Field]], 4 | host: Option[String], 5 | raw: String, 6 | log: String) extends FieldGetter { 7 | 8 | override def get( id: Int): Option[String] = { 9 | fields.flatMap( _.find( _.key.fid == id)).flatMap( _.value) 10 | } 11 | } 12 | 13 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/VariableKeys.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.common 2 | 3 | import scala.collection.immutable.HashMap 4 | 5 | /** 6 | * Defines the extension key. 7 | * @param keys 8 | */ 9 | 10 | case class VariableKeys( keys: HashMap[Int, FieldKey] ) { 11 | 12 | lazy val length: Int = keys.size 13 | lazy val size: Int = keys.size 14 | def getKey( fid: Int): Option[ FieldKey] = keys.get(fid) orElse Keys.ReservedKeys.get(fid) 15 | 16 | def defaultName( id: Int): String = "fid_" + id 17 | 18 | /** 19 | * @param ids 20 | * @return 21 | */ 22 | def getName( id : Int): Option[String] = keys.get(id).map( _.name) 23 | 24 | /** 25 | * @param ids 26 | * @return 27 | */ 28 | def getNameOrDefault( id : Int): String = getName(id).getOrElse( defaultName(id)) 29 | 30 | /** 31 | * get all listed Names : name may be omitted if not exists. 32 | * @param ids 33 | * @return 34 | */ 35 | def getNames( ids : Array[Int]): Array[String] = ids.flatMap( getName) 36 | 37 | /** 38 | * get all listed Names : name may be None if not exists. 39 | * @param ids 40 | * @return 41 | */ 42 | def getAllNames( ids : Array[Int]): Array[Option[String]] = ids.map( getName) 43 | 44 | } 45 | 46 | object VariableKeys { 47 | 48 | val defaultVariableKeys = VariableKeys(None) 49 | 50 | def apply( fields: Option[Array[FieldKey]] = None ) : VariableKeys = { 51 | new VariableKeys( Keys.mergedKeys ( fields) ) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/common/package.scala: -------------------------------------------------------------------------------- 1 | package com.adenium 2 | /** Provides a common data structure and classes. 3 | * 4 | * */ 5 | package object common { 6 | } 7 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/kafka/KfConsumer.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.kafka 2 | 3 | import java.util.Properties 4 | 5 | import com.adenium.utils.May._ 6 | import com.adenium.utils.Timer._ 7 | import kafka.consumer._ 8 | import kafka.serializer._ 9 | 10 | import scala.collection.JavaConversions._ 11 | 12 | /** 13 | * KafkaConsumer [[https://kafka.apache.org/quickstart]] 14 | * 15 | * @param stream 16 | * @param connector 17 | */ 18 | class KfConsumer ( stream: KafkaStream[ Array[Byte], Array[Byte]], 19 | connector: ConsumerConnector ) { 20 | 21 | def read( handler: Array[Byte] => Unit): Unit = { 22 | 23 | for (messageAndTopic <- stream) { 24 | maybeWarn2 { 25 | 26 | handler( messageAndTopic.message() ) 27 | }( "[ Kafka Message handler ]") 28 | } 29 | } 30 | 31 | def readTuple( handler: (Array[Byte], Array[Byte]) => Unit): Unit = { 32 | 33 | for ( messageAndTopic <- stream) { 34 | maybeWarn2 { 35 | 36 | handler( messageAndTopic.key(), messageAndTopic.message() ) 37 | }( "[ Kafka Message handler ]") 38 | } 39 | } 40 | 41 | def close(): Unit = connector.shutdown() 42 | } 43 | 44 | /** 45 | * [[https://kafka.apache.org/0100/javadoc/index.html?org/apache/kafka/clients/consumer/KafkaConsumer.html]] 46 | */ 47 | object KfConsumer { 48 | 49 | def apply( topic: String, 50 | groupId: String, 51 | zkstr: String, 52 | readFromStartOfStream: Boolean = true): KfConsumer = { 53 | 54 | val props = new Properties() 55 | props.put("group.id", groupId) 56 | props.put("zookeeper.connect", zkstr) 57 | props.put("auto.offset.reset", if (readFromStartOfStream) "smallest" else "largest") 58 | 59 | val config = new ConsumerConfig(props) 60 | val connector = Consumer.create(config) 61 | val filter= Whitelist(topic) 62 | 63 | state( s"[ KafkaConsumer ] kafka consumer create = $topic") 64 | val stream = TimeLog { 65 | 66 | connector.createMessageStreamsByFilter( filter, 1, new DefaultDecoder(), new DefaultDecoder()).get(0) 67 | 68 | }(s"[ KafkaConsumer ] (topic, zkstr) = ${(topic, zkstr)}" ) 69 | 70 | new KfConsumer( stream, connector) 71 | } 72 | 73 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/kafka/KfPaths.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.kafka 2 | 3 | object KfPaths { 4 | 5 | val consumers = "/consumers" 6 | val brokerIds = "/brokers/ids" 7 | val brokerTopics = "/brokers/topics" 8 | val topicConfigs = "/config/topics" 9 | val topicConfigChanges = "/config/changes" 10 | val controller = "/controller" 11 | val controllerEpoch = "/controller_epoch" 12 | val reassignPartitions = "/admin/reassign_partitions" 13 | val deleteTopics = "/admin/delete_topics" 14 | val preferredReplicaLeaderElection = "/admin/preferred_replica_election" 15 | val admin = "/admin" 16 | 17 | def topic(topic_name: String): String = brokerTopics + "/" + topic_name 18 | def topicPartitions(topic_name: String): String = topic(topic_name) + "/partitions" 19 | def topicPartition(topic_name: String, partitionId: Int): String = topicPartitions(topic_name) + "/" + partitionId 20 | def topicPartitionState(topic_name: String, partitionId: Int): String = topicPartition(topic_name, partitionId) + "/" + "state" 21 | def topicConfig(topic_name: String): String = topicConfigs + "/" + topic_name 22 | def deleteTopic(topic_name: String): String = deleteTopics + "/" + topic_name 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/kafka/KfProducer.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.kafka 2 | 3 | import java.util.Properties 4 | import java.util.concurrent.Future 5 | 6 | import com.adenium.utils.May._ 7 | import org.apache.kafka.clients.producer._ 8 | 9 | /** 10 | * A client that consumes records from a Kafka cluster. 11 | * This client transparently handles the failure of Kafka brokers, and transparently adapts as topic partitions it fetches migrate within the cluster. This client also interacts with the broker to allow groups of consumers to load balance consumption using consumer groups. 12 | * 13 | * The consumer maintains TCP connections to the necessary brokers to fetch data. Failure to close the consumer after use will leak these connections. The consumer is not thread-safe. See Multi-threaded Processing for more details. 14 | * 15 | * 16 | * @param brokers 17 | * @param producer 18 | */ 19 | class KfProducer( brokers: String, producer: KafkaProducer[String, String]) extends Serializable { 20 | 21 | def send( record: ProducerRecord[String, String], callBack: Callback = null) 22 | : Option[Future[RecordMetadata]] = { 23 | 24 | maybeWarn { producer.send( record, callBack) } 25 | } 26 | 27 | def close(): Unit = Option( producer).foreach ( _.close()) 28 | } 29 | 30 | /** 31 | * A Kafka client that publishes records to the Kafka cluster. 32 | * The producer is thread safe and sharing a single producer instance across threads will generally be faster than having multiple instances. 33 | * 34 | * Here is a simple example of using the producer to send records with strings containing sequential numbers as the key/value pairs. 35 | * [[https://kafka.apache.org/10/javadoc/index.html?org/apache/kafka/clients/producer/KafkaProducer.html]] 36 | */ 37 | object KfProducer { 38 | 39 | def apply( broker: String, props: Option[Properties] = None): KfProducer = { 40 | 41 | val prop = props.getOrElse { 42 | val p = new Properties() 43 | p.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, broker) 44 | p.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") 45 | p.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") 46 | p.put(ProducerConfig.ACKS_CONFIG, "0") 47 | p.put(ProducerConfig.RETRIES_CONFIG, "3") 48 | p.put(ProducerConfig.LINGER_MS_CONFIG, "0") 49 | //props.put(ProducerConfig.BATCH_SIZE_CONFIG, "1") 50 | p 51 | } 52 | 53 | val prod = new KafkaProducer[String, String](prop) 54 | new KfProducer( broker, prod ) 55 | } 56 | 57 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/kafka/KfQueue.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.kafka 2 | 3 | import java.util.concurrent.Future 4 | 5 | import kafka.common.TopicAndPartition 6 | import org.apache.kafka.clients.producer._ 7 | 8 | object KfQueue { 9 | 10 | def apply(broker: String): KfQueue = { 11 | val f = () => { 12 | val wrapper = KfProducer(broker) 13 | sys.addShutdownHook { 14 | wrapper.close() 15 | } 16 | wrapper 17 | } 18 | new KfQueue(f) 19 | } 20 | } 21 | 22 | /** 23 | * kafka util wrapped class 24 | * @param createWrapper 25 | */ 26 | class KfQueue( createWrapper: () => KfProducer) extends Serializable { 27 | 28 | lazy val producer: KfProducer = createWrapper() 29 | 30 | def close(): Unit = producer.close() 31 | 32 | def partition(pn: Option[Int], n: Int): Int = { 33 | 34 | pn match { 35 | case Some(p) => if (p < n) p else scala.util.Random.nextInt(n) 36 | case _ => scala.util.Random.nextInt(n) 37 | } 38 | 39 | } 40 | 41 | def send2kf( key: String, value: String, topic: String) 42 | : Option[ Future[RecordMetadata]] = { 43 | 44 | producer.send(new ProducerRecord[String, String]( topic, key, value)) 45 | 46 | } 47 | } 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/spark/SparkUtil.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.spark 2 | 3 | import com.adenium.utils.Logger 4 | import com.adenium.utils.May.maybe 5 | import com.adenium.utils.May.maybeInfo 6 | import org.apache.spark.util.LongAccumulator 7 | 8 | /** 9 | * Spark util wrapped class 10 | */ 11 | object SparkUtil { 12 | 13 | case class Accumulate ( acs : Seq[ LongAccumulator ] ) { 14 | 15 | def logStr: Seq[String ] = { 16 | acs map { ac => s"${ac.name.getOrElse( ac.id.toString )} = ${ac.value}" } 17 | } 18 | 19 | def log (): Unit = { 20 | acs foreach { ac => Logger.logInfo( ac.name.getOrElse( ac.id.toString) + " = " + ac.value )} 21 | } 22 | 23 | def add ( is: Seq[ Int] ): Unit = { 24 | acs.zip( is).foreach{ case (a, i) => maybe { a add i } } // swallow exception 25 | } 26 | 27 | def add( nls: Iterator[ Seq[ Int]]): Unit = { 28 | //ex: Iterator [Seq( ls, ncnt, bcnt, fcnt )] 29 | nls foreach { is => this.add( is) } 30 | } 31 | 32 | def add2sum(is: Seq[ Int], f: Seq[Int] => Int = _.sum ): Unit = { 33 | acs.zip( f(is) +: is).foreach { case (a, i) => 34 | maybeInfo( a add i )("add2sum failed") 35 | } 36 | } 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/zookeeper/ZkClient.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.zookeeper 2 | 3 | import com.adenium.externals.kafka.KfPaths.topicPartitions 4 | import com.adenium.externals.zookeeper.ZkUtil._ 5 | import com.adenium.utils.May._ 6 | import kafka.common.TopicAndPartition 7 | import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory} 8 | import org.apache.curator.retry.ExponentialBackoffRetry 9 | import org.apache.spark.streaming.kafka.OffsetRange 10 | 11 | object ZkClient { 12 | 13 | /** 14 | * zookeeper retry-policy 15 | * @param baseSleepTimeMs 16 | * @param maxRetries 17 | */ 18 | case class Retry( baseSleepTimeMs: Int, maxRetries: Int) { 19 | def policy = new ExponentialBackoffRetry( baseSleepTimeMs, maxRetries) 20 | } 21 | 22 | object Retry { 23 | val default = Retry(1000, 3) 24 | } 25 | 26 | /** 27 | * note: zkclient contains only conntection-maker ( not connection itself ) 28 | * @param connectionString 29 | * @param retry 30 | * @return 31 | */ 32 | def apply( connectionString: String, retry : Retry = Retry.default): ZkClient = { 33 | val f = () => { 34 | val cur = CuratorFrameworkFactory.newClient( connectionString, retry.policy) 35 | cur.start() 36 | sys.addShutdownHook { cur.close() } 37 | cur 38 | } 39 | 40 | new ZkClient( f ) 41 | } 42 | } 43 | 44 | class ZkClient( connector: () => CuratorFramework ) extends Serializable { 45 | 46 | lazy val curator: Option[ CuratorFramework ] = maybe { connector() } 47 | 48 | /** 49 | * Save the zookeeper offset to the zookeeper node. 50 | * This offset range is used for failover. 51 | * When Adenium Freamework is running in restore mode, it will process the data in its offset range. 52 | * 53 | * @param path 54 | * @param topic 55 | * @param ar 56 | * @return 57 | */ 58 | def setOffsets(path: String, topic: String, ar: Array[ OffsetRange ]) 59 | : Option[String] = { 60 | 61 | curator.flatMap { cur => 62 | 63 | val offsetStr = ar 64 | .map ( range => s"${range.topic}:${range.partition}:${range.fromOffset}") 65 | .mkString(",") 66 | 67 | state ( s"saveOffsets: $offsetStr") 68 | 69 | setPersistent( cur, path + "/" + topic, offsetStr ) 70 | } 71 | } 72 | 73 | 74 | /** 75 | * Get the value stored in the Zookeeper node. 76 | * @param path 77 | * @return 78 | */ 79 | def getZkString( path: String) 80 | : Option[String] = curator.flatMap ( readString(_, path) ) 81 | 82 | 83 | /** 84 | * Gets the offset stored with the "setOffsets method". 85 | * @param path 86 | * @param topic 87 | * @return 88 | */ 89 | def getOffsets(path : String, topic: String) 90 | : Option[Map[TopicAndPartition, Long]] = { 91 | 92 | val ret = curator.flatMap { cur => 93 | 94 | val offsetStr = readString( cur, path + "/" + topic) 95 | val offsets = maybeWarn { 96 | offsetStr.map { str => 97 | str.split(",") 98 | .map( _.split(":")) 99 | .map( ar => TopicAndPartition( ar(0), ar(1).toInt) -> ar(2).toLong ) 100 | .toMap 101 | } 102 | }.flatten 103 | 104 | offsets 105 | } 106 | 107 | ret 108 | } 109 | 110 | 111 | /** 112 | * Obtain the partition id managed by kafka. 113 | * @param topic 114 | * @return 115 | */ 116 | def get_Kf_PartitionIds ( topic: String) 117 | : Option[Seq[ Int ] ] = { 118 | 119 | val path = topicPartitions( topic) 120 | 121 | for { 122 | cur <- curator 123 | children <- getChildren( cur, path ) 124 | } yield children.flatMap { c => maybeWarn{ c.toInt } } 125 | 126 | } 127 | 128 | /** 129 | * Obtain the topic and partition managed by kafka. 130 | * @param topic 131 | * @return 132 | */ 133 | def get_kf_TopicAndPartition( topic: String) 134 | : Option[TopicAndPartition ] = { 135 | 136 | get_Kf_PartitionIds( topic ) map ( pids => TopicAndPartition( topic, pids.length )) // TODO : length ?? 137 | 138 | } 139 | 140 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/externals/zookeeper/ZkWatcher.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.externals.zookeeper 2 | 3 | import java.nio.charset.StandardCharsets 4 | 5 | import com.adenium.externals.zookeeper.ZkUtil.setPersistent 6 | import com.adenium.utils.Logger 7 | import com.adenium.utils.May._ 8 | import org.apache.curator.framework.CuratorFramework 9 | import org.apache.zookeeper.Watcher.Event.EventType 10 | import org.apache.zookeeper.data.Stat 11 | import org.apache.zookeeper.{KeeperException, WatchedEvent, Watcher} 12 | 13 | import scala.language.reflectiveCalls 14 | 15 | /** 16 | * 17 | * Zookeeper watcher with Curator watchedEvent. 18 | * Zookeeper watcher for monitoring AdeniumControl messages. 19 | * 20 | * - Curator WatchedEvent - 21 | * A super set of all the various Zookeeper events/background methods. 22 | * IMPORTANT: the methods only return values as specified by the operation that generated them. 23 | * Many methods will return null [[https://curator.apache.org/apidocs/org/apache/curator/framework/api/CuratorEvent.html]] 24 | */ 25 | 26 | object ZkWatcher { 27 | 28 | def onZkChange(cur: CuratorFramework, path: String)(handler: (String, Stat) => Unit) { 29 | 30 | Logger.logInfo("[ watchNodeOrChidlrenChange ] == zknode : " + path) 31 | 32 | def watcher = new Watcher { 33 | def process(event: WatchedEvent) { 34 | Logger.logDebug("[ watchNodeOrChidlrenChange ] == callback invoked " + path + "\ttype: " + event.getType) 35 | event.getType match { 36 | case EventType.NodeDataChanged | EventType.NodeChildrenChanged => updated() 37 | case _ => reset() 38 | } 39 | } 40 | } 41 | 42 | def updated() { 43 | try { 44 | val stat = new Stat() 45 | val msg = cur.getData.storingStatIn(stat).forPath(path) 46 | 47 | setPersistent(cur, path, "") 48 | 49 | val str = new String(msg, StandardCharsets.UTF_8) 50 | 51 | if (str.nonEmpty) { 52 | state("[ Watching ] == arrived msg: " + new String(msg, StandardCharsets.UTF_8)) 53 | handler(str, stat) 54 | } 55 | 56 | if (str.startsWith("stop zkctrl")) { 57 | Logger.logWarning("[ Watching ] == stopped by 'stop zkctrl' message : path =" + path) 58 | } else { 59 | /// create and attach next msg watcher 60 | cur.checkExists.usingWatcher(watcher).forPath(path) 61 | } 62 | 63 | } catch { 64 | case e: KeeperException => 65 | Logger.logWarning("[ watchNodeOrChidlrenChange ] == read node: " + path + "\te: " + e) 66 | reset() 67 | } 68 | } 69 | 70 | def reset() { 71 | setPersistent(cur, path, "") 72 | updated() 73 | } 74 | 75 | reset() 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/ParserPref.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser 2 | 3 | object ParserPref { 4 | 5 | object Implicits { 6 | 7 | // implicit val referenceFiles: ParserRefFiles = ParserRefFiles( path = "") 8 | // implicit val flumeRegex: FlumeRegex= FlumeRegex( """(\d+):::(.*)""".r, """A (\S+) (\S+) (\d+):::(.*)""".r ) 9 | // implicit val fireEyeRegex: SOCFireEyeRegex = SOCFireEyeRegex( "(211.45.57.211)", ".*FIREEYE ALERT: (.*);(.*)".r ) 10 | // implicit val nagiosRegex: SOCNagiosRegex = SOCNagiosRegex("\\snms\\d+$\\s*".r, ".{0,32}SERVICE ALERT: ([^_]*)_([^_]*)_([^_]*);.*".r) 11 | // implicit val syslogEncodings: SyslogEncodings = SyslogEncodings( Array( "CP949") ) 12 | // implicit val wOptionLogOn: WOptionLogOn = WOptionLogOn( true) 13 | 14 | } 15 | 16 | } 17 | 18 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/ParserTester.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser 2 | 3 | import com.adenium.common.Parsed 4 | import com.adenium.parser.reference.File2ParserRef.ParserRefFiles 5 | import com.adenium.parser.reference.{File2ParserRef, ParserRef} 6 | import com.adenium.utils.Args 7 | import com.adenium.utils.May.{maybe, maybeWarn2} 8 | import com.adenium.utils.WOption.WOptionLogOn 9 | 10 | object ParserTester extends App { 11 | 12 | import scala.io.StdIn.readLine 13 | 14 | /* arguments => opts*/ 15 | val opts = Args.args2map( args, "path|logon") 16 | 17 | /* reference data path*/ 18 | val refPath = opts.get("path") map (_.head) getOrElse { 19 | println("Please check reference data path : -path [resource path] " + "\n " + 20 | "Loaded from default path: resources\\") 21 | "resources" 22 | } 23 | 24 | val logOn = opts.get("logon").exists(_ => true) 25 | 26 | /* Init reference data from file*/ 27 | val refFiles: ParserRefFiles = ParserRefFiles.apply( refPath + "\\") 28 | val parserRef: ParserRef = File2ParserRef(refFiles).initialize() 29 | 30 | val versionHeader : String = "adenium" 31 | val TAB: String = "\t" 32 | private def makeTSPString(parsed: Parsed): String = { 33 | val TSP: String = 34 | maybe { 35 | parsed.fields 36 | .map ( _.flatMap { _.valueWithId }.mkString( TAB) ) 37 | .getOrElse(": maybe error : toTSV = Fields is empty.") 38 | }.getOrElse(": exception occurred") 39 | 40 | versionHeader + TAB + TSP 41 | } 42 | 43 | 44 | private def tryParse( str: String ): Unit = { 45 | 46 | implicit val wOptionLogOn: WOptionLogOn = WOptionLogOn(logOn) 47 | 48 | val ret: Parsed = Parser.execute( str, parserRef, verbose = true) 49 | val flds = makeTSPString( ret) 50 | 51 | println( "\n" + flds) 52 | 53 | } 54 | 55 | private def run(): Unit = { 56 | 57 | System.setProperty("line.separator", "\n") 58 | val s = System.currentTimeMillis 59 | 60 | var repeat = true 61 | do { 62 | repeat = readLine match { 63 | case "" | null => 64 | println( "i got blank line. end-of-stream. Bye~!") 65 | false 66 | case "q" | "quit" | "bye" | "exit" => println("Bye !") 67 | false 68 | case str => 69 | maybe( tryParse(str)) 70 | true 71 | } 72 | } while (repeat) 73 | 74 | val e = System.currentTimeMillis 75 | println(s"List(=== running time : ${e - s}===============================================") 76 | } 77 | 78 | run() 79 | 80 | } 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/devices/SOCDeviceKind.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.devices 2 | 3 | import com.adenium.parser.devices 4 | 5 | /** Handling Device type Category 6 | * 7 | * You can classify agent types and apply processing logic defined for each category. 8 | * normal : Regular expression based syslog. 9 | * Can be further defined. 10 | {{{ 11 | Example 12 | val jsonType: devices.SOCDeviceKind.Value = Value // json expression based log server 13 | val logServers: devices.SOCDeviceKind.Value = Value // custom log server 14 | val monitorType: devices.SOCDeviceKind.Value = Value // System status log type 15 | }}} 16 | */ 17 | object SOCDeviceKind extends Enumeration { 18 | 19 | type SOCDeviceKind = Value 20 | val normal: devices.SOCDeviceKind.Value = Value 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/reference/ParserRef.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.reference 2 | 3 | import com.adenium.common.{Field, VariableKeys} 4 | import com.adenium.parser.structs._ 5 | import com.adenium.utils.Logger 6 | 7 | import scala.language.implicitConversions 8 | /** Reference data for normalization 9 | * 10 | - mapHostAgent: Map of host ( ip or name ) 11 | - mapAgent: Map of id 12 | - tokenizeRules: Regular expressions [[com.adenium.parser.structs.TokenizeRule]] 13 | - arrangeRules: Rules that link token fields to normalization fields. [[com.adenium.parser.structs.ArrangeRule]] 14 | - companyIpRange: Agent owner ip ranges [[com.adenium.parser.structs.CompanyIpRange]] 15 | - companyIps: Owner server IP of the installed agent [[com.adenium.parser.structs.CompanyIp]] 16 | - geoIpRange: Ip by country [[com.adenium.parser.structs.GeoIpRange]] 17 | - signatures: Signatures of sensor [[com.adenium.parser.structs.Signature]] 18 | - replaceFields: Changeable field [[com.adenium.parser.structs.ReplaceField]] 19 | - fields: Normalization fields [[com.adenium.common.Field]] 20 | */ 21 | case class ParserRef (mapHostAgent: Option[Map[Long, Array[ Agent]]] = None, 22 | mapAgent: Option[Map[Long, Array[Agent]] ] = None, 23 | tokenizeRules: Option[Map[Long, Array[TokenizeRule]]] = None, 24 | arrangeRules: Option[Map[Int, Array[ArrangeRule]]] = None, 25 | companyIpRange: Option[Map[Long, Array[CompanyIpRange]]] = None, 26 | companyIps: Option[Map[Long, Array[CompanyIp]]] = None, 27 | geoIpRange: Option[Map[Long, Array[GeoIpRange]]] = None, 28 | signatures: Option[Map[(String, Long), Signature]] = None, 29 | replaceFields: Option[Map[Int, Array[ ReplaceField]]] = None, 30 | fields: Option[VariableKeys] = None ) 31 | { 32 | 33 | def <+(update : ParserRef ): ParserRef = { 34 | 35 | ParserRef ( 36 | update.mapHostAgent orElse this.mapHostAgent, 37 | update.mapAgent orElse this.mapAgent, 38 | update.tokenizeRules orElse this.tokenizeRules, 39 | update.arrangeRules orElse this.arrangeRules, 40 | update.companyIpRange orElse this.companyIpRange, 41 | update.companyIps orElse this.companyIps, 42 | update.geoIpRange orElse this.geoIpRange, 43 | update.signatures orElse this.signatures, 44 | update.replaceFields orElse this.replaceFields, 45 | update.fields orElse this.fields 46 | ) 47 | } 48 | } 49 | 50 | object ParserRef { 51 | 52 | def logStatus ( ref: ParserRef ): String = { 53 | 54 | val logStr = 55 | s"""[ParserRef] geoIpRange: ${ref.geoIpRange.map(_.keys.size)}""" + 56 | s"""[ParserRef] companyIpRange: ${ref.companyIpRange.map(_.values.size)}""" + 57 | s"""[ParserRef] signatures: ${ref.signatures.map(_.values.size)}""" + 58 | s"""[ParserRef] mapHostnameAgent: ${ref.mapHostAgent.map(_.values.size)}""" + 59 | s"""[ParserRef] mapAgent: ${ref.mapAgent.map(_.values.size)}""" + 60 | s"""[ParserRef] ruleSetInfo : ${ref.tokenizeRules.map(_.values.size)}""" + 61 | s"""[ParserRef] matchingRuleFieldInfo : ${ref.arrangeRules.map(_.values.size)}""" + 62 | s"""[ParserRef] FieldInfo : ${ref.fields.map(_.size)}""" + 63 | s"""[ParserRef] FieldReplace : ${ref.replaceFields.map(_.values.size)}""" + 64 | s"""[ParserRef] companyServerInfo : ${ref.companyIps.map(_.valuesIterator.length)}""" 65 | 66 | Logger.logInfo( logStr) 67 | 68 | logStr 69 | } 70 | 71 | def initialize(maker: ParserRefMaker): ParserRef = { 72 | maker.initialize() 73 | } 74 | 75 | ///////////////////////////////////////////////////////////////////// 76 | // todo 77 | implicit def ref2VariableField ( r: ParserRef): Option[VariableKeys] = { r.fields } 78 | 79 | implicit def agentRef ( r: Option[Array[ Agent]] ) 80 | : ParserRef = { 81 | r.map { arr => 82 | 83 | val mapIp = arr.groupBy( _.ipLong ) 84 | val mapId = arr.groupBy( _.agentId ) 85 | 86 | ParserRef( mapHostAgent = Some(mapIp), mapAgent = Some( mapId)) 87 | 88 | }.getOrElse( ParserRef() ) 89 | } 90 | 91 | implicit def rules2Tokenize2Ref ( r: Option[ Array[TokenizeRule]] ) 92 | : ParserRef = ParserRef( tokenizeRules = r.map(_.groupBy( _.sensorId)) ) 93 | 94 | implicit def rules2Arrange2Ref ( r: Option[ Array[ArrangeRule]]) 95 | : ParserRef = ParserRef( arrangeRules = r.map(_.groupBy(_.tokenizeRuleId))) 96 | 97 | implicit def companyIpRange2Ref ( r: Option[ Array[CompanyIpRange]]) 98 | : ParserRef = ParserRef( companyIpRange = r.map(_.groupBy( _.companyId)) ) 99 | 100 | implicit def companyIps2Ref ( r: Option[Array[CompanyIp]]) 101 | : ParserRef = ParserRef( companyIps = r.map(_.groupBy(_.companyId)) ) 102 | 103 | 104 | implicit def geoIpRange2Ref ( r: Option[Map[Long, Array[GeoIpRange]]]) 105 | : ParserRef = ParserRef( geoIpRange = r) 106 | 107 | implicit def signatures2Ref ( r: Option[Array[Signature]]) 108 | : ParserRef = ParserRef( signatures = r.map{ e => 109 | e.groupBy( ds => (ds.signature, ds.vendor)) 110 | .map( e => ( e._1, e._2.head ) ) 111 | } ) 112 | 113 | implicit def replaceFields2Ref ( r: Option[Array[ ReplaceField]]) 114 | : ParserRef = ParserRef( replaceFields = r.map(_.groupBy(_.mid))) 115 | 116 | implicit def Fields2Ref ( r: Option[VariableKeys]) 117 | : ParserRef = ParserRef(fields = r) 118 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/reference/ParserRefMaker.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.reference 2 | 3 | /** 4 | * RefMaker trait 5 | * 6 | * You can change the DataSource by inheriting ParserRefMaker. (DB, zookeeper, stream, memory ..) 7 | * 8 | */ 9 | 10 | trait ParserRefMaker { 11 | 12 | def getGeoIPRange(): ParserRef 13 | def getCompanyIpRange(): ParserRef 14 | def getSignatures(): ParserRef 15 | def getAgents(): ParserRef 16 | def getTokenizeRules(): ParserRef 17 | def getArrangeRules(): ParserRef 18 | def getFields(): ParserRef 19 | def getReplaceFields(): ParserRef 20 | def getCompanyIps(): ParserRef 21 | 22 | def initialize(): ParserRef = { 23 | getCompanyIpRange() <+ 24 | getGeoIPRange() <+ 25 | getSignatures() <+ 26 | getAgents() <+ 27 | getTokenizeRules() <+ 28 | getArrangeRules() <+ 29 | getCompanyIps() <+ 30 | getFields() <+ 31 | getReplaceFields() 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/reference/ParserStructs.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.reference 2 | 3 | import scala.util.matching.Regex 4 | //todo ?? 5 | case class SOCServer( hostname: String) 6 | case class SOCServerRegex(regex: Regex) 7 | case class logEncoding( encoding: String) 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/reference/Zk2ParserRef.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.reference 2 | 3 | import com.adenium.app.config.Conf 4 | import com.adenium.common.{FieldKey, VariableKeys} 5 | import com.adenium.externals.zookeeper.ZkUtil.readZKArray 6 | import com.adenium.parser.structs._ 7 | import com.adenium.utils.IpUtil.ip2Long 8 | import com.adenium.utils.Logger 9 | import com.adenium.utils.May.{maybe, maybeWarn} 10 | import org.apache.curator.framework.CuratorFramework 11 | 12 | import scala.reflect.ClassTag 13 | 14 | /** Initialize reference data from Zookeeper node. 15 | * 16 | * Generates reference information for parsing from zookeeper nodes. 17 | * 18 | * */ 19 | case class Zk2ParserRef( ref : ( Option[CuratorFramework], Conf ) ) extends ParserRefMaker { 20 | 21 | val curator: Option[CuratorFramework] = ref._1 22 | val conf: Conf = ref._2 23 | 24 | def makeRefs[T:ClassTag] ( path: String)( f: Array[String] => T) 25 | : Option[Array[T]] = { 26 | 27 | maybeWarn { readZKArray( curator)( path)( f ) } 28 | } 29 | 30 | /** Generate country-specific IP range reference information. */ 31 | override def getGeoIPRange(): ParserRef = { 32 | 33 | val fn = conf.zkp.geoip 34 | 35 | makeRefs( fn ) { 36 | case Array(a1, a2, a3) => 37 | GeoIpRange( a1, 38 | a2.replaceAll(",", "").toLong, 39 | a3.replaceAll(",", "").toLong ) 40 | }.map( _.groupBy(_.key)) 41 | } 42 | 43 | /** Generate owner IP (public, private) information for the agent */ 44 | override def getCompanyIps(): ParserRef = { 45 | 46 | val fn = conf.zkp.cips 47 | 48 | // CustomerServerInfo 49 | // company_seq public_ip private_ip service_active\n 50 | 51 | makeRefs( fn) { 52 | case Array(a1, a2, a3) => 53 | CompanyIp( 54 | a1.toLong, 55 | a2, 56 | a3) 57 | } 58 | } 59 | 60 | /** Generate Agent's owner IP bands information */ 61 | override def getCompanyIpRange(): ParserRef = { 62 | 63 | val fn = conf.zkp.cips 64 | 65 | makeRefs( fn) { 66 | case Array(a1, a2, a3) => 67 | CompanyIpRange( 68 | a1.toInt, 69 | ip2Long(a2).getOrElse(0L), 70 | ip2Long(a3).getOrElse(0L)) 71 | } 72 | } 73 | 74 | /** Generates message signature information */ 75 | override def getSignatures(): ParserRef = { 76 | 77 | val fn = conf.zkp.sig 78 | 79 | makeRefs( fn) { 80 | case Array(a1, a2, a3, a4, a5, a6) => 81 | Signature( a1.toLong, 82 | a2, 83 | a3, 84 | a4, 85 | a5, 86 | a6) 87 | 88 | } 89 | } 90 | 91 | /** Generates agent information */ 92 | override def getAgents(): ParserRef = { 93 | 94 | val fn = conf.zkp.agn 95 | 96 | //agent_seq agent_ip agent_type_id active_flag agent_type company_seq company_group_seq agent_name agent_vendor_name company_name service_id agent_vendor_seq 97 | 98 | makeRefs( fn) { 99 | case Array( a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11) => 100 | Agent(a1, 101 | a2.toInt, 102 | a3.toLong, 103 | a4, 104 | a5.toLong, 105 | a6.toLong, 106 | a7, 107 | a8, 108 | a9.toLong, 109 | a10, 110 | a11 == "Y" 111 | ) 112 | } 113 | } 114 | 115 | /** Generates regex information */ 116 | override def getTokenizeRules(): ParserRef = { 117 | 118 | val fn = conf.zkp.tokrul 119 | // agent_type_id ruleId, regEx, agentTypeName 120 | 121 | makeRefs( fn) { 122 | case Array( a1, a2, a3, a4 ) => 123 | TokenizeRule( 124 | maybe(a1.toInt)getOrElse 0, 125 | maybe(a2.toLong)getOrElse 0L, 126 | a3, 127 | a4) 128 | } 129 | } 130 | 131 | 132 | /** The rule that links the tokenization result to the normalization field. */ 133 | override def getArrangeRules(): ParserRef = { 134 | 135 | val fn = conf.zkp.arrrul 136 | 137 | // idx set_type set_priority column_idx data_format capture_order order_index column_name column_data_type is_trans 138 | makeRefs( fn) { 139 | case Array(a1, a2, a3) => 140 | ArrangeRule( 141 | a1.toInt, 142 | a2.toInt, 143 | a3.toInt) 144 | } 145 | } 146 | 147 | /** Generate varfields information */ 148 | override def getFields(): ParserRef = { 149 | 150 | val fn = conf.zkp.varfields 151 | 152 | /** 153 | * fid : field Id, (Integer) 154 | * fname : field name used in presenting a field by name, ( Alpha_Numeric_String) 155 | * ftype : field type, currently NOT USED.. YET. (default = FieldType.TString) 156 | * 157 | */ 158 | val keys = makeRefs( fn) { 159 | case Array( fid, fname ) => FieldKey( fid.toInt, fname) 160 | } 161 | 162 | Some( VariableKeys(keys)) 163 | } 164 | 165 | /** Replaceable fields*/ 166 | override def getReplaceFields(): ParserRef = { 167 | 168 | val fn = conf.zkp.repfld 169 | 170 | makeRefs( fn) { 171 | case Array(a1, a2, a3, a4) => 172 | ReplaceField( 173 | a1.toInt, 174 | a2, 175 | a3, 176 | a4.toLong 177 | ) 178 | } 179 | } 180 | 181 | 182 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/reference/package.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser 2 | 3 | /** Provides the generation and initialization of reference data for normalization and parsing. */ 4 | package object reference { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/Agent.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | import com.adenium.utils.May.maybeWarn 4 | import com.adenium.utils.IpUtil.ip2Long 5 | 6 | /** Agent 7 | * 8 | * The Agent has "specific Sensor + Owner Information" 9 | * 10 | * @constructor 11 | * @param host System host information to identify the agent [ from syslog header: host] 12 | * @param ipLong Host to long type : string = 0L, ip = long 13 | * @param agentId Agent's Unique Identifiable ID 14 | * @param sensorType Sensor Type [ FW, WAF, DDOS, IPS, IDS ... ] 15 | * @param companyId ID of Agent owner 16 | * @param companyName Name of Agent owner 17 | * @param companyGroupId Group ID of Agent owner 18 | * @param sensorId Unique ID to identify the Sensor 19 | * @param sensor Model name of Sensor 20 | * @param vendorId Vendor ID of Sensor 21 | * @param vendorName Name of vendor 22 | * @param active use flag [ 'Y' or 'N' ] 23 | */ 24 | case class Agent(host: String, 25 | ipLong:Long, 26 | agentId: Long, 27 | companyId: Long, 28 | companyName: String, 29 | companyGroupId: Long, 30 | sensorId: Long, 31 | sensor: String, 32 | sensorType: String, 33 | vendorId: Long, 34 | vendorName: String, 35 | active: Boolean) 36 | 37 | 38 | object Agent { 39 | def apply(agentIp: String, 40 | agentId: Long, 41 | companyId: Long, 42 | companyName: String, 43 | companyGroupId: Long, 44 | sensorId: Long, 45 | sensor: String, 46 | sensorType: String, 47 | vendorId: Long, 48 | vendorName: String, 49 | active: Boolean): Agent = { 50 | 51 | val ipLong = ip2Long( agentIp).getOrElse(0L) 52 | Agent( 53 | host= agentIp, 54 | ipLong= ipLong, 55 | 56 | agentId= agentId, 57 | sensorType= sensorType, 58 | 59 | active= active, 60 | companyId= companyId, 61 | companyName= companyName, 62 | companyGroupId= companyGroupId, 63 | sensorId= sensorId, 64 | sensor= sensor, 65 | vendorId= vendorId, 66 | vendorName= vendorName 67 | ) 68 | } 69 | 70 | def filter( host: String, agent: Option[Map[Long, Array[Agent] ] ]) 71 | : Option[Array[Agent]] = { 72 | 73 | val ret = 74 | for { 75 | m <- agent 76 | ipLong = ip2Long(host) getOrElse 0L 77 | ar <- m.get( ipLong) 78 | 79 | } yield { 80 | ar.filter ( _.host == host ) 81 | } 82 | 83 | ret 84 | 85 | } 86 | 87 | ////////////////////////////////////////////////// 88 | def filter( companyId: String, agentId: String, agent: Option[ Map[Long, Array[Agent]]]) 89 | : Option[Array[Agent]] = { 90 | 91 | val ret = 92 | for { 93 | m <- agent 94 | cid <- maybeWarn { companyId.toLong } 95 | id <- maybeWarn { agentId.toInt } 96 | 97 | ar <- m.get( id) 98 | 99 | } yield { 100 | 101 | ar.filter( p => p.agentId == id && p.companyId == cid) 102 | } 103 | 104 | ret 105 | } 106 | 107 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/ArrangeRule.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | /** The rule that links the tokenization result to the normalization field. 4 | * 5 | * Associate the regular expression capture sequence with the normalization field ID 6 | * 7 | * @constructor 8 | * @param tokenizeRuleId Regex ID 9 | * @param captureOrder Regex Match group Sequence 10 | * @param fieldId Predefined field id 11 | */ 12 | case class ArrangeRule( tokenizeRuleId: Int, 13 | captureOrder: Int, 14 | fieldId:Int 15 | ) 16 | 17 | object ArrangeRule { 18 | 19 | def findInvalidRules(rules: Option[ Map[Int, Array[ArrangeRule]]] ) 20 | : Option[ Map[Int, Array[ArrangeRule]]] = { 21 | 22 | val invalids = 23 | rules.map { tbl => 24 | tbl.filterNot { case (_, mr) => 25 | mr.map(_.captureOrder).sorted.deep == (1 to mr.length).toArray.deep && // captureOrder is valid? 26 | mr.map(_.fieldId).toSet.size == mr.length // unique? 27 | } 28 | } 29 | invalids 30 | } 31 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/CompanyHost.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | case class CompanyHost( companyId: Long, 4 | name: String, 5 | ip: String) 6 | 7 | object CompanyHost { 8 | 9 | def hostname2Ip( company: Long, hostname: String)( companyHost2Ip: Option[Map[Long, Array[CompanyHost]]]) 10 | : Option[String] 11 | = { 12 | val ret = 13 | companyHost2Ip.flatMap ( 14 | _.get(company).flatMap ( 15 | _.find( _.name == hostname) )).map( _.ip) 16 | 17 | ret 18 | } 19 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/CompanyIp.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | case class CompanyIp( companyId: Long, 4 | publicIp: String, 5 | privateIp: String="") 6 | 7 | object CompanyIp { 8 | 9 | def isCompanyIp(company: Long, companyIps: Option[Map[Long, Array[CompanyIp]]], ips: String*) 10 | : Boolean = { 11 | 12 | val ret = 13 | companyIps.exists( 14 | _.get(company).exists { ar => 15 | ips.exists { ip => 16 | ar.exists(e => 17 | e.publicIp == ip || e.privateIp == ip) 18 | } 19 | }) 20 | 21 | ret 22 | } 23 | 24 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/CompanyIpRange.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | import com.adenium.utils.Logger 4 | import com.adenium.utils.IpUtil.ip2Long 5 | 6 | case class CompanyIpRange( companyId: Long, 7 | sip:Long, 8 | eip:Long ) 9 | 10 | object CompanyIpRange { 11 | 12 | def apply( cid: Long, startIP: String, endIP: String) 13 | : CompanyIpRange = { 14 | 15 | val sip = ip2Long(startIP).getOrElse { 16 | Logger.logWarning( s"""CompanyIpRange : company's sip is not ip format = ${(cid, startIP)}""") 17 | 0L 18 | } 19 | val eip = ip2Long(endIP).getOrElse{ 20 | Logger.logWarning( s"""CompanyIpRange : company's eip is not ip format = ${(cid, endIP)}""") 21 | 0L 22 | } 23 | 24 | CompanyIpRange( cid, sip, eip) 25 | } 26 | 27 | def isCompanyIp( ip: String, arr : Array[CompanyIpRange]): Boolean = 28 | { 29 | ip2Long(ip).exists { i => 30 | arr.exists( p => p.sip <= i && i <= p.eip ) 31 | } 32 | } 33 | 34 | 35 | def isCompanyIp(company: Long, ranges: Option[ Map[Long, Array[CompanyIpRange]]], ips: String*) 36 | : Boolean = { 37 | 38 | val check = 39 | ranges.exists { 40 | _.get(company).exists { ar => 41 | ips.exists { ipStr => 42 | isCompanyIp( ipStr, ar) 43 | } 44 | } 45 | } 46 | check 47 | } 48 | 49 | def locationString ( isCompanyIp: Boolean): String = if ( isCompanyIp) "IN" else "OUT" 50 | 51 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/GeoIpRange.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | import com.adenium.utils.IpUtil.ip2PublicLong 4 | 5 | import scala.annotation.tailrec 6 | 7 | case class GeoIpRange ( key: Long, 8 | sid: Int, 9 | eid: Int, 10 | nation: String, 11 | sip: Long, 12 | eip: Long) 13 | 14 | ///////////////////////////////// 15 | 16 | object GeoIpRange { 17 | 18 | private val factor = math.pow(2, 23).toLong // 8,388,608 19 | 20 | def apply(name: String, sip: Long, eip: Long) 21 | : GeoIpRange = { 22 | 23 | val s = sip min eip 24 | val b = sip max eip 25 | 26 | val sid = math.floor((s / factor).toDouble).toInt 27 | val eid = math.floor((b / factor).toDouble).toInt 28 | val key = s >> 24 29 | 30 | GeoIpRange( key, sid, eid, name, sip, eip ) 31 | 32 | } 33 | 34 | def search(arr: Array[GeoIpRange], ip: Long): Option[Int] = { 35 | 36 | @tailrec 37 | def go( lo: Int, hi: Int): Option[Int] = { 38 | if (lo > hi) 39 | None 40 | else { 41 | val mid: Int = lo + (hi - lo) / 2 42 | arr(mid) match { 43 | case range if range.sip <= ip && ip <= range.eip => Some(mid) 44 | case range if range.eip < ip => go(mid + 1, hi) 45 | case _ => go(lo, mid - 1) 46 | } 47 | } 48 | } 49 | 50 | go(0, arr.length - 1) 51 | } 52 | 53 | def getCountry(ip: Long, geoips: Map[Long, Array[GeoIpRange]] ) 54 | : Option[String] = { 55 | 56 | val fac = math.floor((ip / factor).toDouble).toInt 57 | val key = ip >> 24 58 | 59 | for { 60 | arr <- geoips.get( key) 61 | idx <- search( arr, ip) 62 | } yield { 63 | arr( idx).nation 64 | } 65 | } 66 | 67 | def getCountry( ip: String, geoips: Option[Map[Long, Array[GeoIpRange]]] ) 68 | : Option[String] = { 69 | 70 | val ret = 71 | for { 72 | ranges <- geoips 73 | ipLong <- ip2PublicLong( ip) 74 | country <- getCountry( ipLong, ranges) 75 | } yield { 76 | country 77 | } 78 | ret 79 | } 80 | } 81 | 82 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/ReplaceField.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | case class ReplaceField( mid: Int, 4 | instr: String, 5 | outstr: String, 6 | vendor: Long, 7 | ignoreVendor: Boolean = true) 8 | 9 | object ReplaceField { 10 | 11 | def find( parsed: Array[(Int, String)], agn: Agent, replaces: Option[Map[Int, Array[ReplaceField]]]) 12 | : Array[(Int, String)] = { 13 | 14 | def replaceString( mid: Int, value: String): Option[(Int, String)] = { 15 | for { 16 | rfs <- replaces 17 | arr <- rfs.get( mid ) 18 | rep = arr.filter ( _.instr == value) // todo: may need some performance improvement... 19 | ret <- rep.find( _.vendor == agn.vendorId) orElse rep.find ( _.ignoreVendor ) 20 | 21 | } yield { 22 | mid -> ret.outstr 23 | } 24 | } 25 | 26 | val ret = parsed.flatMap { case( mid, value) => replaceString ( mid, value.trim.toUpperCase) } 27 | ret 28 | 29 | } 30 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/Signature.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | /** Categories of message signatures defined by sensor 4 | * 5 | * @constructor 6 | * @param signature 7 | * @param vendor 8 | * @param category1 9 | * @param category2 10 | * @param category3 11 | * @param category4 12 | */ 13 | case class Signature(vendor: Long = 0, 14 | signature: String="", 15 | category1: String="", 16 | category2: String="", 17 | category3: String="", 18 | category4: String="") 19 | 20 | object Signature { 21 | 22 | def find( signature: String, vendor: Long = 0)(omap: Option[Map[(String, Long), Signature]] ) 23 | : Option[ Signature ] = { 24 | 25 | val ret = 26 | omap.flatMap( map => 27 | map.get(signature, vendor) 28 | ) 29 | ret 30 | } 31 | 32 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/TokenizeRule.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser.structs 2 | 3 | import scala.util.matching.Regex 4 | 5 | /** 6 | * Regular expression for tokenization. 7 | * 8 | * @param id ID to identify the regular expression 9 | * @param sensorId Id of the sensor to which the regular expression is applied 10 | * @param sensorType Sensor Type 11 | * @param regEx Regular expression 12 | */ 13 | case class TokenizeRule(id: Int, 14 | sensorId: Long, 15 | sensorType: String, 16 | regEx: Regex 17 | ) 18 | 19 | object TokenizeRule { 20 | 21 | def apply(id: Int, sensorId: Long, sensorType: String, regex: String ) 22 | : TokenizeRule = { 23 | TokenizeRule( id, sensorId, sensorType, regex.r ) 24 | } 25 | 26 | def filter( agents: Array[Agent], rules: Option[Map[Long, Array[TokenizeRule]]]) 27 | : Array[ TokenizeRule] = { 28 | 29 | // agentTypeID de-dupulication : to be deleted later.. after DB refining 30 | val uniques = agents.groupBy( _.sensorId).values.map( _.head).toArray 31 | 32 | val ret = 33 | uniques.flatMap { agn => 34 | rules.flatMap{ rls => 35 | rls.get( agn.sensorId ) 36 | } 37 | }.flatten 38 | 39 | ret 40 | } 41 | } -------------------------------------------------------------------------------- /src/main/scala/com/adenium/parser/structs/package.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.parser 2 | 3 | /** Reference information data structure for normalization*/ 4 | package object structs { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/Args.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | 3 | /** 4 | * Argument handling class 5 | */ 6 | object Args { 7 | 8 | /** 9 | * 10 | * args: Array[String], keywords 11 | - make Map[String, Array[String ] ] 12 | * 13 | * keyword = "option1|option2|option3|..." 14 | - args = Array( "-option1", "opt11", "opt12", "-option2", "opt21", "-option3", ....) 15 | * 16 | * Map 17 | - option1 --> Array("opt11", "opt12") 18 | - option2 --> Array("opt21") 19 | - option3 --> Array() 20 | * 21 | * @param args : arguments 22 | * @param keywords 23 | * @return 24 | */ 25 | 26 | def args2map (args: Array[String], keywords : String ) 27 | : Map[String, Array[String ] ] = { 28 | 29 | def toArr( args: Array[ String ]) = args.foldLeft( Array[ Array[ String ] ]() ) { 30 | 31 | case (b, arg) => 32 | if ( arg.startsWith( "-" ) ) { 33 | Array( arg ) +: b 34 | } else { 35 | b.headOption foreach { _ => b.update( 0, b.head :+ arg ) } 36 | b 37 | } 38 | } 39 | 40 | val opts = keywords.split( "\\|" ) 41 | 42 | toArr( args ).flatMap { arg => 43 | opts 44 | .find( opt => arg.headOption.exists( ar => ar.dropWhile( _ == '-' ).equals( opt ) ) ) 45 | .map( _ -> arg.drop( 1 ) ) 46 | }.toMap 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/IpUtil.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | 3 | import May.maybe 4 | 5 | /** 6 | * Ip address handling class 7 | */ 8 | object IpUtil { 9 | 10 | /** 11 | * ip --> Long 12 | * @param ip 13 | * @return 14 | */ 15 | def ip2Long(ip: String): Option[Long] = { 16 | 17 | maybe { 18 | 19 | val arr = ip split "\\." 20 | if (arr.length == 4) { 21 | 22 | val ret = 23 | arr(0).trim.toLong * math.pow(2, 24).toLong + 24 | arr(1).trim.toLong * math.pow(2, 16).toLong + 25 | arr(2).trim.toLong * math.pow(2, 8).toLong + 26 | arr(3).trim.toLong 27 | 28 | Some(ret) 29 | } 30 | else None 31 | 32 | }.flatten 33 | } 34 | 35 | /** 36 | * RFC 1918 private IPv4 address space 37 | * 38 | * {{{ 39 | * RFC1918 name IP address range number of addresses 40 | * 24-bit block 10.0.0.0–10.255.255.255 16777216 41 | * 20-bit block 172.16.0.0–172.31.255.255 1048576 42 | * 16-bit block 192.168.0.0–192.168.255.255 65536 43 | * }}} 44 | * * 45 | * [[https://en.wikipedia.org/wiki/Private_network]] 46 | */ 47 | val privateIPs: Array[(Long, Long)] = Array[ (Long, Long)] ( 48 | ip2Long( "10.0.0.0").get -> ip2Long( "10.255.255.255").get, 49 | ip2Long( "172.16.0.0").get -> ip2Long( "172.31.255.255").get, 50 | ip2Long( "192.168.0.0").get -> ip2Long( "192.168.255.255").get, 51 | ip2Long( "127.0.0.1").get -> ip2Long( "127.255.255.254").get 52 | ) 53 | 54 | val invalidIps: Array[Long] = Array[Long]( 55 | ip2Long("0.0.0.0").get, 56 | ip2Long("255.255.255.255").get 57 | ) 58 | 59 | def isSpecialIp( ip: Long): Boolean = privateIPs.exists(r => r._1 <= ip && ip <= r._2 ) 60 | def isInvalidIp( ip: Long): Boolean = invalidIps.contains( ip ) 61 | 62 | def ip2PublicLong( ip: String): Option[Long] = ip2Long(ip).find { ip => !isSpecialIp(ip) && !isInvalidIp(ip) } 63 | } 64 | 65 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/Logger.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | 3 | /** 4 | * Created by SecudiumDev on 2017-10-18. 5 | */ 6 | 7 | object Logger extends Serializable { 8 | 9 | //ref => https://www.mail-archive.com/user@spark.apache.org/msg29010.html 10 | 11 | /** 12 | * error : break conditions 13 | * warning : warning or important, main process's flow, one-time job.. stack tracing 14 | * info : optional info, sub-routine's flow 15 | * debug : 16 | * trace 17 | * 18 | */ 19 | 20 | @transient private lazy val log = org.apache.log4j.Logger.getLogger( getClass.getName) 21 | 22 | def logWarning( str: String ): Unit = log.warn( str) 23 | def logWarning( str: String, e: Throwable): Unit = log.warn( str, e) 24 | def logInfo( str: String, e: Throwable): Unit = log.info( str, e) 25 | def logInfo( str: String): Unit = log.info( str) 26 | def logDebug( str: String ): Unit = log.debug( str) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/May.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | /** 3 | * Exception handling class 4 | */ 5 | 6 | object May { 7 | 8 | def memo( str: => String): Unit = { Logger.logWarning(s"[ Memo ] $str ") } 9 | 10 | def state( str: => String): Unit = { Logger.logWarning(s"[ State ] ========== $str ==========") } 11 | 12 | /** 13 | * f : function returning value t of type T, which can throw an exception. 14 | * log : exception log string 15 | * 16 | * @param f 17 | * @param log 18 | * @tparam A 19 | * @return 20 | */ 21 | def warn[A]( a : => Option[A])(str : => String) 22 | : Option[A] = { 23 | if ( a.isEmpty) 24 | Logger.logWarning(s"[ Warn ] : None = $str") 25 | 26 | a 27 | } 28 | 29 | /** 30 | * 31 | * @param a 32 | * @tparam A 33 | * @return 34 | */ 35 | def maybe[A](a : => A) 36 | : Option[A] = { 37 | try Some(a) catch { case _: Throwable => None } 38 | } 39 | 40 | def maybeInfo[A](a : => A)(str: => String) 41 | : Option[A] = { 42 | try Some(a) catch { case e: Throwable => Logger.logWarning( "[ maybeInfo ]" + str ); None } 43 | } 44 | 45 | def maybeWarn2[A](a : => A)( str: String = "[ maybeWarn ]") 46 | : Option[A] = { 47 | try Some(a) catch { case e: Throwable => Logger.logWarning( str, e ); None } 48 | } 49 | 50 | def maybeWarn[A](a : => A, str: => String = "[ maybeWarn ]") 51 | : Option[A] = { 52 | try Some(a) catch { case e: Throwable => Logger.logWarning( str, e ); None } 53 | } 54 | 55 | /** 56 | * 57 | * loan pattern 58 | * r : Closable Resource which may throw an exception. 59 | * f : function using closable resource r 60 | * needClose : close after f. 61 | * 62 | * @param r 63 | * @param needClose 64 | * @param f 65 | * @tparam A 66 | * @return 67 | */ 68 | def using[A](r : => AutoCloseable, needClose: Boolean = true)(f : AutoCloseable => A) 69 | : Option[A] = { 70 | 71 | maybeWarn( r ).flatMap { rc => 72 | try { 73 | Some( f( rc ) ) 74 | } catch { 75 | 76 | case e: Throwable => 77 | Logger.logInfo( "[ using ]", e ) 78 | None 79 | 80 | } finally { 81 | 82 | if ( needClose ) 83 | maybeWarn( rc.close() ) 84 | 85 | } 86 | } 87 | } 88 | 89 | /** 90 | * 91 | * @param str 92 | * @return 93 | */ 94 | def lift[A, B]( f: A => B) 95 | : A => Option[B] = (a: A) => maybe( f( a)) 96 | 97 | } 98 | 99 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/StringUtil.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | 3 | import com.adenium.utils.May.maybe 4 | 5 | object StringUtil { 6 | def lift( str: String ): Option[String ] = { 7 | if ( str == null || str.isEmpty ) 8 | None 9 | else 10 | Some( str) 11 | } 12 | 13 | def unescapeUnicode(str: String): String = 14 | 15 | """\\u+([0-9a-fA-F]{4})""".r.replaceAllIn(str, 16 | m => Integer.parseInt(m.group(1), 16).toChar match { 17 | case '\\' => """\\""" 18 | case '$' => """\$""" 19 | case c => c.toString 20 | }) 21 | 22 | def unescapeString( s: String): String = { 23 | 24 | maybe { 25 | StringContext.treatEscapes(StringUtil.unescapeUnicode(s)) 26 | } getOrElse s 27 | 28 | } 29 | 30 | /** 31 | * XML Escape / Unescape 32 | * 33 | * https://www.freeformatter.com/xml-escape.html 34 | * ' is replaced with ' 35 | * " is replaced with " 36 | * & is replaced with & 37 | * < is replaced with < 38 | * > is replaced with >* 39 | */ 40 | def XMLEscape(str: String): String = { 41 | str 42 | .replace("&", "&") 43 | .replace("'", "'") 44 | .replace("\"", """) 45 | .replace("<", "<") 46 | .replace(">", ">") 47 | } 48 | 49 | /** 50 | * JSON String Escape / Unescape 51 | * 52 | * https://www.freeformatter.com/json-escape.html 53 | *Backspace is replaced with \b 54 | *Form feed is replaced with \f 55 | *Newline is replaced with \n 56 | *Carriage return is replaced with \r 57 | *Tab is replaced with \t 58 | *Double quote is replaced with \" 59 | *Backslash is replaced with \\ 60 | */ 61 | def JSONEscape( str: String): String ={ 62 | 63 | str 64 | .replace("\\", "\\\\") 65 | .replace("\"", "\\\"") 66 | 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/Timer.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | 3 | import java.time.{Instant, ZoneId} 4 | import java.time.format.DateTimeFormatter 5 | import java.util.Locale 6 | 7 | import com.adenium.utils.May._ 8 | 9 | object Timer { 10 | 11 | def currentMillis: String = Instant.now().toEpochMilli.toString 12 | 13 | ////////////////////////////////////////////////// 14 | 15 | def Timer[A] ( f : => A) : ( A, Long) = { 16 | val s = System.currentTimeMillis 17 | val r = f 18 | val e = System.currentTimeMillis 19 | ( r, e-s ) 20 | } 21 | 22 | def TimeLog[A]( f : => A)(msg: String): A = { 23 | val ( r, t) = Timer( f) 24 | Logger.logWarning( s"[ Time spent ] $t in $msg") 25 | r 26 | } 27 | 28 | 29 | def UnitTimer( f : => Unit) : Long = { 30 | val ( _, t) = Timer( f) 31 | t 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/WOption.scala: -------------------------------------------------------------------------------- 1 | package com.adenium.utils 2 | 3 | /** 4 | * Monoid 5 | * append : some binary operation S X S -> S, which holds associativity rule. 6 | * empty : identity element 7 | */ 8 | trait Monoid[A] { 9 | 10 | def append(a1: A, a2: A): A 11 | def empty: A 12 | 13 | } 14 | 15 | object Monoid { 16 | 17 | /** 18 | * refer to implicit conversion in scala 19 | * @tparam A 20 | * @return 21 | */ 22 | implicit def ListMonoid[ A ]: Monoid[ List[ A ] ] = new Monoid[ List[ A ] ] { 23 | def append(a1: List[ A ], a2: List[ A ]): List[ A ] = a1 ::: a2 24 | def empty: List[ A ] = Nil 25 | } 26 | } 27 | 28 | /** writer Option monad */ 29 | case class WOption[L, +A](log: L, option: Option[A]) { 30 | 31 | /** Returns a some containing the result of applying f to this option's 32 | * value if this option is nonempty. 33 | * Otherwise return none. 34 | * 35 | * @note This is similar to `flatMap` except here, 36 | * f does not need to wrap its result in an option. 37 | * 38 | * @param f the function to apply 39 | * @see flatMap 40 | * @see foreach 41 | */ 42 | def map[B](f: A => B) 43 | : WOption[L, B] = WOption( log, option map f ) 44 | 45 | /** Returns the result of applying f to this option's value if 46 | * this option is nonempty. 47 | * Returns none if this option is empty. 48 | * Slightly different from `map` in that f is expected to 49 | * return an option (which could be none). 50 | * 51 | * ++++++++++++++++++++++++++++++++++++++++++++++++++++ 52 | * @param f the function to apply 53 | * @see map 54 | * @see foreach 55 | */ 56 | def flatMap[B](f: A => WOption[L, B])(implicit m: Monoid[L]) 57 | : WOption[ L, B ] = { 58 | 59 | option.map{ f(_).prependLog( log) }.getOrElse( WOption(log, None: Option[B])) 60 | } 61 | 62 | def appendLog(a: L) (implicit m: Monoid[L]) : WOption[ L, A ] = WOption( m.append(log, a), option) 63 | def prependLog(a: L) (implicit m: Monoid[L]) : WOption[ L, A ] = WOption( m.append( a, log), option) 64 | 65 | def isEmpty: Boolean = option.isEmpty 66 | def nonEmpty: Boolean = option.nonEmpty 67 | def isDefined: Boolean = option.isDefined 68 | def get: A = option.get 69 | 70 | def orElse [B >: A]( alternative : => WOption[L, B])(implicit m: Monoid[L]) 71 | : WOption[ L, B ] = if ( option.nonEmpty) this else alternative.prependLog( log) 72 | 73 | // log messages are dropped.. 74 | def getOrElse [B >: A] ( default : => B)(implicit m: Monoid[L]) 75 | : B = if ( option.isEmpty) default else this.option.get 76 | 77 | } 78 | 79 | object WOption { 80 | import scala.language.{implicitConversions, reflectiveCalls} 81 | import scala.reflect.ClassTag 82 | 83 | def unit[L, A]( option : Option[A]) (implicit m: Monoid[L]) 84 | : WOption[ L, A ] = WOption( m.empty, option) 85 | 86 | def noLog[L, A](a: Option[A])(implicit m: Monoid[L]) 87 | : WOption[L, A ] = WOption.unit[L, A](a) 88 | 89 | /** 90 | * Write a function sequence that combines a list of Option s into one Option 91 | * containing a list of all the Some values in the original list. If the 92 | * original list contains None even once, the result of the function should be 93 | * None; otherwise the result should be Some with a list of all the values. 94 | * 95 | * @param a 96 | * @param m 97 | * @param wOptionLogOn 98 | * @tparam L 99 | * @tparam A 100 | * @return 101 | */ 102 | 103 | def sequence[L, A: ClassTag](a: Array[WOption[L, A]])( implicit m : Monoid[L], wOptionLogOn: WOptionLogOn) 104 | : WOption[L, Array[A]] = { 105 | 106 | if( wOptionLogOn.on ) { 107 | a.foldRight( WOption.unit( Some(Array[A]()) )){ (x, y) => 108 | WOption( 109 | m.append(x.log, y.log), 110 | y.option.flatMap( vy => x.option.map( vx => vx +: vy )).orElse( y.option) ) 111 | } 112 | } 113 | else { 114 | WOption ( m.empty, Some(a.flatMap( _.option)) ) 115 | } 116 | } 117 | 118 | /** implicit case class to decide whether log or not. */ 119 | case class WOptionLogOn(on: Boolean ) { 120 | def log(str: => String): String = if (on) str else "" 121 | } 122 | 123 | implicit class WOptionMaker[L]( a: L) { 124 | /** implicit constructor 125 | * 126 | * create WOption with given log string, and Option value 127 | * "" ~> Option("abc") ==> WOption (List(""), Option("abc")) 128 | */ 129 | def ~>[B]( b: Option[B])( implicit log: WOptionLogOn): WOption[List[L], B] 130 | = if( log.on) WOption(List(a), b) else WOption(Nil, b) 131 | 132 | /** implicit operator : append log to current WOption 133 | * "additional log" +> WOption 134 | */ 135 | def +>[B](b: WOption[List[L], B])(implicit m: Monoid[List[L]], log: WOptionLogOn): WOption[ List[L], B ] 136 | = if( log.on) b.appendLog(List(a)) else b 137 | } 138 | 139 | // implicit def ListWOption[L](a: L)(log: Boolean = implicitly[WOptionLogOn].on) = new { 140 | // 141 | // def ~>[B]( b: Option[B]) 142 | // = WOption(List(a), b) 143 | // 144 | // def +>[B](b: WOption[List[L], B])(implicit m: Monoid[List[L]]): WOption[ List[L], B ] 145 | // = b.appendLog(List(a)) 146 | // } 147 | 148 | 149 | } 150 | 151 | -------------------------------------------------------------------------------- /src/main/scala/com/adenium/utils/package.scala: -------------------------------------------------------------------------------- 1 | package com.adenium 2 | /** Utility Package*/ 3 | package object utils { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /test/sample/AdeniumParser.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SOCLabs/Adenium/fa2eb2ab59149f15106ad8da7180735e09ff8317/test/sample/AdeniumParser.jar -------------------------------------------------------------------------------- /test/sample/RefData/agentInfo.ref: -------------------------------------------------------------------------------- 1 | agentIp agentId companyId companyName companyGroupId sensorId sensor sensorType vendorId vendorName active 2 | 211.169.244.2 12346776 2425 SUNLEAF 77 1234568257 WebFront K2400 WF 26 PIOLINK Y 3 | -------------------------------------------------------------------------------- /test/sample/RefData/arrangeRules.ref: -------------------------------------------------------------------------------- 1 | idx capture_order column_idx 2 | 100 1 8 3 | 100 2 9 4 | 100 3 12 5 | 100 4 13 6 | 100 5 18 7 | 100 6 33 8 | 100 7 4 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /test/sample/RefData/companyIpRange.ref: -------------------------------------------------------------------------------- 1 | company_seq start_ip end_ip 2 | 12346776 211.169.244.2 211.169.244.10 -------------------------------------------------------------------------------- /test/sample/RefData/companyServerIp.ref: -------------------------------------------------------------------------------- 1 | companyId publicIp privateIp 2 | 12346776 211.169.244.2 3 | -------------------------------------------------------------------------------- /test/sample/RefData/fields.ref: -------------------------------------------------------------------------------- 1 | idx column_name 2 | 1 1st Category 3 | 2 2nd Category 4 | 3 Customer 5 | 4 Device Type 6 | 5 Vendor 7 | 6 Event Name 8 | 7 Serverity (Infosec) 9 | 8 Serverity (Device) 10 | 9 Country (Src IP) 11 | 10 Country (Dst IP) 12 | 11 Src IP 13 | 12 Src Port 14 | 13 Dst IP 15 | 14 Dst Port 16 | 15 Protocol 17 | 16 Signature Name 18 | 17 Agent IP 19 | 18 Code 20 | 19 Count 21 | 20 Direction (Src IP) 22 | 21 Direction (Dst IP) 23 | 22 Start Time 24 | 23 End Time 25 | 24 Log Time 26 | 25 Rule Action 27 | 26 Rcvd Bytes 28 | 27 Send Bytes 29 | 28 Message-ID 30 | 29 Src Zone 31 | 30 Dst Zone 32 | 31 Service 33 | 32 Duration 34 | 33 ACL Name 35 | 34 Action 36 | 35 N_CPU 37 | 36 N_USEDMEM 38 | 37 N_BPS 39 | 38 N_PPS 40 | 39 N_SESSION 41 | 40 N_PING 42 | 41 String 43 | 42 Raw Data 44 | 43 URL 45 | 44 URI 46 | 45 File Type 47 | 46 File Hash 48 | 47 Recipient 49 | 48 Sender 50 | 49 Attachment 51 | 50 Src MAC 52 | 51 Dst MAC 53 | 52 URI Params 54 | 53 Customer Group 55 | 54 3rd Category 56 | 55 Device Model 57 | 56 Payload 58 | 57 Header 59 | 58 N_TCP 60 | 59 N_HTTP 61 | 60 N_HTTPS 62 | 61 N_SNMP 63 | 68 Customer Name 64 | 75 Repeat Count 65 | 63 Filier2 66 | 62 Filier3 67 | 64 Filier1 68 | -------------------------------------------------------------------------------- /test/sample/RefData/replaceFields.ref: -------------------------------------------------------------------------------- 1 | mid instr outstr vendor 2 | 4 110600275 시스템 폴더 접근 취약점 26 3 | 4 USER_PASSWD_ATTACK 사용자 패스워드 접근 공격 26 4 | 4 110600277 0027: IP Options: Record Route (RR) Security 26 5 | -------------------------------------------------------------------------------- /test/sample/RefData/signatures.ref: -------------------------------------------------------------------------------- 1 | vendor signature category1 category2 category3 category4 2 | 26 시스템 폴더 접근 취약점 Security Exploit Overflow_Buffers Web_Application_Vulnerability 3 | 26 사용자 패스워드 접근 공격 Security Exploit Overflow_Buffers Web_Application_Vulnerability 4 | 26 0027: IP Options: Record Route (RR) Security Anomaly_Activity Network Anomaly_Traffic 5 | -------------------------------------------------------------------------------- /test/sample/RefData/tokenizeRules.ref: -------------------------------------------------------------------------------- 1 | id sensorId sensorType regex 2 | 100 1234568257 WF src_ip="(.+?)".+src_port="(.+?)".+dest_ip="(.+?)".+dest_port="(.+?)".+host=\"(.+?)\"\,.+block="(yes|no)".+sigid="(.+?)" -------------------------------------------------------------------------------- /test/sample/Sample_logs/WebFront.log: -------------------------------------------------------------------------------- 1 | <44>Aug 7 17:49:53 211.169.244.2 (warning) kernel: [WEBFRONT/0x00726001] Violated SQL Injection - the form field isn't allowed. (log_id="2085090068",app_name="07_purunetedu",app_id="7",src_if="waf",src_ip="116.46.237.195",src_port="52453",dest_ip="211.169.244.10",dest_port="80",forwarded_for="",host="www.purunetedu.com",url="/mystudyroom/questionbank/classlistAjax.prn",sig_warning="Middle",url_param="",block="no",evidence_id="1910044145",owasp="A1",field="subjCd",sigid="110600275",data="0-0-0") 2 | --------------------------------------------------------------------------------