├── LICENSE ├── README.md ├── conf ├── elasticsearch.yml ├── filebeat.service ├── filebeat.yml ├── kibana.service ├── kibana.yml ├── logstash.service └── logstash.yml ├── image ├── kibana.png ├── kibana1.png ├── kibana10.png ├── kibana11.png ├── kibana12.png ├── kibana13.png ├── kibana14.png ├── kibana15.png ├── kibana16.png ├── kibana17.png ├── kibana2.png ├── kibana3.png ├── kibana4.png ├── kibana5.png ├── kibana6.png ├── kibana7.png ├── kibana8.png └── kibana9.png ├── logstash_conf ├── grok.conf └── ruby.conf ├── sample ├── classes.json ├── filebeat-tutorial.log └── logs.jsonl ├── tuto └── tuto_ubuntu /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elastic Stack Tutorial 2 | 김종민님의 웨비나, 박상헌님의 ELK 깃헙 자료, 허민석님의 깃헙 자료를 활용하여 만든 엘라스틱 스택 튜토리얼입니다. 3 | 4 | ## Version 5 | * CentOS 7.x 6 | * Elastic Stack 6.7.0 7 | 8 | ## Tutorial 준비 9 | - [HEAD 크롬 플러그인](https://chrome.google.com/webstore/detail/multi-elasticsearch-head/cpmmilfkofbeimbmgiclohpodggeheim) 10 | - CentOS 7.x가 설치된 RAM 8GB 이상의 계정 이름이 `ec2-user`인 시스템 11 | ```bash 12 | [ec2-user@ip-xxx-xxx-xxx-xxx ~]$ pwd 13 | /home/ec2-user 14 | [ec2-user@ip-xxx-xxx-xxx-xxx ~]$ sudo yum -y install git 15 | [ec2-user@ip-xxx-xxx-xxx-xxx ~]$ git clone https://github.com/yoonje/elastic-stack-tutorial.git 16 | [ec2-user@ip-xxx-xxx-xxx-xxx ~]$ cd elastic-stack-tutorial 17 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ sh tuto 18 | ##################### Menu ############## 19 | $ sh tuto [Command] 20 | #####################%%%%%%############## 21 | 1 : install elk packages 22 | 2 : set elk 23 | 3 : standard input/output, no filters 24 | 4 : standard input/output, grok filter 25 | 5 : standard input/output, ruby filter 26 | ######################################### 27 | ``` 28 | 29 | ## Tutorial 1~2 - Elasticsearch, Kibana, Filebeat 설정 및 실행 30 | 31 | ### Tutorial 1~2에서 해야할 일 32 | ```bash 33 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ sh tuto 1 34 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ sh tuto 2 35 | ``` 36 | 37 | ### Tutorial 1~2에서 벌어진 일 38 | ##### Elasticsearch 39 | * packages/elasticsearch/config/elasticsearch.yml 40 | - network.host, http.cors.enabled, http.cors.allow-origin만 설정 41 | * packages/elasticsearch/config/jvm.options 42 | - Xms1g, Xmx1g를 물리 메모리의 절반으로 수정 43 | ```bash 44 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ vi packages/elasticsearch/config/elasticsearch.yml 45 | network.host: 0.0.0.0 46 | http.cors.enabled: true 47 | http.cors.allow-origin: "*" 48 | ``` 49 | ```bash 50 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ vi packages/elasticsearch/config/jvm.options 51 | -Xms4g 52 | -Xmx4g 53 | ``` 54 | ##### Kibana 55 | * packages/kibana/config/kibana.yml 56 | - server.host: "0.0.0.0" -> 외부에서 접근 가능하도록 변경 57 | - elasticsearch.url: "http://localhost:9200" -> 주석해제(연결할 elasticsearch) 58 | - kibana.index: ".kibana" -> 주석해제(키바나 기본 인덱스) 59 | ```bash 60 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ vi packages/kibana/config/kibana.yml 61 | server.host: "0.0.0.0" 62 | elasticsearch.url: "http://localhost:9200" 63 | kibana.index: ".kibana" 64 | ``` 65 | ##### Filebeat 66 | * packages/filebeat/config/filebeat.yml 67 | - /home/ec2-user/elastic-stack-tutorial/sample/ 밑에 .log 파일을 스트리밍 하도록 추가 68 | - output.elasticsearch:에 hosts: ["localhost:9200"] 추가하여 elasticsearch 등록 69 | ```bash 70 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ vi packages/filebeat/filebeat.yml 71 | filebeat.inputs: 72 | - type: log 73 | enabled: true 74 | paths: 75 | - /home/ec2-user/elastic-stack-tutorial/sample/*.log 76 | output.elasticsearch: 77 | hosts: ["localhost:9200"] 78 | ``` 79 | ##### Elastic Stack과 head 실행 80 | * ES 실행 및 systemd에 service를 등록하여 Elastic Stack 실행 81 | ```bash 82 | $ ~/elastic-stack-tutorial/packages/elasticsearch/bin/elasticsearch -d 83 | ``` 84 | ```bash 85 | $ sudo systemctl start kibana.service 86 | $ sudo systemctl start filebeat.service 87 | ``` 88 | 89 | ## Tutorial 1~2의 결과 확인 90 | * Elasticsearch 반응 확인 91 | ```bash 92 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl localhost:9200 93 | { 94 | "name" : "KSP-DCP", 95 | "cluster_name" : "elasticsearch", 96 | "cluster_uuid" : "rR30tBrtTl6LDq4nkzapxA", 97 | "version" : { 98 | "number" : "6.7.0", 99 | "build_flavor" : "default", 100 | "build_type" : "tar", 101 | "build_hash" : "8453f77", 102 | "build_date" : "2019-03-21T15:32:29.844721Z", 103 | "build_snapshot" : false, 104 | "lucene_version" : "7.7.0", 105 | "minimum_wire_compatibility_version" : "5.6.0", 106 | "minimum_index_compatibility_version" : "5.0.0" 107 | }, 108 | "tagline" : "You Know, for Search" 109 | } 110 | ``` 111 | * Elasticsearch head 연결 후 filebeat index 생성 여부 확인 112 | * Kibana 확인 113 | - http://{IP}:5601 114 | ![Optional Text](image/kibana.png) 115 | 116 | ## Tutorial 3 - Logstash 이용 117 | 118 | ### Tutorial 3에서 해야할 일 119 | `sh tuto 3` 실행 이후 정상적으로 시작되었으면 Hello Yoonje 텍스트를 입력하고 결과 확인 이후 `ctrl+c`로 sh tuto 3의 logstash 중단 120 | ```bash 121 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ sh tuto 3 122 | [2019-03-31T14:07:08,465][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} 123 | Hello Yoonje 124 | /home/ec2-user/elastic-stack-tutorial/packages/logstash-6.7.0/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated 125 | { 126 | "@timestamp" => 2019-03-31T14:27:30.761Z, 127 | "@version" => "1", 128 | "message" => "Hello Yoonje", 129 | "host" => "ip-172-31-0-154.ap-southeast-1.compute.internal" 130 | } 131 | ``` 132 | ### Tutorial 3에서 벌어진 일 133 | `packages/logstash/bin/logstash -e 'input { stdin {} } output { stdout {} }'`를 통해 logstash가 stdin을 stdout으로 출력 134 | 135 | ## Tutorial 4 - Logstash 이용 2 136 | 137 | ### Tutorial 4에서 해야할 일 138 | `sh tuto 4` 실행 이후 정상적으로 시작되었으면 Hello Yoonje 텍스트를 입력하고 결과 확인 이후 `ctrl+c`로 sh tuto 4의 logstash 중단 139 | ```bash 140 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ sh tuto 4 141 | [2019-03-31T14:07:08,465][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} 142 | Hello Yoonje 143 | /home/ec2-user/elastic-stack-tutorial/packages/logstash-6.7.0/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated 144 | { 145 | "host" => "ip-172-31-0-154.ap-southeast-1.compute.internal", 146 | "message" => "Hello Yoonje", 147 | "name" => "Yoonje", 148 | "@version" => "1", 149 | "@timestamp" => 2019-03-31T14:30:27.454Z 150 | } 151 | ``` 152 | ### Tutorial 4에서 벌어진 일 153 | `packages/logstash/bin/logstash -f logstash_conf/grok.conf`를 통해서 grok filter 활용, Hello 뒤에 나오는 이름에 name key를 매칭 154 | 155 | ## Tutorial 5 - Logstash 이용 3 156 | 157 | ### Tutorial 5에서 해야할 일 158 | `sh tuto 5` 실행 이후 정상적으로 시작되었으면 아무 텍스트나 입력하고 결과 확인 이후 `ctrl+c`로 sh tuto 5의 logstash 중단 159 | ```bash 160 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ sh tuto 5 161 | [2019-03-31T14:07:08,465][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} 162 | asdadasd 163 | /home/ec2-user/elastic-stack-tutorial/packages/logstash-6.7.0/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated 164 | { 165 | "host" => "ip-172-31-0-154.ap-southeast-1.compute.internal", 166 | "message" => "asdadasd", 167 | "today" => "2019.03.31" 168 | "@version" => "1", 169 | "@timestamp" => 2019-03-31T14:30:27.454Z 170 | } 171 | ``` 172 | ### Tutorial 5에서 벌어진 일 173 | `packages/logstash/bin/logstash -f logstash_conf/ruby.conf`를 통해서 ruby filter 활용 174 | 175 | ## Tutorial 6 - Elasticsearch 활용 176 | ### Tutorial 6에서 해야할 일 177 | ```bash 178 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -H 'Content-Type: application/json' -XPOST localhost:9200/firstindex/_doc/1 -d '{ "mykey": "myvalue" }' 179 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -XGET localhost:9200/firstindex?pretty 180 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -XGET localhost:9200/firstindex/_doc/1?pretty 181 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -H 'Content-Type: application/json' -XPUT localhost:9200/firstindex/_doc/1 -d '{ "mykey": "yourvalue" }' 182 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -XGET localhost:9200/firstindex/_doc/1 183 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -XDELETE localhost:9200/firstindex/_doc/1 184 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -XDELETE localhost:9200/firstindex 185 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -H 'Content-Type: application/x-ndjson' -XPOST localhost:9200/_bulk?pretty --data-binary @/home/ec2-user/elastic-stack-tutorial/sample/logs.jsonl 186 | [ec2-user@ip-xxx-xxx-xxx-xxx elastic-stack-tutorial]$ curl -H 'Content-Type: application/json' -XPOST localhost:9200/_bulk?pretty --data-binary @/home/ec2-user/elastic-stack-tutorial/sample/classes.json 187 | ``` 188 | 189 | ### Tutorial 6에서 벌어진 일 190 | - 단일 인덱싱 191 | - `curl -H 'Content-Type: application/json' -XPOST localhost:9200/firstindex/_doc/1 -d '{ "mykey": "myvalue" }'`를 통해서 데이터를 ES에 인덱싱 192 | - 매핑 확인 193 | - `curl -XGET localhost:9200/firstindex?pretty` 194 | - 문서 조회 195 | - `curl -XGET localhost:9200/firstindex/_doc/1?pretty` 196 | - 문서 업데이트 197 | - `curl -H 'Content-Type: application/json' -XPUT localhost:9200/firstindex/_doc/1 -d '{ "mykey": "yourvalue" }'` 198 | - 문서 삭제 199 | - `curl -XDELETE localhost:9200/firstindex/_doc/1` 200 | - 인덱스 삭제 201 | - `curl -XDELETE localhost:9200/firstindex` 202 | - 벌크 인덱싱 203 | - `curl -H 'Content-Type: application/x-ndjson' -XPOST localhost:9200/_bulk?pretty --data-binary @/home/ec2-user/elastic-stack-tutorial/sample/logs.jsonl`를 통해 데이터를 ES에 인덱싱 204 | - 벌크 인덱싱2 205 | - `curl -H 'Content-Type: application/json' -XPOST localhost:9200/_bulk?pretty --data-binary @/home/ec2-user/elastic-stack-tutorial/sample/classes.json`를 통해 데이터를 ES에 인덱싱 206 | 207 | ## Tutorial 7 - Kibana 활용 208 | ![Optional Text](image/kibana.png) 209 | Kibana Management 메뉴 선택 210 | 211 | ![Optional Text](image/kibana1.png) 212 | Kibana Management 메뉴에서 인덱스 패턴 정의 213 | 214 | ![Optional Text](image/kibana2.png) 215 | Kibana Management 메뉴에서 인덱스 패턴 timestamp으로 설정 후 생성 216 | 217 | ![Optional Text](image/kibana3.png) 218 | Kibana Management 메뉴에서 인덱스 패턴 확인 219 | 220 | ![Optional Text](image/kibana4.png) 221 | Kibana Discovery 메뉴에서 인덱스 및 데이터 확인 (time range 설정 주의) 222 | 223 | ![Optional Text](image/kibana5.png) 224 | Kibana Visualize 메뉴 선택 225 | 226 | ![Optional Text](image/kibana6.png) 227 | Kibana Visualize 메뉴에서 시각화 타입 선택(tag cloud로 설정) 228 | 229 | ![Optional Text](image/kibana7.png) 230 | Kibana Visualize 메뉴에서 시각화할 인덱스 선택 231 | 232 | ![Optional Text](image/kibana8.png) 233 | Kibana Visualize 메뉴에서 시각화 설정 정의 234 | 235 | ![Optional Text](image/kibana9.png) 236 | Kibana Visualize 메뉴에서 시각화 결과 저장 237 | 238 | ![Optional Text](image/kibana10.png) 239 | Kibana Visualize 메뉴에서 시각화 타입 선택(region map로 설정) 240 | 241 | ![Optional Text](image/kibana11.png) 242 | Kibana Visualize 메뉴에서 시각화할 인덱스 선택 243 | 244 | ![Optional Text](image/kibana12.png) 245 | Kibana Visualize 메뉴에서 시각화 설정 정의 246 | 247 | ![Optional Text](image/kibana13.png) 248 | Kibana Visualize 메뉴에서 시각화 결과 저장 249 | 250 | ![Optional Text](image/kibana14.png) 251 | Kibana Dashboard 메뉴 선택 252 | 253 | ![Optional Text](image/kibana15.png) 254 | Kibana Dashboard 메뉴에서 시각화들 결과 선택 255 | 256 | ![Optional Text](image/kibana16.png) 257 | Kibana Dashboard 메뉴에서 대시보드 확인 258 | 259 | ![Optional Text](image/kibana17.png) 260 | Kibana Dev Tools 메뉴 활용 261 | -------------------------------------------------------------------------------- /conf/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | network.host: 0.0.0.0 2 | http.cors.enabled: true 3 | http.cors.allow-origin: "*" 4 | -------------------------------------------------------------------------------- /conf/filebeat.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=filebeat 3 | Documentation=https://www.elastic.co/guide/en/beats/filebeat/current/index.html 4 | Wants=network-online.target 5 | After=network-online.target 6 | 7 | [Service] 8 | User=ec2-user 9 | Group=ec2-user 10 | ExecStart=/home/ec2-user/elastic-stack-tutorial/packages/filebeat/filebeat -c /home/ec2-user/elastic-stack-tutorial/packages/filebeat/filebeat.yml -path.home /home/ec2-user/elastic-stack-tutorial/packages/filebeat -path.config /home/ec2-user/elastic-stack-tutorial/packages/filebeat -path.data /home/ec2-user/elastic-stack-tutorial/packages/filebeat/data -path.logs /home/ec2-user/elastic-stack-tutorial/packages/filebeat/logs 11 | Restart=always 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /conf/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.inputs: 2 | - type: log 3 | enabled: true 4 | paths: 5 | - /home/ec2-user/elastic-stack-tutorial/sample/*.log 6 | output.elasticsearch: 7 | hosts: ["localhost:9200"] 8 | -------------------------------------------------------------------------------- /conf/kibana.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kibana 3 | 4 | [Service] 5 | Type=simple 6 | User=ec2-user 7 | Group=ec2-user 8 | # Load env vars from /etc/default/ and /etc/sysconfig/ if they exist. 9 | # Prefixing the path with '-' makes it try to load, but if the file doesn't 10 | # exist, it continues onward. 11 | EnvironmentFile=-/etc/default/kibana 12 | EnvironmentFile=-/etc/sysconfig/kibana 13 | ExecStart=/home/ec2-user/elastic-stack-tutorial/packages/kibana/bin/kibana "-c /home/ec2-user/elastic-stack-tutorial/packages/kibana/config/kibana.yml" 14 | Restart=always 15 | WorkingDirectory=/ 16 | 17 | [Install] 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /conf/kibana.yml: -------------------------------------------------------------------------------- 1 | # Kibana is served by a back end server. This setting specifies the port to use. 2 | #server.port: 5601 3 | 4 | # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. 5 | # The default is 'localhost', which usually means remote machines will not be able to connect. 6 | # To allow connections from remote users, set this parameter to a non-loopback address. 7 | server.host: "0.0.0.0" 8 | 9 | # Enables you to specify a path to mount Kibana at if you are running behind a proxy. 10 | # Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath 11 | # from requests it receives, and to prevent a deprecation warning at startup. 12 | # This setting cannot end in a slash. 13 | #server.basePath: "" 14 | 15 | # Specifies whether Kibana should rewrite requests that are prefixed with 16 | # `server.basePath` or require that they are rewritten by your reverse proxy. 17 | # This setting was effectively always `false` before Kibana 6.3 and will 18 | # default to `true` starting in Kibana 7.0. 19 | #server.rewriteBasePath: false 20 | 21 | # The maximum payload size in bytes for incoming server requests. 22 | #server.maxPayloadBytes: 1048576 23 | 24 | # The Kibana server's name. This is used for display purposes. 25 | #server.name: "your-hostname" 26 | 27 | # The URL of the Elasticsearch instance to use for all your queries. 28 | elasticsearch.url: "http://localhost:9200" 29 | 30 | # When this setting's value is true Kibana uses the hostname specified in the server.host 31 | # setting. When the value of this setting is false, Kibana uses the hostname of the host 32 | # that connects to this Kibana instance. 33 | #elasticsearch.preserveHost: true 34 | 35 | # Kibana uses an index in Elasticsearch to store saved searches, visualizations and 36 | # dashboards. Kibana creates a new index if the index doesn't already exist. 37 | kibana.index: ".kibana" 38 | 39 | # The default application to load. 40 | #kibana.defaultAppId: "home" 41 | 42 | # If your Elasticsearch is protected with basic authentication, these settings provide 43 | # the username and password that the Kibana server uses to perform maintenance on the Kibana 44 | # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which 45 | # is proxied through the Kibana server. 46 | #elasticsearch.username: "user" 47 | #elasticsearch.password: "pass" 48 | 49 | # Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. 50 | # These settings enable SSL for outgoing requests from the Kibana server to the browser. 51 | #server.ssl.enabled: false 52 | #server.ssl.certificate: /path/to/your/server.crt 53 | #server.ssl.key: /path/to/your/server.key 54 | 55 | # Optional settings that provide the paths to the PEM-format SSL certificate and key files. 56 | # These files validate that your Elasticsearch backend uses the same key files. 57 | #elasticsearch.ssl.certificate: /path/to/your/client.crt 58 | #elasticsearch.ssl.key: /path/to/your/client.key 59 | 60 | # Optional setting that enables you to specify a path to the PEM file for the certificate 61 | # authority for your Elasticsearch instance. 62 | #elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ] 63 | 64 | # To disregard the validity of SSL certificates, change this setting's value to 'none'. 65 | #elasticsearch.ssl.verificationMode: full 66 | 67 | # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of 68 | # the elasticsearch.requestTimeout setting. 69 | #elasticsearch.pingTimeout: 1500 70 | 71 | # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value 72 | # must be a positive integer. 73 | #elasticsearch.requestTimeout: 30000 74 | 75 | # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side 76 | # headers, set this value to [] (an empty list). 77 | #elasticsearch.requestHeadersWhitelist: [ authorization ] 78 | 79 | # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten 80 | # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. 81 | #elasticsearch.customHeaders: {} 82 | 83 | # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. 84 | #elasticsearch.shardTimeout: 30000 85 | 86 | # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. 87 | #elasticsearch.startupTimeout: 5000 88 | 89 | # Logs queries sent to Elasticsearch. Requires logging.verbose set to true. 90 | #elasticsearch.logQueries: false 91 | 92 | # Specifies the path where Kibana creates the process ID file. 93 | #pid.file: /var/run/kibana.pid 94 | 95 | # Enables you specify a file where Kibana stores log output. 96 | #logging.dest: stdout 97 | 98 | # Set the value of this setting to true to suppress all logging output. 99 | #logging.silent: false 100 | 101 | # Set the value of this setting to true to suppress all logging output other than error messages. 102 | #logging.quiet: false 103 | 104 | # Set the value of this setting to true to log all events, including system usage information 105 | # and all requests. 106 | #logging.verbose: false 107 | 108 | # Set the interval in milliseconds to sample system and process performance 109 | # metrics. Minimum is 100ms. Defaults to 5000. 110 | #ops.interval: 5000 111 | 112 | # Specifies locale to be used for all localizable strings, dates and number formats. 113 | #i18n.locale: "en" 114 | -------------------------------------------------------------------------------- /conf/logstash.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Logstash 3 | Documentation=https://www.elastic.co/products/logstash 4 | After=network.target 5 | #ConditionPathExists=/etc/logstash.conf 6 | 7 | [Service] 8 | User=ec2-user 9 | Group=ec2-user 10 | ExecStart=/home/ec2-user/elastic-stack-tutorial/packages/logstash/bin/logstash 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /conf/logstash.yml: -------------------------------------------------------------------------------- 1 | # Settings file in YAML 2 | # 3 | # Settings can be specified either in hierarchical form, e.g.: 4 | # 5 | # pipeline: 6 | # batch: 7 | # size: 125 8 | # delay: 5 9 | # 10 | # Or as flat keys: 11 | # 12 | # pipeline.batch.size: 125 13 | # pipeline.batch.delay: 5 14 | # 15 | # ------------ Node identity ------------ 16 | # 17 | # Use a descriptive name for the node: 18 | # 19 | # node.name: test 20 | # 21 | # If omitted the node name will default to the machine's host name 22 | # 23 | # ------------ Data path ------------------ 24 | # 25 | # Which directory should be used by logstash and its plugins 26 | # for any persistent needs. Defaults to LOGSTASH_HOME/data 27 | # 28 | # path.data: 29 | # 30 | # ------------ Pipeline Settings -------------- 31 | # 32 | # The ID of the pipeline. 33 | # 34 | # pipeline.id: main 35 | # 36 | # Set the number of workers that will, in parallel, execute the filters+outputs 37 | # stage of the pipeline. 38 | # 39 | # This defaults to the number of the host's CPU cores. 40 | # 41 | # pipeline.workers: 2 42 | # 43 | # How many events to retrieve from inputs before sending to filters+workers 44 | # 45 | # pipeline.batch.size: 125 46 | # 47 | # How long to wait in milliseconds while polling for the next event 48 | # before dispatching an undersized batch to filters+outputs 49 | # 50 | # pipeline.batch.delay: 50 51 | # 52 | # Force Logstash to exit during shutdown even if there are still inflight 53 | # events in memory. By default, logstash will refuse to quit until all 54 | # received events have been pushed to the outputs. 55 | # 56 | # WARNING: enabling this can lead to data loss during shutdown 57 | # 58 | # pipeline.unsafe_shutdown: false 59 | # 60 | # ------------ Pipeline Configuration Settings -------------- 61 | # 62 | # Where to fetch the pipeline configuration for the main pipeline 63 | # 64 | # path.config: 65 | # 66 | # Pipeline configuration string for the main pipeline 67 | # 68 | # config.string: 69 | # 70 | # At startup, test if the configuration is valid and exit (dry run) 71 | # 72 | # config.test_and_exit: false 73 | # 74 | # Periodically check if the configuration has changed and reload the pipeline 75 | # This can also be triggered manually through the SIGHUP signal 76 | # 77 | config.reload.automatic: true 78 | # 79 | # How often to check if the pipeline configuration has changed (in seconds) 80 | # 81 | config.reload.interval: 3s 82 | # 83 | # Show fully compiled configuration as debug log message 84 | # NOTE: --log.level must be 'debug' 85 | # 86 | # config.debug: false 87 | # 88 | # When enabled, process escaped characters such as \n and \" in strings in the 89 | # pipeline configuration files. 90 | # 91 | # config.support_escapes: false 92 | # 93 | # ------------ Module Settings --------------- 94 | # Define modules here. Modules definitions must be defined as an array. 95 | # The simple way to see this is to prepend each `name` with a `-`, and keep 96 | # all associated variables under the `name` they are associated with, and 97 | # above the next, like this: 98 | # 99 | # modules: 100 | # - name: MODULE_NAME 101 | # var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE 102 | # var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE 103 | # var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE 104 | # var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE 105 | # 106 | # Module variable names must be in the format of 107 | # 108 | # var.PLUGIN_TYPE.PLUGIN_NAME.KEY 109 | # 110 | # modules: 111 | # 112 | # ------------ Cloud Settings --------------- 113 | # Define Elastic Cloud settings here. 114 | # Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy 115 | # and it may have an label prefix e.g. staging:dXMtZ... 116 | # This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' 117 | # cloud.id: 118 | # 119 | # Format of cloud.auth is: : 120 | # This is optional 121 | # If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' 122 | # If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' 123 | # cloud.auth: elastic: 124 | # 125 | # ------------ Queuing Settings -------------- 126 | # 127 | # Internal queuing model, "memory" for legacy in-memory based queuing and 128 | # "persisted" for disk-based acked queueing. Defaults is memory 129 | # 130 | # queue.type: memory 131 | # 132 | # If using queue.type: persisted, the directory path where the data files will be stored. 133 | # Default is path.data/queue 134 | # 135 | # path.queue: 136 | # 137 | # If using queue.type: persisted, the page data files size. The queue data consists of 138 | # append-only data files separated into pages. Default is 64mb 139 | # 140 | # queue.page_capacity: 64mb 141 | # 142 | # If using queue.type: persisted, the maximum number of unread events in the queue. 143 | # Default is 0 (unlimited) 144 | # 145 | # queue.max_events: 0 146 | # 147 | # If using queue.type: persisted, the total capacity of the queue in number of bytes. 148 | # If you would like more unacked events to be buffered in Logstash, you can increase the 149 | # capacity using this setting. Please make sure your disk drive has capacity greater than 150 | # the size specified here. If both max_bytes and max_events are specified, Logstash will pick 151 | # whichever criteria is reached first 152 | # Default is 1024mb or 1gb 153 | # 154 | # queue.max_bytes: 1024mb 155 | # 156 | # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint 157 | # Default is 1024, 0 for unlimited 158 | # 159 | # queue.checkpoint.acks: 1024 160 | # 161 | # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint 162 | # Default is 1024, 0 for unlimited 163 | # 164 | # queue.checkpoint.writes: 1024 165 | # 166 | # If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page 167 | # Default is 1000, 0 for no periodic checkpoint. 168 | # 169 | # queue.checkpoint.interval: 1000 170 | # 171 | # ------------ Dead-Letter Queue Settings -------------- 172 | # Flag to turn on dead-letter queue. 173 | # 174 | # dead_letter_queue.enable: false 175 | 176 | # If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries 177 | # will be dropped if they would increase the size of the dead letter queue beyond this setting. 178 | # Default is 1024mb 179 | # dead_letter_queue.max_bytes: 1024mb 180 | 181 | # If using dead_letter_queue.enable: true, the directory path where the data files will be stored. 182 | # Default is path.data/dead_letter_queue 183 | # 184 | # path.dead_letter_queue: 185 | # 186 | # ------------ Metrics Settings -------------- 187 | # 188 | # Bind address for the metrics REST endpoint 189 | # 190 | http.host: "0.0.0.0" 191 | # 192 | # Bind port for the metrics REST endpoint, this option also accept a range 193 | # (9600-9700) and logstash will pick up the first available ports. 194 | # 195 | # http.port: 9600-9700 196 | # 197 | # ------------ Debugging Settings -------------- 198 | # 199 | # Options for log.level: 200 | # * fatal 201 | # * error 202 | # * warn 203 | # * info (default) 204 | # * debug 205 | # * trace 206 | # 207 | # log.level: info 208 | # path.logs: 209 | # 210 | # ------------ Other Settings -------------- 211 | # 212 | # Where to find custom plugins 213 | # path.plugins: [] 214 | # 215 | # ------------ X-Pack Settings (not applicable for OSS build)-------------- 216 | # 217 | # X-Pack Monitoring 218 | # https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html 219 | #xpack.monitoring.enabled: false 220 | #xpack.monitoring.elasticsearch.username: logstash_system 221 | #xpack.monitoring.elasticsearch.password: password 222 | #xpack.monitoring.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] 223 | #xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] 224 | #xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file 225 | #xpack.monitoring.elasticsearch.ssl.truststore.password: password 226 | #xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file 227 | #xpack.monitoring.elasticsearch.ssl.keystore.password: password 228 | #xpack.monitoring.elasticsearch.ssl.verification_mode: certificate 229 | #xpack.monitoring.elasticsearch.sniffing: false 230 | #xpack.monitoring.collection.interval: 10s 231 | #xpack.monitoring.collection.pipeline.details.enabled: true 232 | # 233 | # X-Pack Management 234 | # https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html 235 | #xpack.management.enabled: false 236 | #xpack.management.pipeline.id: ["main", "apache_logs"] 237 | #xpack.management.elasticsearch.username: logstash_admin_user 238 | #xpack.management.elasticsearch.password: password 239 | #xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] 240 | #xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] 241 | #xpack.management.elasticsearch.ssl.truststore.path: /path/to/file 242 | #xpack.management.elasticsearch.ssl.truststore.password: password 243 | #xpack.management.elasticsearch.ssl.keystore.path: /path/to/file 244 | #xpack.management.elasticsearch.ssl.keystore.password: password 245 | #xpack.management.elasticsearch.ssl.verification_mode: certificate 246 | #xpack.management.elasticsearch.sniffing: false 247 | #xpack.management.logstash.poll_interval: 5s 248 | -------------------------------------------------------------------------------- /image/kibana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana.png -------------------------------------------------------------------------------- /image/kibana1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana1.png -------------------------------------------------------------------------------- /image/kibana10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana10.png -------------------------------------------------------------------------------- /image/kibana11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana11.png -------------------------------------------------------------------------------- /image/kibana12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana12.png -------------------------------------------------------------------------------- /image/kibana13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana13.png -------------------------------------------------------------------------------- /image/kibana14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana14.png -------------------------------------------------------------------------------- /image/kibana15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana15.png -------------------------------------------------------------------------------- /image/kibana16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana16.png -------------------------------------------------------------------------------- /image/kibana17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana17.png -------------------------------------------------------------------------------- /image/kibana2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana2.png -------------------------------------------------------------------------------- /image/kibana3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana3.png -------------------------------------------------------------------------------- /image/kibana4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana4.png -------------------------------------------------------------------------------- /image/kibana5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana5.png -------------------------------------------------------------------------------- /image/kibana6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana6.png -------------------------------------------------------------------------------- /image/kibana7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana7.png -------------------------------------------------------------------------------- /image/kibana8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana8.png -------------------------------------------------------------------------------- /image/kibana9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoonje/elastic-stack-tutorial/ca5acc73ac622adc798c14cbff4a03ee3bef1e5c/image/kibana9.png -------------------------------------------------------------------------------- /logstash_conf/grok.conf: -------------------------------------------------------------------------------- 1 | input { 2 | stdin {} 3 | } 4 | 5 | filter { 6 | grok { 7 | match => { "message" => "Hello %{WORD:name}" } 8 | } 9 | } 10 | 11 | output { 12 | stdout {} 13 | } -------------------------------------------------------------------------------- /logstash_conf/ruby.conf: -------------------------------------------------------------------------------- 1 | input { 2 | stdin {} 3 | } 4 | 5 | filter { 6 | ruby { 7 | code => "event.set('today', event.timestamp.time.localtime.strftime('%Y.%m.%d'))" 8 | } 9 | } 10 | 11 | output { 12 | stdout {} 13 | } -------------------------------------------------------------------------------- /sample/classes.json: -------------------------------------------------------------------------------- 1 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "1" } } 2 | {"title" : "Machine Learning","Professor" : "Minsuk Heo","major" : "Computer Science","semester" : ["spring", "fall"],"student_count" : 100,"unit" : 3,"rating" : 5, "submit_date" : "2016-01-02", "school_location" : {"lat" : 36.00, "lon" : -120.00}} 3 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "2" } } 4 | {"title" : "Network","Professor" : "Minsuk Heo","major" : "Computer Science","semester" : ["fall"],"student_count" : 50,"unit" : 3,"rating" : 4, "submit_date" : "2016-02-02", "school_location" : {"lat" : 36.00, "lon" : -120.00}} 5 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "3" } } 6 | {"title" : "Operating System","Professor" : "Minsuk Heo","major" : "Computer Science","semester" : ["spring"],"student_count" : 50,"unit" : 3,"rating" : 4, "submit_date" : "2016-03-02", "school_location" : {"lat" : 36.00, "lon" : -120.00}} 7 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "5" } } 8 | {"title" : "Machine Learning","Professor" : "Tim Cook","major" : "Computer Science","semester" : ["spring"],"student_count" : 40,"unit" : 3,"rating" : 2, "submit_date" : "2016-04-02", "school_location" : {"lat" : 39.00, "lon" : -112.00}} 9 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "6" } } 10 | {"title" : "Network","Professor" : "Tim Cook","major" : "Computer Science","semester" : ["summer"],"student_count" : 30,"unit" : 3,"rating" : 2, "submit_date" : "2016-02-02", "school_location" : {"lat" : 36.00, "lon" : -120.00}} 11 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "7" } } 12 | {"title" : "Operating System","Professor" : "Jeniffer Anderson","major" : "Computer Science","semester" : ["winter"],"student_count" : 30,"unit" : 3,"rating" : 1, "submit_date" : "2016-11-02", "school_location" : {"lat" : 39.97, "lon" : -89.78}} 13 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "8" } } 14 | {"title" : "Algorithm","Professor" : "Tim Cook","major" : "Computer Science","semester" : ["fall"],"student_count" : 80,"unit" : 3,"rating" : 2, "submit_date" : "2016-10-22", "school_location" : {"lat" : 39.97, "lon" : -89.78}} 15 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "9" } } 16 | {"title" : "Data Structure","Professor" : "Tim Cook","major" : "Computer Science","semester" : ["winter"],"student_count" : 50,"unit" : 3,"rating" : 2, "submit_date" : "2016-07-22", "school_location" : {"lat" : 39.97, "lon" : -89.78}} 17 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "10" } } 18 | {"title" : "Computer Graphic","Professor" : "Jeniffer Anderson","major" : "Computer Science","semester" : ["spring"],"student_count" : 60,"unit" : 2,"rating" : 3, "submit_date" : "2016-11-12", "school_location" : {"lat" : 39.97, "lon" : -89.78}} 19 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "11" } } 20 | {"title" : "Music Fundamental","Professor" : "Jay Z","major" : "Music","semester" : ["fall"],"student_count" : 100,"unit" : 3,"rating" : 5, "submit_date" : "2016-05-22", "school_location" : {"lat" : 42.51, "lon" : -74.83}} 21 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "12" } } 22 | {"title" : "Vocal Techniques","Professor" : "Beyonce","major" : "Music","semester" : ["fall"],"student_count" : 30,"unit" : 3,"rating" : 5, "submit_date" : "2016-11-22", "school_location" : {"lat" : 42.51, "lon" : -74.83}} 23 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "13" } } 24 | {"title" : "Guitar Techiniques","Professor" : "Eric Clapton","major" : "Music","semester" : ["spring", "fall"],"student_count" : 20,"unit" : 2,"rating" : 4, "submit_date" : "2016-03-12", "school_location" : {"lat" : 42.51, "lon" : -74.83}} 25 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "14" } } 26 | {"title" : "Finance","Professor" : "Bill Gates","major" : "Accounting","semester" : ["winter"],"student_count" : 50,"unit" : 3,"rating" : 2, "submit_date" : "2016-01-12", "school_location" : {"lat" : 42.51, "lon" : -74.83}} 27 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "15" } } 28 | {"title" : "Marketing","Professor" : "Bill Gates","major" : "Accounting","semester" : ["spring"],"student_count" : 60,"unit" : 2,"rating" : 3, "submit_date" : "2016-01-22", "school_location" : {"lat" : 42.51, "lon" : -74.83}} 29 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "16" } } 30 | {"title" : "Accounting Information Systems","Professor" : "Tom Cruise","major" : "Accounting","semester" : ["fall"],"student_count" : 100,"unit" : 2,"rating" : 1, "submit_date" : "2016-11-12", "school_location" : {"lat" : 42.51, "lon" : -74.83}} 31 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "17" } } 32 | {"title" : "Individual Taxation","Professor" : "Tom Cruise","major" : "Accounting","semester" : ["fall"],"student_count" : 30,"unit" : 1,"rating" : 2, "submit_date" : "2016-08-02", "school_location" : {"lat" : 42.32, "lon" : -94.74}} 33 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "18" } } 34 | {"title" : "Auditing","Professor" : "Victoria Park","major" : "Accounting","semester" : ["spring", "fall"],"student_count" : 20,"unit" : 2,"rating" : 3, "submit_date" : "2016-09-13", "school_location" : {"lat" : 42.32, "lon" : -94.74}} 35 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "19" } } 36 | {"title" : "Cell Biology","Professor" : "Anjella Kim","major" : "Medical","semester" : ["fall"],"student_count" : 40,"unit" : 5,"rating" : 5, "submit_date" : "2016-02-22", "school_location" : {"lat" : 42.32, "lon" : -94.74}} 37 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "20" } } 38 | {"title" : "Physiology","Professor" : "Jack Berk","major" : "Medical","semester" : ["summer"],"student_count" : 30,"unit" : 5,"rating" : 4, "submit_date" : "2016-11-12", "school_location" : {"lat" : 32.69, "lon" : -99.44}} 39 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "21" } } 40 | {"title" : "Neuroscience","Professor" : "Jihee Yang","major" : "Medical","semester" : ["spring", "fall"],"student_count" : 20,"unit" : 5,"rating" : 4, "submit_date" : "2016-06-03", "school_location" : {"lat" : 32.69, "lon" : -99.44}} 41 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "22" } } 42 | {"title" : "Immunology","Professor" : "Meredith Lee","major" : "Medical","semester" : ["winter"],"student_count" : 30,"unit" : 3,"rating" : 2, "submit_date" : "2016-06-21", "school_location" : {"lat" : 32.69, "lon" : -99.44}} 43 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "23" } } 44 | {"title" : "Genetics","Professor" : "David Pollack","major" : "Medical","semester" : ["spring"],"student_count" : 20,"unit" : 3,"rating" : 3, "submit_date" : "2016-06-30", "school_location" : {"lat" : 28.22, "lon" : -81.87}} 45 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "24" } } 46 | {"title" : "Biochemistry","Professor" : "John Miller","major" : "Medical","semester" : ["fall"],"student_count" : 30,"unit" : 3,"rating" : 4, "submit_date" : "2016-01-11", "school_location" : {"lat" : 28.22, "lon" : -81.87}} 47 | { "index" : { "_index" : "classes", "_type" : "class", "_id" : "25" } } 48 | {"title" : "Anatomy","Professor" : "Tom Johnson","major" : "Medical","semester" : ["fall"],"student_count" : 30,"unit" : 5,"rating" : 3, "submit_date" : "2016-11-12", "school_location" : {"lat" : 28.22, "lon" : -81.87}} 49 | -------------------------------------------------------------------------------- /sample/filebeat-tutorial.log: -------------------------------------------------------------------------------- 1 | 83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 2 | 83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1" 200 171717 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 3 | 83.149.9.216 - - [04/Jan/2015:05:13:44 +0000] "GET /presentations/logstash-monitorama-2013/plugin/highlight/highlight.js HTTP/1.1" 200 26185 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 4 | 83.149.9.216 - - [04/Jan/2015:05:13:44 +0000] "GET /presentations/logstash-monitorama-2013/plugin/zoom-js/zoom.js HTTP/1.1" 200 7697 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 5 | 83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/plugin/notes/notes.js HTTP/1.1" 200 2892 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 6 | 83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/sad-medic.png HTTP/1.1" 200 430406 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 7 | 83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/css/fonts/Roboto-Bold.ttf HTTP/1.1" 200 38720 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 8 | 83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/css/fonts/Roboto-Regular.ttf HTTP/1.1" 200 41820 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 9 | 83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 200 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 10 | 83.149.9.216 - - [04/Jan/2015:05:13:43 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard.png HTTP/1.1" 200 321631 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 11 | 83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] "GET /presentations/logstash-monitorama-2013/images/Dreamhost_logo.svg HTTP/1.1" 200 2126 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 12 | 83.149.9.216 - - [04/Jan/2015:05:13:43 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard2.png HTTP/1.1" 200 394967 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 13 | 83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] "GET /presentations/logstash-monitorama-2013/images/apache-icon.gif HTTP/1.1" 200 8095 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 14 | 83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] "GET /presentations/logstash-monitorama-2013/images/nagios-sms5.png HTTP/1.1" 200 78075 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 15 | 83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] "GET /presentations/logstash-monitorama-2013/images/redis.png HTTP/1.1" 200 25230 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 16 | 83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] "GET /presentations/logstash-monitorama-2013/images/elasticsearch.png HTTP/1.1" 200 8026 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 17 | 83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] "GET /presentations/logstash-monitorama-2013/images/logstashbook.png HTTP/1.1" 200 54662 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 18 | 83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] "GET /presentations/logstash-monitorama-2013/images/github-contributions.png HTTP/1.1" 200 34245 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 19 | 83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] "GET /presentations/logstash-monitorama-2013/css/print/paper.css HTTP/1.1" 200 4254 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 20 | 83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] "GET /presentations/logstash-monitorama-2013/images/1983_delorean_dmc-12-pic-38289.jpeg HTTP/1.1" 200 220562 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 21 | 83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] "GET /presentations/logstash-monitorama-2013/images/simple-inputs-filters-outputs.jpg HTTP/1.1" 200 1168622 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 22 | 83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] "GET /presentations/logstash-monitorama-2013/images/tiered-outputs-to-inputs.jpg HTTP/1.1" 200 1079983 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 23 | 83.149.9.216 - - [04/Jan/2015:05:13:53 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 24 | 24.236.252.67 - - [04/Jan/2015:05:14:10 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" 25 | 93.114.45.13 - - [04/Jan/2015:05:14:32 +0000] "GET /articles/dynamic-dns-with-dhcp/ HTTP/1.1" 200 18848 "http://www.google.ro/url?sa=t&rct=j&q=&esrc=s&source=web&cd=2&ved=0CCwQFjAB&url=http%3A%2F%2Fwww.semicomplete.com%2Farticles%2Fdynamic-dns-with-dhcp%2F&ei=W88AU4n9HOq60QXbv4GwBg&usg=AFQjCNEF1X4Rs52UYQyLiySTQxa97ozM4g&bvm=bv.61535280,d.d2k" "Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0" 26 | 93.114.45.13 - - [04/Jan/2015:05:14:32 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/articles/dynamic-dns-with-dhcp/" "Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0" 27 | 93.114.45.13 - - [04/Jan/2015:05:14:33 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/articles/dynamic-dns-with-dhcp/" "Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0" 28 | 93.114.45.13 - - [04/Jan/2015:05:14:33 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0" 29 | 93.114.45.13 - - [04/Jan/2015:05:14:33 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://www.semicomplete.com/articles/dynamic-dns-with-dhcp/" "Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0" 30 | 93.114.45.13 - - [04/Jan/2015:05:14:33 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://www.semicomplete.com/style2.css" "Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0" 31 | 66.249.73.135 - - [04/Jan/2015:05:15:03 +0000] "GET /blog/tags/ipv6 HTTP/1.1" 200 12251 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 32 | 50.16.19.13 - - [04/Jan/2015:05:15:15 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "http://www.semicomplete.com/blog/tags/puppet?flav=rss20" "Tiny Tiny RSS/1.11 (http://tt-rss.org/)" 33 | 66.249.73.185 - - [04/Jan/2015:05:15:23 +0000] "GET / HTTP/1.1" 200 37932 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 34 | 110.136.166.128 - - [04/Jan/2015:05:16:11 +0000] "GET /projects/xdotool/ HTTP/1.1" 200 12292 "http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=5&cad=rja&sqi=2&ved=0CFYQFjAE&url=http%3A%2F%2Fwww.semicomplete.com%2Fprojects%2Fxdotool%2F&ei=6cwAU_bRHo6urAeI0YD4Ag&usg=AFQjCNE3V_aCf3-gfNcbS924S6jZ6FqffA&bvm=bv.61535280,d.bmk" "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0" 35 | 46.105.14.53 - - [04/Jan/2015:05:16:17 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/" 36 | 110.136.166.128 - - [04/Jan/2015:05:16:22 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0" 37 | 110.136.166.128 - - [04/Jan/2015:05:16:22 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0" 38 | 110.136.166.128 - - [04/Jan/2015:05:16:22 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0" 39 | 110.136.166.128 - - [04/Jan/2015:05:16:22 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0" 40 | 123.125.71.35 - - [04/Jan/2015:05:16:31 +0000] "GET /blog/tags/release HTTP/1.1" 200 40693 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 41 | 110.136.166.128 - - [04/Jan/2015:05:16:22 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://www.semicomplete.com/style2.css" "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0" 42 | 50.150.204.184 - - [04/Jan/2015:05:17:06 +0000] "GET /images/googledotcom.png HTTP/1.1" 200 65748 "http://www.google.com/search?q=https//:google.com&source=lnms&tbm=isch&sa=X&ei=4-r8UvDrKZOgkQe7x4CICw&ved=0CAkQ_AUoAA&biw=320&bih=441" "Mozilla/5.0 (Linux; U; Android 4.0.4; en-us; LG-MS770 Build/IMM76I) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 43 | 207.241.237.225 - - [04/Jan/2015:05:17:35 +0000] "GET /blog/tags/examples HTTP/1.0" 200 9208 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" 44 | 200.49.190.101 - - [04/Jan/2015:05:17:39 +0000] "GET /reset.css HTTP/1.1" 200 1015 "-" "-" 45 | 200.49.190.100 - - [04/Jan/2015:05:17:37 +0000] "GET /blog/tags/web HTTP/1.1" 200 44019 "-" "QS304 Profile/MIDP-2.0 Configuration/CLDC-1.1" 46 | 200.49.190.101 - - [04/Jan/2015:05:17:41 +0000] "GET /style2.css HTTP/1.1" 200 4877 "-" "-" 47 | 200.49.190.101 - - [04/Jan/2015:05:17:48 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "-" "QS304 Profile/MIDP-2.0 Configuration/CLDC-1.1" 48 | 66.249.73.185 - - [04/Jan/2015:05:18:48 +0000] "GET /reset.css HTTP/1.1" 200 1015 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 49 | 66.249.73.135 - - [04/Jan/2015:05:18:55 +0000] "GET /blog/tags/munin HTTP/1.1" 200 9746 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 50 | 66.249.73.135 - - [04/Jan/2015:05:19:16 +0000] "GET /blog/tags/firefox?flav=rss20 HTTP/1.1" 200 16021 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 51 | 66.249.73.135 - - [04/Jan/2015:05:20:03 +0000] "GET /blog/geekery/eventdb-ideas.html HTTP/1.1" 200 11418 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 52 | 67.214.178.190 - - [04/Jan/2015:05:20:48 +0000] "GET / HTTP/1.0" 200 37932 "http://www.semicomplete.com/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:21.0) Gecko/20100101 Firefox/21.0" 53 | 67.214.178.190 - - [04/Jan/2015:05:20:59 +0000] "GET /blog/geekery/installing-windows-8-consumer-preview.html HTTP/1.0" 200 8948 "http://www.semicomplete.com/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:21.0) Gecko/20100101 Firefox/21.0" 54 | 207.241.237.220 - - [04/Jan/2015:05:21:16 +0000] "GET /blog/tags/projects HTTP/1.0" 200 28370 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" 55 | 46.105.14.53 - - [04/Jan/2015:05:21:17 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/" 56 | 207.241.237.227 - - [04/Jan/2015:05:21:19 +0000] "GET /blog/geekery/soekris-gpio.html HTTP/1.0" 200 9587 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" 57 | 91.177.205.119 - - [04/Jan/2015:05:22:03 +0000] "GET /blog/geekery/xvfb-firefox.html HTTP/1.1" 200 10975 "http://en.wikipedia.org/wiki/Xvfb" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" 58 | 91.177.205.119 - - [04/Jan/2015:05:22:03 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" 59 | 91.177.205.119 - - [04/Jan/2015:05:22:03 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" 60 | 91.177.205.119 - - [04/Jan/2015:05:22:03 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" 61 | 91.177.205.119 - - [04/Jan/2015:05:22:03 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" 62 | 91.177.205.119 - - [04/Jan/2015:05:22:04 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Win64; x64; Trident/6.0)" 63 | 66.249.73.185 - - [04/Jan/2015:05:22:13 +0000] "GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1" 404 294 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 64 | 207.241.237.228 - - [04/Jan/2015:05:22:23 +0000] "GET /blog/tags/defcon HTTP/1.0" 200 24142 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" 65 | 207.241.237.101 - - [04/Jan/2015:05:22:27 +0000] "GET /blog/tags/regex HTTP/1.0" 200 14888 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" 66 | 87.169.99.232 - - [04/Jan/2015:05:23:41 +0000] "GET /presentations/puppet-at-loggly/puppet-at-loggly.pdf.html HTTP/1.1" 200 24747 "https://www.google.de/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 67 | 209.85.238.199 - - [04/Jan/2015:05:23:47 +0000] "GET /blog/tags/firefox?flav=rss20 HTTP/1.1" 200 16021 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 3 subscribers; feed-id=14171215010336145331)" 68 | 209.85.238.199 - - [04/Jan/2015:05:24:00 +0000] "GET /test.xml HTTP/1.1" 200 1370 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=11390274670024826467)" 69 | 81.220.24.207 - - [04/Jan/2015:05:24:57 +0000] "GET /blog/geekery/ssl-latency.html HTTP/1.1" 200 17147 "http://www.google.fr/url?sa=t&rct=j&q=&esrc=s&source=web&cd=5&ved=0CE4QFjAE&url=http%3A%2F%2Fwww.semicomplete.com%2Fblog%2Fgeekery%2Fssl-latency.html&ei=ZdEAU9mGGuWX1AW09IDoBw&usg=AFQjCNHw6zioJpizqX8Q0YpKKaF4zdCSEg&bvm=bv.61535280,d.d2k" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" 70 | 81.220.24.207 - - [04/Jan/2015:05:24:57 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" 71 | 81.220.24.207 - - [04/Jan/2015:05:24:57 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" 72 | 81.220.24.207 - - [04/Jan/2015:05:24:57 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" 73 | 81.220.24.207 - - [04/Jan/2015:05:24:57 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" 74 | 81.220.24.207 - - [04/Jan/2015:05:24:58 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" 75 | 66.249.73.135 - - [04/Jan/2015:05:25:05 +0000] "GET /blog/geekery/vmware-cpu-performance.html HTTP/1.1" 200 12908 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 76 | 46.105.14.53 - - [04/Jan/2015:05:26:17 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/" 77 | 218.30.103.62 - - [04/Jan/2015:05:27:05 +0000] "GET /robots.txt HTTP/1.1" 200 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 78 | 218.30.103.62 - - [04/Jan/2015:05:27:10 +0000] "GET /robots.txt HTTP/1.1" 200 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 79 | 218.30.103.62 - - [04/Jan/2015:05:27:15 +0000] "GET /projects/fex/ HTTP/1.1" 200 14352 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 80 | 74.125.40.20 - - [04/Jan/2015:05:27:22 +0000] "GET /?flav=rss20 HTTP/1.1" 200 29941 "-" "FeedBurner/1.0 (http://www.FeedBurner.com)" 81 | 71.212.224.97 - - [04/Jan/2015:05:27:34 +0000] "GET /projects/xdotool/ HTTP/1.1" 200 12292 "http://suckless.org/rocks" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 82 | 71.212.224.97 - - [04/Jan/2015:05:27:34 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 83 | 71.212.224.97 - - [04/Jan/2015:05:27:35 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 84 | 71.212.224.97 - - [04/Jan/2015:05:27:35 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 85 | 71.212.224.97 - - [04/Jan/2015:05:27:35 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 86 | 218.30.103.62 - - [04/Jan/2015:05:27:36 +0000] "GET /projects/xdotool/xdotool.xhtml HTTP/1.1" 304 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 87 | 108.174.55.234 - - [04/Jan/2015:05:27:45 +0000] "GET /?flav=rss20 HTTP/1.1" 200 29941 "-" "-" 88 | 218.30.103.62 - - [04/Jan/2015:05:27:57 +0000] "GET /blog/geekery/c-vs-python-bdb.html HTTP/1.1" 200 11388 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 89 | 121.107.188.202 - - [04/Jan/2015:05:27:57 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1" 200 171717 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" 90 | 218.30.103.62 - - [04/Jan/2015:05:28:21 +0000] "GET /blog/productivity/better-zsh-xterm-title-fix.html HTTP/1.1" 200 10185 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 91 | 218.30.103.62 - - [04/Jan/2015:05:28:43 +0000] "GET /blog/geekery/xvfb-firefox.html HTTP/1.1" 200 10975 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 92 | 218.30.103.62 - - [04/Jan/2015:05:29:06 +0000] "GET /blog/geekery/puppet-facts-into-mcollective.html HTTP/1.1" 200 9872 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 93 | 198.46.149.143 - - [04/Jan/2015:05:29:13 +0000] "GET /blog/geekery/disabling-battery-in-ubuntu-vms.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29 HTTP/1.1" 200 9316 "-" "Tiny Tiny RSS/1.11 (http://tt-rss.org/)" 94 | 198.46.149.143 - - [04/Jan/2015:05:29:13 +0000] "GET /blog/geekery/solving-good-or-bad-problems.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+semicomplete%2Fmain+%28semicomplete.com+-+Jordan+Sissel%29 HTTP/1.1" 200 10756 "-" "Tiny Tiny RSS/1.11 (http://tt-rss.org/)" 95 | 218.30.103.62 - - [04/Jan/2015:05:29:26 +0000] "GET /blog/geekery/jquery-interface-puffer.html%20target= HTTP/1.1" 200 202 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 96 | 218.30.103.62 - - [04/Jan/2015:05:29:48 +0000] "GET /blog/geekery/ec2-reserved-vs-ondemand.html HTTP/1.1" 200 11834 "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 97 | 66.249.73.135 - - [04/Jan/2015:05:30:06 +0000] "GET /blog/web/firefox-scrolling-fix.html HTTP/1.1" 200 8956 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 98 | 86.1.76.62 - - [04/Jan/2015:05:30:37 +0000] "GET /projects/xdotool/ HTTP/1.1" 200 12292 "http://www.haskell.org/haskellwiki/Xmonad/Frequently_asked_questions" "Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20140205 Firefox/24.0 Iceweasel/24.3.0" 99 | 86.1.76.62 - - [04/Jan/2015:05:30:37 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20140205 Firefox/24.0 Iceweasel/24.3.0" 100 | 86.1.76.62 - - [04/Jan/2015:05:30:37 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20140205 Firefox/24.0 Iceweasel/24.3.0" 101 | -------------------------------------------------------------------------------- /tuto: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ES_URL="https://artifacts.elastic.co/downloads/elasticsearch" 4 | LS_URL="https://artifacts.elastic.co/downloads/logstash" 5 | KB_URL="https://artifacts.elastic.co/downloads/kibana" 6 | FB_URL="https://artifacts.elastic.co/downloads/beats/filebeat" 7 | 8 | VER="6.7.0" 9 | 10 | ES="elasticsearch-${VER}.tar.gz" 11 | LS="logstash-${VER}.tar.gz" 12 | KB="kibana-${VER}-linux-x86_64.tar.gz" 13 | FB="filebeat-${VER}-linux-x86_64.tar.gz" 14 | 15 | ES_PKGS=${ES_URL}/${ES} 16 | LS_PKGS=${LS_URL}/${LS} 17 | KB_PKGS=${KB_URL}/${KB} 18 | FB_PKGS=${FB_URL}/${FB} 19 | 20 | ES_DIR="elasticsearch" 21 | LS_DIR="logstash" 22 | KB_DIR="kibana" 23 | FB_DIR="filebeat" 24 | PKGS_DIR="packages" 25 | CONF_DIR="conf" 26 | SYS_DIR="/etc/systemd/system" 27 | 28 | ES_CONF="elasticsearch.yml" 29 | KB_CONF="kibana.yml" 30 | FB_CONF="filebeat.yml" 31 | 32 | LS_SVC="logstash.service" 33 | KB_SVC="kibana.service" 34 | FB_SVC="filebeat.service" 35 | 36 | USER=$(whoami) 37 | PWD=$(pwd) 38 | 39 | if [ $USER == "root" ]; then 40 | echo "root is not permitted" 41 | exit -1 42 | fi 43 | 44 | if [ $PWD != "/home/ec2-user/elastic-stack-tutorial" ]; then 45 | pwd 46 | echo "/home/ec2-user/elastic-stack-tutorial path is permitted only" 47 | exit -1 48 | fi 49 | 50 | function install_elk_packages 51 | { 52 | 53 | git pull 54 | 55 | sudo yum -y install wget java 56 | sudo yum -y install bzip2 epel-release 57 | sudo yum -y install npm 58 | 59 | mkdir ${PKGS_DIR} 60 | cd ${PKGS_DIR} 61 | 62 | wget ${ES_PKGS} 63 | wget ${LS_PKGS} 64 | wget ${KB_PKGS} 65 | wget ${FB_PKGS} 66 | 67 | tar xfz ${ES} 68 | tar xfz ${LS} 69 | tar xfz ${KB} 70 | tar xfz ${FB} 71 | 72 | ln -s ./elasticsearch-${VER} elasticsearch 73 | ln -s ./logstash-${VER} logstash 74 | ln -s ./kibana-${VER}-linux-x86_64 kibana 75 | ln -s ./filebeat-${VER}-linux-x86_64 filebeat 76 | 77 | sudo sh -c "echo 'ec2-user soft nofile 65536' >> /etc/security/limits.conf" 78 | sudo sh -c "echo 'ec2-user hard nofile 65536' >> /etc/security/limits.conf" 79 | sudo sh -c "echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf" 80 | sudo sysctl -p 81 | sudo su ec2-user 82 | } 83 | 84 | function es_start 85 | { 86 | cp ${CONF_DIR}/${ES_CONF} ${PKGS_DIR}/${ES_DIR}/config/elasticsearch.yml 87 | ${PKGS_DIR}/${ES_DIR}/bin/elasticsearch -d 88 | } 89 | 90 | function kb_start 91 | { 92 | cp ${CONF_DIR}/${KB_CONF} ${PKGS_DIR}/${KB_DIR}/config/${KB_CONF} 93 | sudo cp ${CONF_DIR}/${KB_SVC} ${SYS_DIR}/${KB_SVC} 94 | daemon_reload 95 | sudo systemctl start ${KB_SVC} 96 | } 97 | 98 | function fb_start 99 | { 100 | cp ${CONF_DIR}/${FB_CONF} ${PKGS_DIR}/${FB_DIR}/${FB_CONF} 101 | sudo cp ${CONF_DIR}/${FB_SVC} ${SYS_DIR}/${FB_SVC} 102 | daemon_reload 103 | sudo systemctl start ${FB_SVC} 104 | } 105 | 106 | function no_filter 107 | { 108 | ${PKGS_DIR}/${LS_DIR}/bin/logstash -e 'input { stdin {} } output { stdout {} }' 109 | } 110 | 111 | function grok_filter 112 | { 113 | ${PKGS_DIR}/${LS_DIR}/bin/logstash -f logstash_conf/grok.conf 114 | } 115 | 116 | function ruby_filter 117 | { 118 | ${PKGS_DIR}/${LS_DIR}/bin/logstash -f logstash_conf/ruby.conf 119 | } 120 | 121 | function daemon_reload 122 | { 123 | sudo systemctl daemon-reload 124 | } 125 | 126 | 127 | if [ -z $1 ]; then 128 | echo "##################### Menu ##############" 129 | echo " $ ./tuto [Command]" 130 | echo "#####################%%%%%%##############" 131 | echo " 1 : install elk packages" 132 | echo " 2 : start es, kibana, filebeat" 133 | echo " 3 : standard input/output, no filters" 134 | echo " 4 : standard input/output, grok filter" 135 | echo " 5 : standard input/output, ruby filter" 136 | echo "#########################################"; 137 | exit 1; 138 | fi 139 | 140 | case "$1" in 141 | "1" ) install_elk_packages;; 142 | "2" ) es_start; kb_start; fb_start;; 143 | "3" ) no_filter;; 144 | "4" ) grok_filter;; 145 | "5" ) ruby_filter;; 146 | *) echo "Incorrect Command" ;; 147 | esac 148 | -------------------------------------------------------------------------------- /tuto_ubuntu: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ES_URL="https://artifacts.elastic.co/downloads/elasticsearch" 4 | LS_URL="https://artifacts.elastic.co/downloads/logstash" 5 | KB_URL="https://artifacts.elastic.co/downloads/kibana" 6 | FB_URL="https://artifacts.elastic.co/downloads/beats/filebeat" 7 | 8 | VER="7.6.1" 9 | 10 | ES="elasticsearch-${VER}-linux-x86_64.tar.gz" 11 | LS="logstash-${VER}.tar.gz" 12 | KB="kibana-${VER}-linux-x86_64.tar.gz" 13 | FB="filebeat-${VER}-linux-x86_64.tar.gz" 14 | 15 | ES_PKGS=${ES_URL}/${ES} 16 | LS_PKGS=${LS_URL}/${LS} 17 | KB_PKGS=${KB_URL}/${KB} 18 | FB_PKGS=${FB_URL}/${FB} 19 | 20 | ES_DIR="elasticsearch" 21 | LS_DIR="logstash" 22 | KB_DIR="kibana" 23 | FB_DIR="filebeat" 24 | PKGS_DIR="packages" 25 | CONF_DIR="conf" 26 | SYS_DIR="/etc/systemd/system" 27 | 28 | ES_CONF="elasticsearch.yml" 29 | KB_CONF="kibana.yml" 30 | FB_CONF="filebeat.yml" 31 | 32 | LS_SVC="logstash.service" 33 | KB_SVC="kibana.service" 34 | FB_SVC="filebeat.service" 35 | 36 | USER=$(whoami) 37 | PWD=$(pwd) 38 | 39 | if [ $USER == "root" ]; then 40 | echo "root is not permitted" 41 | exit -1 42 | fi 43 | 44 | if [ $PWD != "/home/user/elastic-stack-tutorial" ]; then 45 | pwd 46 | echo "/home/user/elastic-stack-tutorial path is permitted only" 47 | exit -1 48 | fi 49 | 50 | function install_elk_packages 51 | { 52 | 53 | git pull 54 | 55 | sudo apt -y install wget 56 | sudo apt -y install default-jdk 57 | sudo apt -y install default-jre 58 | sudo apt -y install bzip2 epel-release 59 | sudo apt -y install npm 60 | 61 | mkdir ${PKGS_DIR} 62 | cd ${PKGS_DIR} 63 | 64 | wget ${ES_PKGS} 65 | wget ${LS_PKGS} 66 | wget ${KB_PKGS} 67 | wget ${FB_PKGS} 68 | 69 | tar xfz ${ES} 70 | tar xfz ${LS} 71 | tar xfz ${KB} 72 | tar xfz ${FB} 73 | 74 | ln -s ./elasticsearch-${VER} elasticsearch 75 | ln -s ./logstash-${VER} logstash 76 | ln -s ./kibana-${VER}-linux-x86_64 kibana 77 | ln -s ./filebeat-${VER}-linux-x86_64 filebeat 78 | 79 | sudo sh -c "echo 'user soft nofile 65536' >> /etc/security/limits.conf" 80 | sudo sh -c "echo 'user hard nofile 65536' >> /etc/security/limits.conf" 81 | sudo sh -c "echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf" 82 | sudo sysctl -p 83 | sudo su user 84 | } 85 | 86 | function es_start 87 | { 88 | cp ${CONF_DIR}/${ES_CONF} ${PKGS_DIR}/${ES_DIR}/config/elasticsearch.yml 89 | ${PKGS_DIR}/${ES_DIR}/bin/elasticsearch -d 90 | } 91 | 92 | function kb_start 93 | { 94 | cp ${CONF_DIR}/${KB_CONF} ${PKGS_DIR}/${KB_DIR}/config/${KB_CONF} 95 | sudo cp ${CONF_DIR}/${KB_SVC} ${SYS_DIR}/${KB_SVC} 96 | daemon_reload 97 | sudo systemctl start ${KB_SVC} 98 | } 99 | 100 | function fb_start 101 | { 102 | cp ${CONF_DIR}/${FB_CONF} ${PKGS_DIR}/${FB_DIR}/${FB_CONF} 103 | sudo cp ${CONF_DIR}/${FB_SVC} ${SYS_DIR}/${FB_SVC} 104 | daemon_reload 105 | sudo systemctl start ${FB_SVC} 106 | } 107 | 108 | function no_filter 109 | { 110 | ${PKGS_DIR}/${LS_DIR}/bin/logstash -e 'input { stdin {} } output { stdout {} }' 111 | } 112 | 113 | function grok_filter 114 | { 115 | ${PKGS_DIR}/${LS_DIR}/bin/logstash -f logstash_conf/grok.conf 116 | } 117 | 118 | function ruby_filter 119 | { 120 | ${PKGS_DIR}/${LS_DIR}/bin/logstash -f logstash_conf/ruby.conf 121 | } 122 | 123 | function daemon_reload 124 | { 125 | sudo systemctl daemon-reload 126 | } 127 | 128 | 129 | if [ -z $1 ]; then 130 | echo "##################### Menu ##############" 131 | echo " $ ./tuto [Command]" 132 | echo "#####################%%%%%%##############" 133 | echo " 1 : install elk packages" 134 | echo " 2 : start es, kibana, filebeat" 135 | echo " 3 : standard input/output, no filters" 136 | echo " 4 : standard input/output, grok filter" 137 | echo " 5 : standard input/output, ruby filter" 138 | echo "#########################################"; 139 | exit 1; 140 | fi 141 | 142 | case "$1" in 143 | "1" ) install_elk_packages;; 144 | "2" ) es_start; kb_start; fb_start;; 145 | "3" ) no_filter;; 146 | "4" ) grok_filter;; 147 | "5" ) ruby_filter;; 148 | *) echo "Incorrect Command" ;; 149 | esac 150 | --------------------------------------------------------------------------------