├── ElasticStack-Ansible ├── Deploy-VMs │ └── Vagrantfile ├── README.md ├── ansible_hosts ├── hosts ├── playbooks │ ├── deploy_elk.yml │ └── roles │ │ ├── all │ │ ├── elastic │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ └── elasticsearch.yml.j2 │ │ ├── tests │ │ │ ├── inventory │ │ │ └── test.yml │ │ └── vars │ │ │ └── main.yml │ │ ├── kibana │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ └── kibana.yml.j2 │ │ ├── tests │ │ │ ├── inventory │ │ │ └── test.yml │ │ └── vars │ │ │ └── main.yml │ │ ├── logstash │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ └── logstash.yml.j2 │ │ ├── tests │ │ │ ├── inventory │ │ │ └── test.yml │ │ └── vars │ │ │ └── main.yml │ │ ├── ngnix │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ ├── tests │ │ │ ├── inventory │ │ │ └── test.yml │ │ └── vars │ │ │ └── main.yml │ │ └── preinstall │ │ ├── README.md │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── meta │ │ └── main.yml │ │ ├── tasks │ │ ├── main.yml │ │ └── oracle_java.yml │ │ ├── tests │ │ ├── inventory │ │ └── test.yml │ │ └── vars │ │ └── main.yml └── pubkey.yml ├── ElasticStack-Kubernetes ├── Elastic-Stack-Kubernetes.png ├── README.md ├── es-client │ ├── es-client-hpo.yaml │ ├── es-client-service.yaml │ └── es-client.yaml ├── es-configs │ └── es-configMap.yaml ├── es-data │ ├── es-data-with-pvs.yaml │ └── es-data.yaml ├── es-master │ └── es-master.yaml ├── es-namespace │ └── namespace.yaml ├── kibana │ ├── cllient-auth.yaml │ ├── kibana-configMap.yaml │ ├── kibana-ingress.yaml │ ├── kibana-service.yaml │ └── kibana.yaml └── storageClass │ ├── ceph-secret.yaml │ └── storageClass-ceph.yaml ├── README.md └── elastic_stack.PNG /ElasticStack-Ansible/Deploy-VMs/Vagrantfile: -------------------------------------------------------------------------------- 1 | #### Writtten By: Oryan Omer#### 2 | #Declare Environment 3 | GUI = false # Enable/Disable GUI 4 | RAM = 2048 # Default memory size in MB 5 | 6 | # Network configuration 7 | DOMAIN = ".xxx" 8 | NETWORK = "192.168.50." 9 | NETMASK = "255.255.255.0" 10 | SYNC_FOLDER = "/opt/elk" 11 | # Default Virtualbox .box 12 | # See: https://wiki.debian.org/Teams/Cloud/VagrantBaseBoxes 13 | BOX = "generic/ubuntu1804" 14 | 15 | HOSTS = { 16 | "elk-node1" => [NETWORK+"150", RAM, GUI, BOX], 17 | "elk-node2" => [NETWORK+"151", RAM, GUI, BOX], 18 | "elk-node3" => [NETWORK+"152", RAM, GUI, BOX], 19 | "elk-node4" => [NETWORK+"153", RAM, GUI, BOX], 20 | "elk-node5" => [NETWORK+"154", RAM, GUI, BOX], 21 | # "keystore" => [NETWORK+"105", RAM, GUI, BOX], 22 | } 23 | 24 | Vagrant.configure(2) do |config| 25 | #Conf each Vm 26 | config.hostmanager.enabled = false 27 | config.hostmanager.manage_host = true 28 | config.hostmanager.manage_guest = true 29 | HOSTS.each do | (name, cfg) | 30 | 31 | #Config VM Settings 32 | ipaddr, ram, gui, box = cfg 33 | config.vm.define name do |machine| 34 | machine.vm.box = box 35 | machine.vm.guest = :debian 36 | machine.vm.hostname = name 37 | 38 | machine.vm.provider :libvirt do |libvirt| 39 | libvirt.memory = ram 40 | libvirt.cpus = 2 41 | # libvirt.connect_via_ssh = true 42 | end 43 | 44 | 45 | #Conf Network 46 | machine.vm.network :private_network, :ip => ipaddr 47 | 48 | machine.vm.network :public_network, 49 | :dev => "virbr0", 50 | :mode => "bridge", 51 | :type => "bridge" 52 | 53 | #sync ELK FOlder 54 | machine.vm.synced_folder SYNC_FOLDER, "/opt/elk/", create: true 55 | machine.ssh.forward_agent = true 56 | # machine.ssh.insert_key = true 57 | 58 | 59 | 60 | #install Ansible on master 61 | if name == "elk-node1" 62 | machine.vm.provision "shell", inline: "sudo apt-get update -y \ 63 | && sudo apt-get install software-properties-common -y \ 64 | && sudo apt-add-repository ppa:ansible/ansible \ 65 | && sudo apt-get -y update \ 66 | && sudo apt-get install ansible -y" 67 | end 68 | 69 | 70 | 71 | 72 | # Install Docker 73 | machine.vm.provision "shell", inline: "sudo apt-get update -y" 74 | machine.vm.provision "shell", inline: "sudo apt-get install python python-pip -y" 75 | machine.vm.provision "shell", inline: "sudo apt-get install apt-transport-https ca-certificates curl software-properties-common -y" 76 | machine.vm.provision "shell", inline: "sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -" 77 | machine.vm.provision "shell", inline: 'sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable"' 78 | machine.vm.provision "shell", inline: "sudo apt-get update -y" 79 | machine.vm.provision "shell", inline: "sudo apt-get install docker-ce -y" 80 | 81 | #Install Docker-Compose 82 | machine.vm.provision "shell", inline: 'sudo curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \ 83 | && sudo chmod +x /usr/local/bin/docker-compose' 84 | 85 | end 86 | 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/README.md: -------------------------------------------------------------------------------- 1 | # Deploy Elastic Stack on VMS's 2 | 3 | # Instructions: 4 | * At first, ensure pubkey is distributed across all hosts. 5 | 6 | * Second, update ansible_hosts file, which will match to your hosts 7 | 8 | * Third, run the playbook and don't forget to add ansible_hosts.ini file. 9 | 10 | 11 | ## Enjoy (: 12 | 13 | ![elastic stack](https://www.itopstimes.com/wp-content/uploads/2019/01/2b6024713b7ace798502139fd5ab8fe4-490x208.png) 14 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/ansible_hosts: -------------------------------------------------------------------------------- 1 | 2 | [elastic_master_servers] 3 | elk-node1 node_type=master 4 | 5 | [elastic_data_servers] 6 | elk-node2 node_type=data 7 | elk-node3 node_type=data 8 | 9 | [elastic_servers:children] 10 | elastic_master_servers 11 | elastic_data_servers 12 | 13 | [kibana_servers] 14 | elk-node4 15 | elk-node5 16 | 17 | 18 | [logstash_servers] 19 | elk-node5 20 | 21 | [elk_servers:children] 22 | elastic_servers 23 | kibana_servers 24 | logstash_servers 25 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/hosts: -------------------------------------------------------------------------------- 1 | 192.168.50.150 elk_node1 2 | 192.168.50.151 elk_node2 3 | 192.168.50.152 elk_node3 4 | 192.168.50.153 elk_node4 5 | 192.168.50.154 elk_node5 6 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/deploy_elk.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: elk_servers 3 | become: yes 4 | tags: preinstall 5 | roles: 6 | - preinstall 7 | 8 | - hosts: elastic_servers 9 | become: yes 10 | tags: elastic 11 | roles: 12 | - elastic 13 | 14 | - hosts: logstash_servers 15 | become: yes 16 | tags: logstash 17 | roles: 18 | - logstash 19 | 20 | - hosts: kibana_servers 21 | become: yes 22 | tags: kibana 23 | roles: 24 | - kibana 25 | 26 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/all: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OryanOmer/ElasticStack/682775964ac43d9baa3152a3bfad6fe7ed25feda/ElasticStack-Ansible/playbooks/roles/all -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for playbooks/roles/elastic -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for playbooks/roles/elastic -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for playbooks/roles/elastic 3 | - name: Copy Elastic Image to Server 4 | copy: 5 | src: "/opt/elk/installs/{{ elastic_image }}" 6 | dest: "/opt/{{ elastic_image }}" 7 | 8 | - name: Install Elastic Image 9 | apt: 10 | deb: "/opt/{{ elastic_image }}" 11 | state: installed 12 | 13 | - name: Genrate Jinja Template For Elastic servers 14 | template: 15 | src: elasticsearch.yml.j2 16 | dest: /etc/elasticsearch/elasticsearch.yml 17 | 18 | - name: chown logs directories for elastic user 19 | shell: "mkdir /opt/elastic/logs -p | chown elasticsearch /opt/elastic/ -R" 20 | 21 | - name: Start elastic service 22 | become: elasticsearch 23 | service: 24 | name: elasticsearch 25 | state: restarted 26 | 27 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/templates/elasticsearch.yml.j2: -------------------------------------------------------------------------------- 1 | # ======================== Elasticsearch Configuration ========================= 2 | # 3 | # NOTE: Elasticsearch comes with reasonable defaults for most settings. 4 | # Before you set out to tweak and tune the configuration, make sure you 5 | # understand what are you trying to accomplish and the consequences. 6 | # 7 | # The primary way of configuring a node is via this file. This template lists 8 | # the most important settings you may want to configure for a production cluster. 9 | # 10 | # Please consult the documentation for further information on configuration options: 11 | # https://www.elastic.co/guide/en/elasticsearch/reference/index.html 12 | # 13 | # ---------------------------------- Cluster ----------------------------------- 14 | # 15 | # Use a descriptive name for your cluster: 16 | # 17 | cluster.name: "{{ cluster_name }}" 18 | # 19 | # ------------------------------------ Node ------------------------------------ 20 | # 21 | # Use a descriptive name for the node: 22 | # 23 | node.name: "{{ ansible_hostname }}" 24 | {% if node_type == "master" %} 25 | node.master: true 26 | node.data: false 27 | {% endif %} 28 | 29 | {% if node_type == "data" %} 30 | node.master: false 31 | node.data: true 32 | {% endif %} 33 | 34 | {% if node_type == "client" %} 35 | node.master: false 36 | node.data: false 37 | {% endif %} 38 | 39 | # Add custom attributes to the node: 40 | # 41 | #node.attr.rack: r1 42 | # 43 | # ----------------------------------- Paths ------------------------------------ 44 | # 45 | # Path to directory where to store the data (separate multiple locations by comma): 46 | # 47 | path.data: "{{ elastic_data_path }}" 48 | # 49 | # Path to log files: 50 | # 51 | path.logs: "{{ elastic_logs_path }}" 52 | # 53 | # ----------------------------------- Memory ----------------------------------- 54 | # 55 | # Lock the memory on startup: 56 | # 57 | #bootstrap.memory_lock: true 58 | # 59 | # Make sure that the heap size is set to about half the memory available 60 | # on the system and that the owner of the process is allowed to use this 61 | # limit. 62 | # 63 | # Elasticsearch performs poorly when the system is swapping the memory. 64 | # 65 | # ---------------------------------- Network ----------------------------------- 66 | # 67 | # Set the bind address to a specific IP (IPv4 or IPv6): 68 | # 69 | network.host: 0.0.0.0 70 | # 71 | # Set a custom port for HTTP: 72 | # 73 | #http.port: 9200 74 | # 75 | # For more information, consult the network module documentation. 76 | # 77 | # --------------------------------- Discovery ---------------------------------- 78 | # 79 | # Pass an initial list of hosts to perform discovery when new node is started: 80 | # The default list of hosts is ["127.0.0.1", "[::1]"] 81 | # 82 | discovery.zen.ping.unicast.hosts: [{% for i in groups['elastic_servers'] %}"{{ i }}",{% endfor %}] 83 | 84 | # Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1): 85 | # 86 | #discovery.zen.minimum_master_nodes: 87 | # 88 | # For more information, consult the zen discovery module documentation. 89 | # 90 | # ---------------------------------- Gateway ----------------------------------- 91 | # 92 | # Block initial recovery after a full cluster restart until N nodes are started: 93 | # 94 | #gateway.recover_after_nodes: 3 95 | # 96 | # For more information, consult the gateway module documentation. 97 | # 98 | # ---------------------------------- Various ----------------------------------- 99 | # 100 | # Require explicit names when deleting indices: 101 | # 102 | #action.destructive_requires_name: true 103 | #threadpool.bulk.queue_size: 5000 104 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - playbooks/roles/elastic -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/elastic/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for playbooks/roles/elastic 3 | elastic_logs_path: "/opt/elastic/logs" 4 | elastic_data_path: "/var/lib/elasticsearch/" 5 | cluster_name: "xxx_cluster" 6 | elastic_image: "elasticsearch-6.5.4.deb" 7 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for playbooks/roles/kibana -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for playbooks/roles/kibana -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for playbooks/roles/kibana 3 | - name: Copy kibana Image to Server 4 | copy: 5 | src: "/opt/elk/installs/{{ kibana_image }}" 6 | dest: "/opt/{{ kibana_image }}" 7 | 8 | - name: Install kibana Image 9 | apt: 10 | deb: "/opt/{{ kibana_image }}" 11 | state: installed 12 | 13 | - name: Genrate Jinja Template For kibana servers 14 | template: 15 | src: kibana.yml.j2 16 | dest: /etc/kibana/config/kibana.yml 17 | 18 | - name: Start kibana service 19 | become: kibana 20 | service: 21 | name: kibana 22 | enabled: yes 23 | state: restarted 24 | 25 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/templates/kibana.yml.j2: -------------------------------------------------------------------------------- 1 | # Kibana is served by a back end server. This setting specifies the port to use. 2 | #server.port: 5601 3 | 4 | # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. 5 | # The default is 'localhost', which usually means remote machines will not be able to connect. 6 | # To allow connections from remote users, set this parameter to a non-loopback address. 7 | server.host: "0.0.0.0" 8 | 9 | # Enables you to specify a path to mount Kibana at if you are running behind a proxy. 10 | # Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath 11 | # from requests it receives, and to prevent a deprecation warning at startup. 12 | # This setting cannot end in a slash. 13 | #server.basePath: "" 14 | 15 | # Specifies whether Kibana should rewrite requests that are prefixed with 16 | # `server.basePath` or require that they are rewritten by your reverse proxy. 17 | # This setting was effectively always `false` before Kibana 6.3 and will 18 | # default to `true` starting in Kibana 7.0. 19 | #server.rewriteBasePath: false 20 | 21 | # The maximum payload size in bytes for incoming server requests. 22 | #server.maxPayloadBytes: 1048576 23 | 24 | # The Kibana server's name. This is used for display purposes. 25 | server.name: "{{ ansible_hostname }}" 26 | 27 | # The URL of the Elasticsearch instance to use for all your queries. 28 | elasticsearch.url: "elk-node1:9200" 29 | 30 | # When this setting's value is true Kibana uses the hostname specified in the server.host 31 | # setting. When the value of this setting is false, Kibana uses the hostname of the host 32 | # that connects to this Kibana instance. 33 | #elasticsearch.preserveHost: true 34 | 35 | # Kibana uses an index in Elasticsearch to store saved searches, visualizations and 36 | # dashboards. Kibana creates a new index if the index doesn't already exist. 37 | #kibana.index: ".kibana" 38 | 39 | # The default application to load. 40 | #kibana.defaultAppId: "home" 41 | 42 | # If your Elasticsearch is protected with basic authentication, these settings provide 43 | # the username and password that the Kibana server uses to perform maintenance on the Kibana 44 | # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which 45 | # is proxied through the Kibana server. 46 | #elasticsearch.username: "user" 47 | #elasticsearch.password: "pass" 48 | 49 | # Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. 50 | # These settings enable SSL for outgoing requests from the Kibana server to the browser. 51 | #server.ssl.enabled: false 52 | #server.ssl.certificate: /path/to/your/server.crt 53 | #server.ssl.key: /path/to/your/server.key 54 | 55 | # Optional settings that provide the paths to the PEM-format SSL certificate and key files. 56 | # These files validate that your Elasticsearch backend uses the same key files. 57 | #elasticsearch.ssl.certificate: /path/to/your/client.crt 58 | #elasticsearch.ssl.key: /path/to/your/client.key 59 | 60 | # Optional setting that enables you to specify a path to the PEM file for the certificate 61 | # authority for your Elasticsearch instance. 62 | #elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ] 63 | 64 | # To disregard the validity of SSL certificates, change this setting's value to 'none'. 65 | #elasticsearch.ssl.verificationMode: full 66 | 67 | # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of 68 | # the elasticsearch.requestTimeout setting. 69 | #elasticsearch.pingTimeout: 1500 70 | 71 | # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value 72 | # must be a positive integer. 73 | #elasticsearch.requestTimeout: 30000 74 | 75 | # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side 76 | # headers, set this value to [] (an empty list). 77 | #elasticsearch.requestHeadersWhitelist: [ authorization ] 78 | 79 | # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten 80 | # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. 81 | #elasticsearch.customHeaders: {} 82 | 83 | # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. 84 | #elasticsearch.shardTimeout: 30000 85 | 86 | # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. 87 | #elasticsearch.startupTimeout: 5000 88 | 89 | # Logs queries sent to Elasticsearch. Requires logging.verbose set to true. 90 | #elasticsearch.logQueries: false 91 | 92 | # Specifies the path where Kibana creates the process ID file. 93 | #pid.file: /var/run/kibana.pid 94 | 95 | # Enables you specify a file where Kibana stores log output. 96 | #logging.dest: stdout 97 | 98 | # Set the value of this setting to true to suppress all logging output. 99 | #logging.silent: false 100 | 101 | # Set the value of this setting to true to suppress all logging output other than error messages. 102 | #logging.quiet: false 103 | 104 | # Set the value of this setting to true to log all events, including system usage information 105 | # and all requests. 106 | #logging.verbose: false 107 | 108 | # Set the interval in milliseconds to sample system and process performance 109 | # metrics. Minimum is 100ms. Defaults to 5000. 110 | #ops.interval: 5000 111 | 112 | # Specifies locale to be used for all localizable strings, dates and number formats. 113 | #i18n.locale: "en" 114 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - playbooks/roles/kibana -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/kibana/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for playbooks/roles/kibana 3 | kibana_image: "kibana-6.5.4-amd64.deb" 4 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for playbooks/roles/logstash -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for playbooks/roles/logstash -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for playbooks/roles/logstash 3 | - name: Copy Logstash Image to Server 4 | copy: 5 | src: "/opt/elk/installs/{{ logstash_image }}" 6 | dest: "/opt/{{ logstash_image }}" 7 | 8 | - name: Install Logstash Image 9 | apt: 10 | deb: "/opt/{{ logstash_image }}" 11 | state: installed 12 | 13 | - name: Genrate Jinja Template For Logstash servers 14 | template: 15 | src: logstash.yml.j2 16 | dest: /etc/logstash/logstash.yml 17 | 18 | - name: Distrbute pipeline files 19 | synchronize: 20 | src: pipelines/ 21 | dest: /etc/logstash/conf.d/ 22 | recursive: yes 23 | 24 | - name: Start logstash service 25 | become: logstash 26 | service: 27 | name: logstash 28 | enabled: yes 29 | state: restarted 30 | 31 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/templates/logstash.yml.j2: -------------------------------------------------------------------------------- 1 | # Settings file in YAML 2 | # 3 | # Settings can be specified either in hierarchical form, e.g.: 4 | # 5 | # pipeline: 6 | # batch: 7 | # size: 125 8 | # delay: 5 9 | # 10 | # Or as flat keys: 11 | # 12 | # pipeline.batch.size: 125 13 | # pipeline.batch.delay: 5 14 | # 15 | # ------------ Node identity ------------ 16 | # 17 | # Use a descriptive name for the node: 18 | # 19 | node.name: "{{ ansible_hostname }}" 20 | # 21 | # If omitted the node name will default to the machine's host name 22 | # 23 | # ------------ Data path ------------------ 24 | # 25 | # Which directory should be used by logstash and its plugins 26 | # for any persistent needs. Defaults to LOGSTASH_HOME/data 27 | # 28 | path.data: /var/lib/logstash 29 | # 30 | # ------------ Pipeline Settings -------------- 31 | # 32 | # The ID of the pipeline. 33 | # 34 | # pipeline.id: main 35 | # 36 | # Set the number of workers that will, in parallel, execute the filters+outputs 37 | # stage of the pipeline. 38 | # 39 | # This defaults to the number of the host's CPU cores. 40 | # 41 | # pipeline.workers: 2 42 | # 43 | # How many events to retrieve from inputs before sending to filters+workers 44 | # 45 | # pipeline.batch.size: 125 46 | # 47 | # How long to wait in milliseconds while polling for the next event 48 | # before dispatching an undersized batch to filters+outputs 49 | # 50 | # pipeline.batch.delay: 50 51 | # 52 | # Force Logstash to exit during shutdown even if there are still inflight 53 | # events in memory. By default, logstash will refuse to quit until all 54 | # received events have been pushed to the outputs. 55 | # 56 | # WARNING: enabling this can lead to data loss during shutdown 57 | # 58 | # pipeline.unsafe_shutdown: false 59 | # 60 | # ------------ Pipeline Configuration Settings -------------- 61 | # 62 | # Where to fetch the pipeline configuration for the main pipeline 63 | # 64 | # path.config: 65 | # 66 | # Pipeline configuration string for the main pipeline 67 | # 68 | # config.string: 69 | # 70 | # At startup, test if the configuration is valid and exit (dry run) 71 | # 72 | # config.test_and_exit: false 73 | # 74 | # Periodically check if the configuration has changed and reload the pipeline 75 | # This can also be triggered manually through the SIGHUP signal 76 | # 77 | # config.reload.automatic: false 78 | # 79 | # How often to check if the pipeline configuration has changed (in seconds) 80 | # 81 | # config.reload.interval: 3s 82 | # 83 | # Show fully compiled configuration as debug log message 84 | # NOTE: --log.level must be 'debug' 85 | # 86 | # config.debug: false 87 | # 88 | # When enabled, process escaped characters such as \n and \" in strings in the 89 | # pipeline configuration files. 90 | # 91 | # config.support_escapes: false 92 | # 93 | # ------------ Module Settings --------------- 94 | # Define modules here. Modules definitions must be defined as an array. 95 | # The simple way to see this is to prepend each `name` with a `-`, and keep 96 | # all associated variables under the `name` they are associated with, and 97 | # above the next, like this: 98 | # 99 | # modules: 100 | # - name: MODULE_NAME 101 | # var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE 102 | # var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE 103 | # var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE 104 | # var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE 105 | # 106 | # Module variable names must be in the format of 107 | # 108 | # var.PLUGIN_TYPE.PLUGIN_NAME.KEY 109 | # 110 | # modules: 111 | # 112 | # ------------ Cloud Settings --------------- 113 | # Define Elastic Cloud settings here. 114 | # Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy 115 | # and it may have an label prefix e.g. staging:dXMtZ... 116 | # This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' 117 | # cloud.id: 118 | # 119 | # Format of cloud.auth is: : 120 | # This is optional 121 | # If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' 122 | # If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' 123 | # cloud.auth: elastic: 124 | # 125 | # ------------ Queuing Settings -------------- 126 | # 127 | # Internal queuing model, "memory" for legacy in-memory based queuing and 128 | # "persisted" for disk-based acked queueing. Defaults is memory 129 | # 130 | # queue.type: memory 131 | # 132 | # If using queue.type: persisted, the directory path where the data files will be stored. 133 | # Default is path.data/queue 134 | # 135 | # path.queue: 136 | # 137 | # If using queue.type: persisted, the page data files size. The queue data consists of 138 | # append-only data files separated into pages. Default is 64mb 139 | # 140 | # queue.page_capacity: 64mb 141 | # 142 | # If using queue.type: persisted, the maximum number of unread events in the queue. 143 | # Default is 0 (unlimited) 144 | # 145 | # queue.max_events: 0 146 | # 147 | # If using queue.type: persisted, the total capacity of the queue in number of bytes. 148 | # If you would like more unacked events to be buffered in Logstash, you can increase the 149 | # capacity using this setting. Please make sure your disk drive has capacity greater than 150 | # the size specified here. If both max_bytes and max_events are specified, Logstash will pick 151 | # whichever criteria is reached first 152 | # Default is 1024mb or 1gb 153 | # 154 | # queue.max_bytes: 1024mb 155 | # 156 | # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint 157 | # Default is 1024, 0 for unlimited 158 | # 159 | # queue.checkpoint.acks: 1024 160 | # 161 | # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint 162 | # Default is 1024, 0 for unlimited 163 | # 164 | # queue.checkpoint.writes: 1024 165 | # 166 | # If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page 167 | # Default is 1000, 0 for no periodic checkpoint. 168 | # 169 | # queue.checkpoint.interval: 1000 170 | # 171 | # ------------ Dead-Letter Queue Settings -------------- 172 | # Flag to turn on dead-letter queue. 173 | # 174 | # dead_letter_queue.enable: false 175 | 176 | # If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries 177 | # will be dropped if they would increase the size of the dead letter queue beyond this setting. 178 | # Default is 1024mb 179 | # dead_letter_queue.max_bytes: 1024mb 180 | 181 | # If using dead_letter_queue.enable: true, the directory path where the data files will be stored. 182 | # Default is path.data/dead_letter_queue 183 | # 184 | # path.dead_letter_queue: 185 | # 186 | # ------------ Metrics Settings -------------- 187 | # 188 | # Bind address for the metrics REST endpoint 189 | # 190 | http.host: "0.0.0.0" 191 | # 192 | # Bind port for the metrics REST endpoint, this option also accept a range 193 | # (9600-9700) and logstash will pick up the first available ports. 194 | # 195 | # http.port: 9600-9700 196 | # 197 | # ------------ Debugging Settings -------------- 198 | # 199 | # Options for log.level: 200 | # * fatal 201 | # * error 202 | # * warn 203 | # * info (default) 204 | # * debug 205 | # * trace 206 | # 207 | # log.level: info 208 | path.logs: /var/log/logstash 209 | # 210 | # ------------ Other Settings -------------- 211 | # 212 | # Where to find custom plugins 213 | # path.plugins: [] 214 | # 215 | # ------------ X-Pack Settings (not applicable for OSS build)-------------- 216 | # 217 | # X-Pack Monitoring 218 | # https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html 219 | #xpack.monitoring.enabled: false 220 | #xpack.monitoring.elasticsearch.username: logstash_system 221 | #xpack.monitoring.elasticsearch.password: password 222 | #xpack.monitoring.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] 223 | #xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] 224 | #xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file 225 | #xpack.monitoring.elasticsearch.ssl.truststore.password: password 226 | #xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file 227 | #xpack.monitoring.elasticsearch.ssl.keystore.password: password 228 | #xpack.monitoring.elasticsearch.ssl.verification_mode: certificate 229 | #xpack.monitoring.elasticsearch.sniffing: false 230 | #xpack.monitoring.collection.interval: 10s 231 | #xpack.monitoring.collection.pipeline.details.enabled: true 232 | # 233 | # X-Pack Management 234 | # https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html 235 | #xpack.management.enabled: false 236 | #xpack.management.pipeline.id: ["main", "apache_logs"] 237 | #xpack.management.elasticsearch.username: logstash_admin_user 238 | #xpack.management.elasticsearch.password: password 239 | #xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] 240 | #xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] 241 | #xpack.management.elasticsearch.ssl.truststore.path: /path/to/file 242 | #xpack.management.elasticsearch.ssl.truststore.password: password 243 | #xpack.management.elasticsearch.ssl.keystore.path: /path/to/file 244 | #xpack.management.elasticsearch.ssl.keystore.password: password 245 | #xpack.management.elasticsearch.ssl.verification_mode: certificate 246 | #xpack.management.elasticsearch.sniffing: false 247 | #xpack.management.logstash.poll_interval: 5s 248 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - playbooks/roles/logstash -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/logstash/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for playbooks/roles/logstash 3 | logstash_image: logstash-6.5.4.deb -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for roles/ngnix -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for roles/ngnix -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for roles/ngnix -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - roles/ngnix -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/ngnix/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for roles/ngnix -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for playbooks/roles/preinstall -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for playbooks/roles/preinstall -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for playbooks/roles/preinstall 3 | - name: Install Java 4 | apt_repository: 5 | repo: ppa:webupd8team/java 6 | state: present 7 | update_cache: yes 8 | validate_certs: no 9 | 10 | - name: Copy Oracle-Java8 tar file 11 | copy: 12 | src: jdk-8u192-linux-x64.tar.gz 13 | dest: /opt/jdk-8u192-linux-x64.tar.gz 14 | 15 | - name: mkdir java file 16 | file: 17 | path: /opt/java 18 | state: directory 19 | mode: 0777 20 | owner: user 21 | 22 | - name: unarchive java in the installtion directory 23 | become: yes 24 | unarchive: 25 | src: "/opt/jdk-8u192-linux-x64.tar.gz" 26 | dest: "/opt/java" 27 | remote_src: no 28 | tags: archive 29 | 30 | - name: Java | Update alternatives 31 | alternatives: 32 | name: "{{ item }}" 33 | link: "/usr/bin/{{ item }}" 34 | path: "/opt/java/jdk1.8.0_192/bin/{{ item }}" 35 | with_items: 36 | - java 37 | - javac 38 | - jar 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/tasks/oracle_java.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Download Java 3 | command: "wget --no-check-certificate https://download.oracle.com/otn-pub/java/jdk/8u192-b12/750e1c8617c5452694857ad95c3ee230/jdk-8u192-linux-x64.tar.gz && dpkg -i jdk-8u192-linux-x64.tar.gz" 4 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - playbooks/roles/preinstall -------------------------------------------------------------------------------- /ElasticStack-Ansible/playbooks/roles/preinstall/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for playbooks/roles/preinstall 3 | download_url: "https://download.oracle.com/otn-pub/java/jdk/8u191-b12-demos/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64-demos.tar.gz" 4 | download_folder: /opt 5 | java_name: "{{download_folder}}/jdk1.8.0_05" 6 | java_archive: "{{download_folder}}/jdk-8u5-linux-x64.tar.gz" 7 | -------------------------------------------------------------------------------- /ElasticStack-Ansible/pubkey.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | vars: 4 | username: "user" 5 | ansible_ssh_user: vagrant 6 | ansible_ssh_pass: vagrant 7 | ansible_sudo_user: vagrant 8 | become: yes 9 | tasks: 10 | - name: ensure python is installed 11 | apt: 12 | name: python 13 | state: present 14 | 15 | - name: Create User 16 | user: 17 | name: "{{ username }}" 18 | comment: "{{ username }}" 19 | shell: /bin/bash 20 | groups: sudo 21 | append: yes 22 | ## run command 'mkpasswd --method=sha-512' to create your own encrypted password ## 23 | password: $6$8pwTAqXac2r$Noi1EUOz2PwGg4LIPVNfqFeFTPMKGTBfpZj.GxWfQrMVuJemd1dhl7VUwfc6hGtqDthlo4./vaw7cygje5Qqe. 24 | ssh_key_type: rsa 25 | tags: user 26 | 27 | - name: Grant Permission to .ssh directory on users 28 | file: 29 | path: "{{ item }}/.ssh" 30 | recurse: yes 31 | mode: 0777 32 | with_items: 33 | - "/home/user" 34 | - "/root" 35 | 36 | - name: Genrate Keys for user" 37 | shell: ssh-keygen -t rsa -b 4096 -C "xxx" -f "/home/user/.ssh/id_rsa" -q -N "" 38 | become: user 39 | when: ansible_hostname == "elk-node1" 40 | 41 | - name: "Genrate Keys for root user" 42 | shell: ssh-keygen -t rsa -b 4096 -C "xxx" -f "/root/.ssh/id_rsa" -q -N "" 43 | when: ansible_hostname == "elk-node-1" 44 | 45 | 46 | - name: Enable sudo without password 47 | lineinfile: 48 | path: /etc/sudoers 49 | state: present 50 | line: "{{ username }} ALL=(ALL) NOPASSWD:ALL" 51 | tags: sudo 52 | 53 | - name: Set Authorized Keys For users 54 | authorized_key: 55 | user: "{{ item.user }}" 56 | state: present 57 | key: "{{ lookup('file', '{{ item.file }}/.ssh/id_rsa.pub') }}" 58 | tags: keys 59 | with_items: 60 | - {user: "{{ username }}", file: "/home/user" } 61 | - {user: "root", file: "/root" } 62 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/Elastic-Stack-Kubernetes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OryanOmer/ElasticStack/682775964ac43d9baa3152a3bfad6fe7ed25feda/ElasticStack-Kubernetes/Elastic-Stack-Kubernetes.png -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/README.md: -------------------------------------------------------------------------------- 1 | # Deploy Elastic Stack on Kubernetes Cluster 2 | 3 | This guide will helps you deploy Elastic Stack 7.1 on each Kubernetes cluster: eks, gke, rancher and kops. 4 | 5 | Here is an image of the architecture of the elastic stack: 6 | ![ElasticStack](https://github.com/OryanOmer/ElasticStack/blob/master/ElasticStack-Kubernetes/Elastic-Stack-Kubernetes.png) 7 | 8 | ### Quick brief about the architecture: 9 | * Master Nodes - responsible for managing the cluster state and health. 10 | * Data Nodes- responsible keeps the data and perform data related operations such as CRUD, search, and aggregations. 11 | * Client Nodes- Also called “coordinating node”, responsible for client requests and query data from the cluster. 12 | * Kibana -Kibana lets you visualize your Elasticsearch data and navigate the Elastic Stack. 13 | 14 | One of the main changes at this relase is security, at this version you have the option to manage users rbac in diffrent spaces. 15 | After you will bring up your Elastic stack environment and connect to the kibana, you have to provide username and password to login to the kibana, for make things easy the default username and password is elastic:password. 16 | 17 | ### Prerequisites: 18 | * Kuberntes up and running, supported at 1.13+. 19 | 20 | * Clone the repository to your server. 21 | ``` bash 22 | git clone https://github.com/OryanOmer/ElasticStack.git && cd ./ElasticStack/ElasticStack-Kubernetes 23 | ``` 24 | 25 | 26 | ### Instructions: 27 | * First, we will deploy isolated namespace for our Elastic Stack using the command: 28 | ``` bash 29 | kubectl apply -f es-namespace/namespace.yaml 30 | ``` 31 | 32 | * Second, we will deploy configMap for our Elastic Stack environment (If you want to change params for the elastic, for example, JVM OPS, Elastic data-path..etc) you can do that there, but be careful and keep on alignment cross the nodes. 33 | 34 | After finishing edit the configMap.yaml, deploy the configMap by the command: 35 | 36 | ``` bash 37 | kubectl apply -f es-configs/es-configMap.yaml 38 | ``` 39 | 40 | * Third, now we will deploy the elastic master containers by the command: 41 | ``` bash 42 | kubectl apply -f es-master/es-master.yaml 43 | ``` 44 | the master containers are statefulSet with headless-service. 45 | 46 | * Four,we will deploy the elastic data containers, There are 2 options for deploy data nodes: 47 | 1. data nodes without pvc. 48 | 2. data nodes with pvc. 49 | 50 | For option one press the command: 51 | ``` bash 52 | kubectl apply -f es-data/es-data.yaml 53 | ``` 54 | 55 | For option 2, you should edit the ./es-data/es-data-pvs.yaml file and change < StorageClass > to your StorageClass. 56 | After that press the command: 57 | ``` bash 58 | kubectl apply -f es-data/es-data-pvs.yaml 59 | ``` 60 | In addition, i added an example of *Storage Class* yaml for your convenience 61 | 62 | the data containers are statefulSet with headless-service. 63 | 64 | * Five, we will deploy the elastic proxy containers(coordinating node) to listen for client requests by the command: 65 | ``` bash 66 | kubectl apply -f es-client/es-client.yaml 67 | kubectl apply -f es-client/es-client-service.yaml 68 | ``` 69 | the proxy containers are done by replicaSet of 3 containers with NodePort service for client access. 70 | 71 | * Also, there is an option to deploy HPA(Horizontal Pod AutoScaler) for the clients nodes, the default metric is for CPU Utilization over 80%. 72 | For deploy HPA on elastic client nodes press the command: 73 | ``` bash 74 | kubectl apply -f es-client/es-client-hpo.yaml 75 | ``` 76 | 77 | * At the end, we will deploy the kibana to access the elastic cluster via UI. 78 | You can edit the kibana.yaml in the kibana-configMap.yaml as you want. 79 | ``` bash 80 | kubectl apply -f kibana/kibana-configMap.yaml 81 | kubectl apply -f kibana/kibana.yaml 82 | kubectl apply -f kibana/kibana-service.yaml 83 | kubectl apply -f kibana/kibana-ingress.yaml 84 | ``` 85 | The kibana containers are done by replicaSet of 3 containers with NodePort Service. 86 | 87 | 88 | 89 | ### Purge the cluster by the command: 90 | ``` bash 91 | kubectl delete -f kibana/kibana-service.yaml 92 | kubectl delete -f kibana/kibana.yaml 93 | kubectl delete -f kibana/kibana-configMap.yaml 94 | kubectl delete -f es-client/es-client-service.yaml 95 | kubectl delete -f es-client/es-client.yaml 96 | kubectl delete -f es-data/es-data.yaml 97 | kubectl delete -f es-master/es-master.yaml 98 | kubectl delete -f es-configs/es-configMap.yaml 99 | kubectl delete -f es-namespace/namespace.yaml 100 | ``` 101 | 102 | ![E&K](https://anchormen.nl/wp-content/uploads/2017/12/elasticsearch-on-kubernetes.jpg) 103 | 104 | 105 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-client/es-client-hpo.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v1 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: es-client 5 | namespace: elasticsearch 6 | spec: 7 | maxReplicas: 5 8 | minReplicas: 3 9 | scaleTargetRef: 10 | apiVersion: extensions/v1beta1 11 | kind: Deployment 12 | name: es-client 13 | targetCPUUtilizationPercentage: 80 -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-client/es-client-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: elasticsearch 5 | namespace: elasticsearch 6 | labels: 7 | role: client 8 | app: elasticsearch 9 | spec: 10 | selector: 11 | role: client 12 | app: elasticsearch 13 | type: NodePort 14 | ports: 15 | - port: 9200 16 | targetPort: 9200 17 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-client/es-client.yaml: -------------------------------------------------------------------------------- 1 | #Headleass Service 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: es-client 7 | namespace: elasticsearch 8 | labels: 9 | role: client 10 | app: elasticsearch 11 | spec: 12 | selector: 13 | app: elasticsearch 14 | role: client 15 | clusterIP: None 16 | ports: 17 | - port: 9300 18 | targetPort: 9300 19 | 20 | --- 21 | #es-client Deployment 22 | apiVersion: apps/v1beta2 23 | kind: Deployment 24 | metadata: 25 | name: es-client 26 | namespace: elasticsearch 27 | labels: 28 | app: elasticsearch 29 | role: client 30 | spec: 31 | selector: 32 | matchLabels: 33 | app: elasticsearch 34 | role: client 35 | replicas: 3 36 | template: 37 | metadata: 38 | name: es-client 39 | namespace: elasticsearch 40 | labels: 41 | app: elasticsearch 42 | role: client 43 | spec: 44 | terminationGracePeriodSeconds: 0 45 | initContainers: #init containers for disable swap and change max map count and increase 46 | - name: init-sysctl #change sysctl 47 | image: busybox 48 | command: 49 | - sysctl 50 | - -w 51 | - vm.max_map_count=262144 52 | securityContext: 53 | privileged: true 54 | - name: increase-the-ulimit 55 | image: busybox 56 | command: 57 | - sh 58 | - -c 59 | - ulimit -n 65536 60 | securityContext: 61 | privileged: true 62 | containers: #define elasticsearch container 63 | - name: es-client 64 | resources: 65 | limits: 66 | cpu: 1 67 | imagePullPolicy: IfNotPresent 68 | securityContext: 69 | privileged: true 70 | image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1 71 | env: 72 | - name: ELASTIC_PASSWORD 73 | valueFrom: 74 | configMapKeyRef: 75 | name: es-config 76 | key: ELASTIC_PASSWORD 77 | - name: ES_JAVA_OPTS 78 | valueFrom: 79 | configMapKeyRef: 80 | name: es-config 81 | key: ES_JAVA_OPTS 82 | ports: 83 | - containerPort: 9300 84 | name: es-tunnel 85 | - containerPort: 9200 86 | name: es-http 87 | volumeMounts: 88 | - name: es-config 89 | mountPath: /usr/share/elasticsearch/config/elasticsearch.yml 90 | subPath: elasticsearch.yml 91 | volumes: 92 | - name: es-config 93 | configMap: 94 | name: es-config 95 | items: 96 | - key: elasticsearch-client.yml 97 | path: elasticsearch.yml 98 | 99 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-configs/es-configMap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: es-config 5 | namespace: elasticsearch 6 | data: 7 | elasticsearch-data.yml: | 8 | cluster.name: es-oryan 9 | node.name: ${HOSTNAME}.es-data 10 | network.host: 0.0.0.0 11 | transport.tcp.port: 9300 12 | http.port: "9200" 13 | bootstrap.memory_lock: false 14 | discovery.zen.minimum_master_nodes: 2 15 | discovery.zen.ping.unicast.hosts: ["es-master-1.es-master","es-master-2.es-master","es-master-0.es-master"] 16 | xpack.security.enabled: true 17 | xpack.monitoring.enabled: true 18 | xpack.security.transport.ssl.client_authentication: optional 19 | xpack.security.transport.ssl.verification_mode: none 20 | xpack.license.self_generated.type: trial 21 | xpack.monitoring.elasticsearch.collection.enabled: true 22 | cluster.initial_master_nodes: 23 | - "es-master-1.es-master" 24 | - "es-master-2.es-master" 25 | - "es-master-0.es-master" 26 | 27 | node.data: "true" 28 | node.ingest: "false" 29 | node.master: "false" 30 | 31 | elasticsearch-master.yml: | 32 | cluster.name: es-oryan 33 | node.name: ${HOSTNAME}.es-master 34 | network.host: 0.0.0.0 35 | transport.tcp.port: 9300 36 | http.port: "9200" 37 | bootstrap.memory_lock: false 38 | discovery.zen.minimum_master_nodes: 2 39 | discovery.zen.ping.unicast.hosts: ["es-master-1.es-master","es-master-2.es-master","es-master-0.es-master"] 40 | xpack.security.enabled: true 41 | xpack.monitoring.enabled: true 42 | xpack.security.transport.ssl.client_authentication: optional 43 | xpack.security.transport.ssl.verification_mode: none 44 | xpack.license.self_generated.type: trial 45 | xpack.monitoring.elasticsearch.collection.enabled: true 46 | cluster.initial_master_nodes: 47 | - "es-master-1.es-master" 48 | - "es-master-2.es-master" 49 | - "es-master-0.es-master" 50 | node.data: false 51 | node.ingest: false 52 | node.master: true 53 | elasticsearch-client.yml: | 54 | cluster.name: es-oryan 55 | node.name: ${HOSTNAME}.es-client 56 | network.host: 0.0.0.0 57 | transport.tcp.port: 9300 58 | http.port: "9200" 59 | bootstrap.memory_lock: false 60 | discovery.zen.minimum_master_nodes: 2 61 | discovery.zen.ping.unicast.hosts: ["es-master-1.es-master","es-master-2.es-master","es-master-0.es-master"] 62 | xpack.security.enabled: true 63 | xpack.monitoring.enabled: true 64 | xpack.security.transport.ssl.client_authentication: optional 65 | xpack.security.transport.ssl.verification_mode: none 66 | xpack.license.self_generated.type: trial 67 | xpack.monitoring.elasticsearch.collection.enabled: true 68 | cluster.initial_master_nodes: 69 | - "es-master-1.es-master" 70 | - "es-master-2.es-master" 71 | - "es-master-0.es-master" 72 | node.data: "false" 73 | node.ingest: "true" 74 | node.master: "false" 75 | 76 | ES_JAVA_OPTS: "-Xms512m -Xmx512m" 77 | ELASTIC_USERNAME: "elastic" 78 | ELASTIC_PASSWORD: "password" 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-data/es-data-with-pvs.yaml: -------------------------------------------------------------------------------- 1 | #Headleass Service 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | namespace: elasticsearch 6 | name: es-data 7 | labels: 8 | app: elasticsearch 9 | role: data 10 | spec: 11 | clusterIP: None 12 | selector: 13 | app: elasticsearch 14 | role: data 15 | ports: 16 | - port: 9200 17 | name: http 18 | - port: 9300 19 | name: node-to-node 20 | 21 | --- 22 | #es-data Statefulset 23 | apiVersion: apps/v1 24 | kind: StatefulSet 25 | metadata: 26 | name: es-data 27 | namespace: elasticsearch 28 | labels: 29 | app: elasticsearch 30 | role: data 31 | spec: 32 | selector: 33 | matchLabels: 34 | app: elasticsearch 35 | role: data 36 | serviceName: es-data 37 | updateStrategy: 38 | type: RollingUpdate 39 | replicas: 3 40 | template: 41 | metadata: 42 | name: es-data 43 | namespace: elasticsearch 44 | labels: 45 | app: elasticsearch 46 | role: data 47 | spec: 48 | initContainers: #init containers for disable swap and change max map count and increase 49 | - name: init-sysctl #change sysctl 50 | image: busybox 51 | command: 52 | - sysctl 53 | - -w 54 | - vm.max_map_count=262144 55 | securityContext: 56 | privileged: true 57 | - name: increase-the-ulimit 58 | image: busybox 59 | command: 60 | - sh 61 | - -c 62 | - ulimit -n 65536 63 | securityContext: 64 | privileged: true 65 | containers: #define elasticsearch container 66 | - name: es-data 67 | imagePullPolicy: IfNotPresent 68 | securityContext: 69 | privileged: true 70 | image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1 71 | env: 72 | - name: ELASTIC_PASSWORD 73 | valueFrom: 74 | configMapKeyRef: 75 | name: es-config 76 | key: ELASTIC_PASSWORD 77 | - name: ES_JAVA_OPTS 78 | valueFrom: 79 | configMapKeyRef: 80 | name: es-config 81 | key: ES_JAVA_OPTS 82 | ports: 83 | - containerPort: 9300 84 | name: node-to-node 85 | - containerPort: 9200 86 | name: http 87 | volumeMounts: 88 | - name: es-config 89 | mountPath: /usr/share/elasticsearch/config/elasticsearch.yml 90 | subPath: elasticsearch.yml 91 | - name: es-data 92 | mountPath: /usr/share/elasticsearch/data 93 | volumes: 94 | - name: es-config 95 | configMap: 96 | name: es-config 97 | items: 98 | - key: elasticsearch-data.yml 99 | path: elasticsearch.yml 100 | volumeClaimTemplates: 101 | - metadata: 102 | name: es-data 103 | namespace: elasticsearch 104 | spec: 105 | accessModes: [ "ReadWriteOnce" ] 106 | storageClassName: "" 107 | resources: 108 | requests: 109 | storage: 10Gi 110 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-data/es-data.yaml: -------------------------------------------------------------------------------- 1 | #Headleass Service 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | namespace: elasticsearch 6 | name: es-data 7 | labels: 8 | app: elasticsearch 9 | role: data 10 | spec: 11 | clusterIP: None 12 | selector: 13 | app: elasticsearch 14 | role: data 15 | ports: 16 | - port: 9200 17 | name: http 18 | - port: 9300 19 | name: node-to-node 20 | 21 | --- 22 | #es-data Statefulset 23 | apiVersion: apps/v1 24 | kind: StatefulSet 25 | metadata: 26 | name: es-data 27 | namespace: elasticsearch 28 | labels: 29 | app: elasticsearch 30 | role: data 31 | spec: 32 | selector: 33 | matchLabels: 34 | app: elasticsearch 35 | role: data 36 | serviceName: es-data 37 | updateStrategy: 38 | type: RollingUpdate 39 | replicas: 3 40 | template: 41 | metadata: 42 | name: es-data 43 | namespace: elasticsearch 44 | labels: 45 | app: elasticsearch 46 | role: data 47 | spec: 48 | initContainers: #init containers for disable swap and change max map count and increase 49 | - name: init-sysctl #change sysctl 50 | image: busybox 51 | command: 52 | - sysctl 53 | - -w 54 | - vm.max_map_count=262144 55 | securityContext: 56 | privileged: true 57 | - name: increase-the-ulimit 58 | image: busybox 59 | command: 60 | - sh 61 | - -c 62 | - ulimit -n 65536 63 | securityContext: 64 | privileged: true 65 | containers: #define elasticsearch container 66 | - name: es-data 67 | imagePullPolicy: IfNotPresent 68 | securityContext: 69 | privileged: true 70 | image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1 71 | env: 72 | - name: ELASTIC_PASSWORD 73 | valueFrom: 74 | configMapKeyRef: 75 | name: es-config 76 | key: ELASTIC_PASSWORD 77 | - name: ES_JAVA_OPTS 78 | valueFrom: 79 | configMapKeyRef: 80 | name: es-config 81 | key: ES_JAVA_OPTS 82 | ports: 83 | - containerPort: 9300 84 | name: node-to-node 85 | - containerPort: 9200 86 | name: http 87 | volumeMounts: 88 | - name: es-config 89 | mountPath: /usr/share/elasticsearch/config/elasticsearch.yml 90 | subPath: elasticsearch.yml 91 | volumes: 92 | - name: es-config 93 | configMap: 94 | name: es-config 95 | items: 96 | - key: elasticsearch-data.yml 97 | path: elasticsearch.yml 98 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-master/es-master.yaml: -------------------------------------------------------------------------------- 1 | #Headleass Service 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: es-master 7 | namespace: elasticsearch 8 | labels: 9 | role: master 10 | app: elasticsearch 11 | spec: 12 | selector: 13 | app: elasticsearch 14 | role: master 15 | clusterIP: None 16 | ports: 17 | - port: 9300 18 | protocol: TCP 19 | targetPort: 9300 20 | 21 | --- 22 | #es-master Statefulset 23 | apiVersion: apps/v1beta2 24 | kind: StatefulSet 25 | metadata: 26 | name: es-master 27 | namespace: elasticsearch 28 | labels: 29 | app: elasticsearch 30 | role: master 31 | spec: 32 | selector: 33 | matchLabels: 34 | app: elasticsearch 35 | role: master 36 | serviceName: es-master 37 | updateStrategy: 38 | type: RollingUpdate 39 | replicas: 3 40 | template: 41 | metadata: 42 | name: es-master 43 | labels: 44 | app: elasticsearch 45 | role: master 46 | spec: 47 | terminationGracePeriodSeconds: 0 48 | initContainers: #init containers for disable swap and change max map count and increase ulimit 49 | - name: init-sysctl #change sysctl 50 | image: busybox 51 | command: 52 | - sysctl 53 | - -w 54 | - vm.max_map_count=262144 55 | securityContext: 56 | privileged: true 57 | - name: increase-the-ulimit 58 | image: busybox 59 | command: 60 | - sh 61 | - -c 62 | - ulimit -n 65536 63 | securityContext: 64 | privileged: true 65 | containers: #define elasticsearch container 66 | - name: es-master 67 | imagePullPolicy: IfNotPresent 68 | securityContext: 69 | privileged: true 70 | image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1 71 | env: 72 | - name: ELASTIC_PASSWORD 73 | valueFrom: 74 | configMapKeyRef: 75 | name: es-config 76 | key: ELASTIC_PASSWORD 77 | - name: ES_JAVA_OPTS 78 | valueFrom: 79 | configMapKeyRef: 80 | name: es-config 81 | key: ES_JAVA_OPTS 82 | ports: 83 | - containerPort: 9300 84 | name: es-tunnel 85 | - containerPort: 9200 86 | name: es-http 87 | volumeMounts: 88 | - name: es-config 89 | mountPath: /usr/share/elasticsearch/config/elasticsearch.yml 90 | subPath: elasticsearch.yml 91 | volumes: 92 | - name: es-config 93 | configMap: 94 | name: es-config 95 | items: 96 | - key: elasticsearch-master.yml 97 | path: elasticsearch.yml 98 | 99 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/es-namespace/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: elasticsearch -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/kibana/cllient-auth.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: es-client 5 | namespace: elasticsearch 6 | labels: 7 | role: es-client 8 | app: elasticsearch 9 | type: Opaque 10 | data: 11 | auth: cXdlMTIzCg== 12 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/kibana/kibana-configMap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: kibana 5 | namespace: elasticsearch 6 | labels: 7 | app: kibana 8 | role: kibana 9 | data: 10 | kibana.yml: | 11 | server.port: "5601" 12 | server.name: "kibana" 13 | server.host: "0.0.0.0" 14 | elasticsearch.url: http://es-client:9200 15 | xpack.security.sessionTimeout: 600000 #session timeout 16 | xpack.license.self_generated.type: base #security liceance base 17 | xpack.security.authProviders: [basic] 18 | xpack.security.enabled: true 19 | xpack.monitoring.enabled: true 20 | server.ssl.enabled: false 21 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/kibana/kibana-ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: kibana-ingress 5 | namespace: elasticsearch 6 | labels: 7 | role: kibana 8 | app: elasticsearch 9 | annotations: 10 | nginx.ingress.kubernetes.io/affinity: cookie 11 | spec: 12 | rules: 13 | - http: 14 | paths: 15 | - path: / 16 | backend: 17 | serviceName: kibana-svc 18 | servicePort: 5601 19 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/kibana/kibana-service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: kibana-svc 6 | namespace: elasticsearch 7 | labels: 8 | app: kibana 9 | role: kibana 10 | spec: 11 | type: NodePort 12 | sessionAffinity: ClientIP ##Ensure session won't be directed to another kibana node 13 | ports: 14 | - name: "ui" 15 | port: 5601 16 | targetPort: 5601 17 | selector: 18 | app: kibana 19 | role: kibana 20 | 21 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/kibana/kibana.yaml: -------------------------------------------------------------------------------- 1 | #ClusterIP 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: kibana 6 | namespace: elasticsearch 7 | labels: 8 | app: kibana 9 | role: kibana 10 | spec: 11 | sessionAffinity: ClientIP 12 | ports: 13 | - name: kibana 14 | port: 5601 15 | targetPort: 5601 16 | selector: 17 | app: kibana 18 | role: kibana 19 | 20 | 21 | #Deployment For kibana- 3 Replicas 22 | --- 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: kibana 27 | namespace: elasticsearch 28 | labels: 29 | role: kibana 30 | app: kibana 31 | spec: 32 | replicas: 3 33 | selector: 34 | matchLabels: 35 | app: kibana 36 | role: kibana 37 | template: 38 | metadata: 39 | namespace: elasticsearch 40 | labels: 41 | app: kibana 42 | role: kibana 43 | spec: 44 | containers: 45 | - name: kibana 46 | image: docker.elastic.co/kibana/kibana:7.1.1 47 | ports: 48 | - containerPort: 5601 49 | env: #add env for automatic authentication with es 50 | - name: ELASTICSEARCH_PASSWORD 51 | valueFrom: 52 | configMapKeyRef: 53 | name: es-config 54 | key: ELASTIC_PASSWORD 55 | - name: ELASTICSEARCH_USERNAME 56 | valueFrom: 57 | configMapKeyRef: 58 | name: es-config 59 | key: ELASTIC_USERNAME 60 | volumeMounts: 61 | - name: config 62 | mountPath: /usr/share/kibana/kibana.yml 63 | subPath: kibana.yml 64 | restartPolicy: Always 65 | volumes: 66 | - name: config 67 | configMap: 68 | name: kibana 69 | -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/storageClass/ceph-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: ceph-secret 5 | namespace: kube-system 6 | data: 7 | key: 8 | type: kubernetes.io/rbd -------------------------------------------------------------------------------- /ElasticStack-Kubernetes/storageClass/storageClass-ceph.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1beta1 2 | kind: StorageClass 3 | metadata: 4 | name: dynamic 5 | annotations: 6 | storageclass.beta.kubernetes.io/is-default-class: "true" 7 | provisioner: kubernetes.io/rbd 8 | parameters: 9 | monitors: :6789 10 | adminId: admin 11 | adminSecretName: ceph-secret 12 | adminSecretNamespace: kube-system 13 | pool: 14 | userId: admin 15 | userSecretName: ceph-secret -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ElasticStack 2 | 3 | ### In This Repo, you will have three different Platform to deploy Elastic Stack at your environment 4 | 5 | * The first is deploying Elastic stack on VMS. 6 | The deployment process is done by Ansible playbooks. 7 | 8 | * The second is deploying Elastic Stack on Docker Swarm. 9 | 10 | * The third is deploying Elastic Stack on Kubernetes. 11 | 12 | ### Each option has is own description and deployment process in his folder. 13 | 14 | > I will happy to get feedback!. 15 | 16 | 17 | ![Elastic Image](https://www.itopstimes.com/wp-content/uploads/2019/01/2b6024713b7ace798502139fd5ab8fe4-490x208.png) 18 | -------------------------------------------------------------------------------- /elastic_stack.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OryanOmer/ElasticStack/682775964ac43d9baa3152a3bfad6fe7ed25feda/elastic_stack.PNG --------------------------------------------------------------------------------