├── .github └── dependabot.yml ├── .gitignore ├── LICENSE ├── img ├── heap-usage-annotated.png ├── heap-usage.png └── stacktraces.png ├── java ├── .env ├── backend │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── net │ │ │ └── xeraa │ │ │ └── backend │ │ │ ├── Address.java │ │ │ ├── BackendApplication.java │ │ │ ├── BackendController.java │ │ │ ├── CsvReader.java │ │ │ ├── GeoPoint.java │ │ │ ├── Marketing.java │ │ │ ├── Person.java │ │ │ ├── PersonGenerator.java │ │ │ └── PersonRepository.java │ │ └── resources │ │ ├── application.yml │ │ ├── logback-spring.xml │ │ └── prenoms.csv ├── build.gradle ├── docker-compose.yml ├── frontend │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── net │ │ │ └── xeraa │ │ │ └── frontend │ │ │ ├── FrontendApplication.java │ │ │ └── FrontendController.java │ │ └── resources │ │ ├── application.yml │ │ ├── logback-spring.xml │ │ ├── static │ │ ├── elastic-logo.svg │ │ ├── favicon.ico │ │ ├── index.html │ │ └── style.css │ │ └── templates │ │ ├── add.html │ │ ├── call.html │ │ ├── generate.html │ │ ├── good.html │ │ └── search.html ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew └── settings.gradle ├── lightsail ├── ansible.cfg ├── configure_all.yml ├── configure_backend.yml ├── deploy_backend.yml ├── deploy_frontend.yml ├── files │ ├── ab.sh │ ├── alerting_heapsize.json │ ├── david.pub │ └── security_role_dashboard.json ├── include_deploy_boot.yml ├── include_event.yml ├── inventory ├── restart_frontend.yml ├── templates │ ├── auditbeat.yml │ ├── backend.conf │ ├── filebeat.yml │ ├── frontend.conf │ ├── heartbeat.yml │ ├── metricbeat.yml │ ├── nginx.conf │ ├── packetbeat.yml │ ├── security_user_dashboard.json │ ├── tls.conf │ └── urls.txt ├── terraform.tf ├── variables.tf └── variables.yml ├── local ├── .env ├── config │ ├── apm-server.yml │ ├── filebeat.yml │ ├── heartbeat.yml │ ├── metricbeat.yml │ └── packetbeat.yml ├── docker-compose.yml └── scripts │ ├── apm.sh │ ├── kibana.sh │ └── wait-for ├── readme.md └── workshop ├── ansible.cfg ├── configure_all.yml ├── configure_php.yml ├── deploy_backend.yml ├── deploy_bad.yml ├── deploy_frontend.yml ├── files ├── ab.sh ├── alerting_heapsize.json └── mysite │ ├── _config │ ├── logging.yml │ └── routes.yml │ └── code │ └── controllers │ └── ErrorController.php ├── include_deploy_boot.yml ├── include_event.yml ├── inventory ├── templates ├── apm-server.yml ├── auditbeat.yml ├── backend.conf ├── env ├── filebeat.yml ├── frontend.conf ├── heartbeat.yml ├── metricbeat.yml ├── mysite.yml ├── nginx.conf ├── osquery.conf ├── packetbeat.yml ├── secret.txt └── urls.txt ├── terraform.tf ├── variables.tf └── variables.yml /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gradle 4 | directory: "/java" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle/ 2 | .idea/ 3 | .terraform/ 4 | build/ 5 | out/ 6 | 7 | *.backup 8 | *.iml 9 | *.log 10 | *.retry 11 | *.tfstate 12 | *.tfstate.lock.info 13 | 14 | /java/*end.json 15 | /java/*end/*.json 16 | /java/**/*.gz 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Philipp Krenn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /img/heap-usage-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xeraa/microservice-monitoring/3e5e914bde83b8ff55773f83b2b1de3c6b0ab82f/img/heap-usage-annotated.png -------------------------------------------------------------------------------- /img/heap-usage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xeraa/microservice-monitoring/3e5e914bde83b8ff55773f83b2b1de3c6b0ab82f/img/heap-usage.png -------------------------------------------------------------------------------- /img/stacktraces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xeraa/microservice-monitoring/3e5e914bde83b8ff55773f83b2b1de3c6b0ab82f/img/stacktraces.png -------------------------------------------------------------------------------- /java/.env: -------------------------------------------------------------------------------- 1 | ELASTIC_PASSWORD=changeme 2 | -------------------------------------------------------------------------------- /java/backend/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'org.springframework.boot' 2 | apply plugin: 'io.spring.dependency-management' 3 | 4 | 5 | jar { 6 | baseName = 'backend' 7 | version = '1.0' 8 | } 9 | 10 | 11 | dependencies { 12 | compile( 13 | 'org.springframework.boot:spring-boot-starter-web', 14 | "co.elastic.logging:logback-ecs-encoder:${javaEcsLoggingVersion}", 15 | 'org.springframework.boot:spring-boot-starter-actuator', 16 | 'org.jolokia:jolokia-core', 17 | 'mysql:mysql-connector-java', // MySQL Connector-J 18 | 'org.springframework.boot:spring-boot-starter-data-jpa'// JPA Data (repositories, entities, Hibernate,...) 19 | ) 20 | } 21 | 22 | 23 | bootJar { 24 | launchScript() 25 | } 26 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/Address.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import javax.persistence.Embedded; 4 | import javax.persistence.Entity; 5 | import javax.persistence.GeneratedValue; 6 | import javax.persistence.GenerationType; 7 | import javax.persistence.Id; 8 | 9 | @Entity 10 | public class Address { 11 | @Id 12 | @GeneratedValue(strategy= GenerationType.AUTO) 13 | private Integer id; 14 | 15 | private String country; 16 | private String zipcode; 17 | private String city; 18 | private String countrycode; 19 | private GeoPoint location; 20 | 21 | public Integer getId() { 22 | return id; 23 | } 24 | 25 | public void setId(Integer id) { 26 | this.id = id; 27 | } 28 | 29 | public String getCountry() { 30 | return country; 31 | } 32 | 33 | public void setCountry(String country) { 34 | this.country = country; 35 | } 36 | 37 | public String getZipcode() { 38 | return zipcode; 39 | } 40 | 41 | public void setZipcode(String zipcode) { 42 | this.zipcode = zipcode; 43 | } 44 | 45 | public String getCity() { 46 | return city; 47 | } 48 | 49 | public void setCity(String city) { 50 | this.city = city; 51 | } 52 | 53 | public String getCountrycode() { 54 | return countrycode; 55 | } 56 | 57 | public void setCountrycode(String countrycode) { 58 | this.countrycode = countrycode; 59 | } 60 | 61 | @Embedded 62 | public GeoPoint getLocation() { 63 | return location; 64 | } 65 | 66 | public void setLocation(GeoPoint location) { 67 | this.location = location; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/BackendApplication.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.web.client.RestTemplate; 7 | 8 | @SpringBootApplication 9 | public class BackendApplication { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(BackendApplication.class, args); 13 | } 14 | 15 | @Bean 16 | public RestTemplate getRestTemplate() { 17 | return new RestTemplate(); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/BackendController.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.apache.logging.log4j.util.Strings; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.scheduling.annotation.Async; 9 | import org.springframework.ui.Model; 10 | import org.springframework.web.bind.annotation.GetMapping; 11 | import org.springframework.web.bind.annotation.RequestMapping; 12 | import org.springframework.web.bind.annotation.RequestParam; 13 | import org.springframework.web.bind.annotation.ResponseBody; 14 | import org.springframework.web.bind.annotation.RestController; 15 | import org.springframework.web.client.RestTemplate; 16 | 17 | import java.io.IOException; 18 | import java.util.List; 19 | import java.util.Random; 20 | 21 | @RestController 22 | public class BackendController { 23 | 24 | private static final Logger log = LoggerFactory.getLogger(BackendController.class); 25 | 26 | @Autowired 27 | private RestTemplate restTemplate; 28 | 29 | private Random random = new Random(); 30 | 31 | @Value("${APP_BACKEND:#{'http://localhost:8081'}}") 32 | private String backendUrl; 33 | 34 | @Value("${APP_FRONTEND:#{'http://localhost:8080'}}") 35 | private String frontendUrl; 36 | 37 | @RequestMapping("/slow") 38 | public String home() throws InterruptedException { 39 | final String callUrl = backendUrl + "/slow-call"; 40 | String returnValue = this.backgroundTask1(); 41 | returnValue += this.backgroundTask2(); 42 | returnValue += restTemplate.getForObject(callUrl, String.class); 43 | log.info("You called something slow"); 44 | return returnValue + "slow..."; 45 | } 46 | 47 | @Async 48 | public String backgroundTask1() throws InterruptedException { 49 | int millis = this.random.nextInt(1000); 50 | Thread.sleep(millis); 51 | //this.tracer.addTag("background-sleep-millis", String.valueOf(millis)); 52 | log.info("Background task ran with a delay of {} ms", millis); 53 | return "This "; 54 | } 55 | 56 | public String backgroundTask2() throws InterruptedException { 57 | int millis = this.random.nextInt(1000); 58 | Thread.sleep(millis); 59 | //this.tracer.addTag("background-sleep-millis", String.valueOf(millis)); 60 | log.info("Background task ran with a delay of {} ms", millis); 61 | return "is "; 62 | } 63 | 64 | @RequestMapping("/slow-call") 65 | public String call() throws InterruptedException { 66 | log.info("Calling another method from /slow"); 67 | return "so "; 68 | } 69 | 70 | @Autowired // This means to get the bean called userRepository 71 | // Which is auto-generated by Spring, we will use it to handle the data 72 | private PersonRepository personRepository; 73 | 74 | @GetMapping(path="/add") // Map ONLY GET Requests 75 | public @ResponseBody 76 | Person addNewPerson (@RequestParam(value="name", required=false, defaultValue="") String name) throws IOException { 77 | Person person = PersonGenerator.personGenerator(); 78 | if (Strings.isNotEmpty(name)) { 79 | person.setName(name); 80 | } 81 | return personRepository.save(person); 82 | } 83 | 84 | @GetMapping(path="/all") 85 | public @ResponseBody Iterable getAllUsers() { 86 | // This returns a JSON or XML with the users 87 | return personRepository.findAll(); 88 | } 89 | 90 | @GetMapping("/search") 91 | public @ResponseBody Iterable search(@RequestParam String q) { 92 | // This returns a JSON or XML with the users 93 | return personRepository.findByNameLike("%" + q + "%"); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/CsvReader.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | 4 | import java.io.BufferedReader; 5 | import java.io.FileNotFoundException; 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.io.InputStreamReader; 9 | import java.util.ArrayList; 10 | 11 | public class CsvReader { 12 | public static ArrayList readAsStrings(String url) throws FileNotFoundException, IOException { 13 | /**returns all of the data in a file as Strings given the File object*/ 14 | ArrayList data = new ArrayList(); 15 | InputStream ips= CsvReader.class.getResourceAsStream(url); 16 | InputStreamReader ipsr = new InputStreamReader(ips); 17 | BufferedReader reader = new BufferedReader(ipsr); 18 | String nextLine = reader.readLine(); 19 | while (nextLine != null) { 20 | data.add(nextLine); 21 | nextLine = reader.readLine(); 22 | } 23 | reader.close();//just a good idea aparently 24 | 25 | return data; 26 | } 27 | 28 | public static ArrayList extractFromCommas(String dataLine) { 29 | //Gives back the data that is found between commas in a String 30 | ArrayList data = new ArrayList(); 31 | String theString = ""; 32 | for (int i = 0; i < dataLine.length(); i++) { //go down the whole string 33 | if (dataLine.charAt(i) == ',') { 34 | if (i == 0) { 35 | //do nothing 36 | } else { 37 | data.add(theString); //this means that the next comma has been reached 38 | theString = ""; //reset theString Variable 39 | } 40 | } else { 41 | theString = theString + dataLine.charAt(i); //otherwise, just keep piling the chars onto the cumulative string 42 | } 43 | } 44 | if (!theString.equalsIgnoreCase("")) //only if the last position is not occupied with nothing then add the end on 45 | { 46 | data.add(theString); 47 | } 48 | return data; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/GeoPoint.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import javax.persistence.Embeddable; 4 | 5 | @Embeddable 6 | public class GeoPoint { 7 | private double lat; 8 | private double lon; 9 | 10 | public GeoPoint() { 11 | } 12 | 13 | public GeoPoint(double lat, double lon) { 14 | this.lat = lat; 15 | this.lon = lon; 16 | } 17 | 18 | public void setLat(double lat) { 19 | this.lat = lat; 20 | } 21 | 22 | public void setLon(double lon) { 23 | this.lon = lon; 24 | } 25 | 26 | public final double getLat() { 27 | return this.lat; 28 | } 29 | 30 | public final double getLon() { 31 | return this.lon; 32 | } 33 | 34 | public String toString() { 35 | return "[" + lat + ", " + lon + "]"; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/Marketing.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | 5 | import javax.persistence.Entity; 6 | import javax.persistence.GeneratedValue; 7 | import javax.persistence.GenerationType; 8 | import javax.persistence.Id; 9 | 10 | /** 11 | * We define here marketing meta data: 12 | * Number of clicks on each segment 13 | */ 14 | @Entity 15 | public class Marketing { 16 | private Integer id = null; 17 | 18 | private Integer cars; 19 | private Integer shoes; 20 | private Integer toys; 21 | private Integer fashion; 22 | private Integer music; 23 | private Integer garden; 24 | private Integer electronic; 25 | private Integer hifi; 26 | private Integer food; 27 | 28 | /** 29 | * Gets id (primary key). 30 | */ 31 | @Id 32 | @GeneratedValue(strategy= GenerationType.AUTO) 33 | public Integer getId() { 34 | return id; 35 | } 36 | 37 | /** 38 | * Sets id (primary key). 39 | */ 40 | public void setId(Integer id) { 41 | this.id = id; 42 | } 43 | 44 | public Integer getCars() { 45 | return cars; 46 | } 47 | 48 | public void setCars(Integer cars) { 49 | this.cars = cars; 50 | } 51 | 52 | public Integer getShoes() { 53 | return shoes; 54 | } 55 | 56 | public void setShoes(Integer shoes) { 57 | this.shoes = shoes; 58 | } 59 | 60 | public Integer getToys() { 61 | return toys; 62 | } 63 | 64 | public void setToys(Integer toys) { 65 | this.toys = toys; 66 | } 67 | 68 | public Integer getFashion() { 69 | return fashion; 70 | } 71 | 72 | public void setFashion(Integer fashion) { 73 | this.fashion = fashion; 74 | } 75 | 76 | public Integer getMusic() { 77 | return music; 78 | } 79 | 80 | public void setMusic(Integer music) { 81 | this.music = music; 82 | } 83 | 84 | public Integer getGarden() { 85 | return garden; 86 | } 87 | 88 | public void setGarden(Integer garden) { 89 | this.garden = garden; 90 | } 91 | 92 | public Integer getElectronic() { 93 | return electronic; 94 | } 95 | 96 | public void setElectronic(Integer electronic) { 97 | this.electronic = electronic; 98 | } 99 | 100 | public Integer getHifi() { 101 | return hifi; 102 | } 103 | 104 | public void setHifi(Integer hifi) { 105 | this.hifi = hifi; 106 | } 107 | 108 | public Integer getFood() { 109 | return food; 110 | } 111 | 112 | public void setFood(Integer food) { 113 | this.food = food; 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/Person.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import javax.persistence.CascadeType; 4 | import javax.persistence.Entity; 5 | import javax.persistence.GeneratedValue; 6 | import javax.persistence.GenerationType; 7 | import javax.persistence.Id; 8 | import javax.persistence.OneToOne; 9 | import java.util.Date; 10 | 11 | @Entity 12 | public class Person { 13 | 14 | @Id 15 | @GeneratedValue(strategy= GenerationType.AUTO) 16 | private Integer id; 17 | 18 | private String name = null; 19 | private Date dateOfBirth = null; 20 | private String gender = null; 21 | private Integer children; 22 | 23 | @OneToOne(cascade = CascadeType.ALL) 24 | private Marketing marketing; 25 | @OneToOne(cascade = CascadeType.ALL) 26 | private Address address; 27 | 28 | public Integer getId() { 29 | return id; 30 | } 31 | 32 | public void setId(Integer id) { 33 | this.id = id; 34 | } 35 | 36 | public String getName() { 37 | return name; 38 | } 39 | 40 | public void setName(String name) { 41 | this.name = name; 42 | } 43 | 44 | public Date getDateOfBirth() { 45 | return dateOfBirth; 46 | } 47 | 48 | public void setDateOfBirth(Date dateOfBirth) { 49 | this.dateOfBirth = dateOfBirth; 50 | } 51 | 52 | public String getGender() { 53 | return gender; 54 | } 55 | 56 | public void setGender(String gender) { 57 | this.gender = gender; 58 | } 59 | 60 | public Marketing getMarketing() { 61 | return marketing; 62 | } 63 | 64 | public void setMarketing(Marketing marketing) { 65 | this.marketing = marketing; 66 | } 67 | 68 | public Address getAddress() { 69 | return address; 70 | } 71 | 72 | public void setAddress(Address address) { 73 | this.address = address; 74 | } 75 | 76 | public Integer getChildren() { 77 | return children; 78 | } 79 | 80 | public void setChildren(Integer children) { 81 | this.children = children; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/PersonGenerator.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import java.io.IOException; 4 | import java.time.LocalDate; 5 | import java.time.Period; 6 | import java.time.ZoneId; 7 | import java.util.ArrayList; 8 | import java.util.Date; 9 | import java.util.Random; 10 | 11 | public class PersonGenerator { 12 | 13 | private static Random random = new Random(); 14 | 15 | public static ArrayList names; 16 | 17 | static { 18 | try { 19 | PersonGenerator.names = CsvReader.readAsStrings("/prenoms.csv"); 20 | } catch (IOException e) { 21 | System.err.println("Can not generate names from CSV"); 22 | } 23 | } 24 | 25 | public static Person personGenerator() throws IOException { 26 | Person person = new Person(); 27 | buildGender(person); 28 | person.setDateOfBirth(buildBirthDate()); 29 | person.setMarketing(buildMeta()); 30 | person.setAddress(buildAddress()); 31 | person.setChildren(buildChildren()); 32 | 33 | return person; 34 | } 35 | 36 | private static Marketing buildMeta() { 37 | Marketing marketing = new Marketing(); 38 | int nbMeta = numberGenerator(1, 5); 39 | 40 | for (int i = 0; i < nbMeta; i++) { 41 | int nbConsult = numberGenerator(30, 2000); 42 | int typeMeta = numberGenerator(0, 9); 43 | switch (typeMeta) { 44 | case 0: 45 | marketing.setShoes(nbConsult); 46 | break; 47 | case 1: 48 | marketing.setToys(nbConsult); 49 | break; 50 | case 2: 51 | marketing.setFashion(nbConsult); 52 | break; 53 | case 3: 54 | marketing.setMusic(nbConsult); 55 | break; 56 | case 4: 57 | marketing.setGarden(nbConsult); 58 | break; 59 | case 5: 60 | marketing.setElectronic(nbConsult); 61 | break; 62 | case 6: 63 | marketing.setHifi(nbConsult); 64 | break; 65 | case 7: 66 | marketing.setCars(nbConsult); 67 | break; 68 | case 8: 69 | marketing.setFood(nbConsult); 70 | break; 71 | default: 72 | System.err.println(" ->" + typeMeta); 73 | break; 74 | } 75 | 76 | 77 | } 78 | 79 | return marketing; 80 | 81 | } 82 | 83 | private static Date buildBirthDate() { 84 | return Date.from( 85 | LocalDate.now().minus(Period.ofDays(random.nextInt(365*100))) // Up to 100 years ago 86 | .atStartOfDay(ZoneId.systemDefault()).toInstant()); 87 | } 88 | 89 | private static void buildGender(Person person) throws IOException { 90 | int pos = numberGenerator(0, names.size()); 91 | 92 | String line = names.get(pos); 93 | ArrayList temp = CsvReader.extractFromCommas(line); 94 | person.setName(temp.get(0) + " " + CsvReader.extractFromCommas( 95 | names.get(numberGenerator(0, names.size()))).get(0)); 96 | person.setGender(temp.get(1)); 97 | } 98 | 99 | private static Address buildAddress() throws IOException { 100 | Address address = new Address(); 101 | generateCountry(address); 102 | Long result = Math.round(Math.random() * 2); 103 | 104 | if ("FR".equals(address.getCountrycode())) { 105 | switch (result.intValue()) { 106 | case 0: 107 | address.setCity("Paris"); 108 | address.setZipcode("75000"); 109 | address.setLocation(new GeoPoint(doubleGenerator(48.819918, 48.900552), doubleGenerator(2.25929, 2.4158559))); 110 | break; 111 | case 1: 112 | address.setCity("Nantes"); 113 | address.setZipcode("44000"); 114 | address.setLocation(new GeoPoint(doubleGenerator(47.157742, 47.270729), doubleGenerator(-1.623467, -1.471032))); 115 | break; 116 | case 2: 117 | address.setCity("Cergy"); 118 | address.setZipcode("95000"); 119 | address.setLocation(new GeoPoint(doubleGenerator(49.019583, 49.059419), doubleGenerator(2.003001, 2.090892))); 120 | break; 121 | default: 122 | System.err.println("buildAddress ->" + result.intValue()); 123 | break; 124 | } 125 | } 126 | 127 | if ("GB".equals(address.getCountrycode())) { 128 | switch (result.intValue()) { 129 | case 0: 130 | address.setCity("London"); 131 | address.setZipcode("98888"); 132 | address.setLocation(new GeoPoint(doubleGenerator(51.444014, 51.607633), doubleGenerator(-0.294245, 0.064184))); 133 | break; 134 | case 1: 135 | address.setCity("Plymouth"); 136 | address.setZipcode("5226"); 137 | address.setLocation(new GeoPoint(doubleGenerator(50.345272, 50.434797), doubleGenerator(-4.190161, -4.034636))); 138 | break; 139 | case 2: 140 | address.setCity("Liverpool"); 141 | address.setZipcode("86767"); 142 | address.setLocation(new GeoPoint(doubleGenerator(53.345346, 53.496339), doubleGenerator(-3.047485, -2.564774))); 143 | break; 144 | default: 145 | System.err.println("buildAddress ->" + result.intValue()); 146 | break; 147 | } 148 | } 149 | 150 | if ("DE".equals(address.getCountrycode())) { 151 | switch (result.intValue()) { 152 | case 0: 153 | address.setCity("Berlin"); 154 | address.setZipcode("9998"); 155 | address.setLocation(new GeoPoint(doubleGenerator(52.364796, 52.639827), doubleGenerator(13.115778, 13.769465))); 156 | break; 157 | case 1: 158 | address.setCity("Bonn"); 159 | address.setZipcode("0099"); 160 | address.setLocation(new GeoPoint(doubleGenerator(50.649948, 50.766049), doubleGenerator(7.025075, 7.214589))); 161 | break; 162 | case 2: 163 | address.setCity("Munich"); 164 | address.setZipcode("45445"); 165 | address.setLocation(new GeoPoint(doubleGenerator(48.081337, 48.238441), doubleGenerator(11.371548, 11.711437))); 166 | break; 167 | default: 168 | System.err.println("buildAddress ->" + result.intValue()); 169 | break; 170 | } 171 | } 172 | 173 | if ("IT".equals(address.getCountrycode())) { 174 | switch (result.intValue()) { 175 | case 0: 176 | address.setCity("Rome"); 177 | address.setZipcode("00100"); 178 | address.setLocation(new GeoPoint(doubleGenerator(41.797211, 41.980805), doubleGenerator(12.373950, 12.601393))); 179 | break; 180 | case 1: 181 | address.setCity("Turin"); 182 | address.setZipcode("10100"); 183 | address.setLocation(new GeoPoint(doubleGenerator(45.007912, 45.122125), doubleGenerator(7.593528, 7.747337))); 184 | break; 185 | case 2: 186 | address.setCity("Ischia"); 187 | address.setZipcode("80100"); 188 | address.setLocation(new GeoPoint(doubleGenerator(40.704982, 40.758477), doubleGenerator(13.859360, 13.953002))); 189 | break; 190 | default: 191 | System.err.println("buildAddress ->" + result.intValue()); 192 | break; 193 | } 194 | } 195 | 196 | return address; 197 | } 198 | 199 | private static String generateCountry(Address address) { 200 | 201 | int result = numberGenerator(0,4); 202 | 203 | switch (result) { 204 | case 0: 205 | address.setCountry("France"); 206 | address.setCountrycode("FR"); 207 | break; 208 | case 1: 209 | address.setCountry("Germany"); 210 | address.setCountrycode("DE"); 211 | break; 212 | case 2: 213 | address.setCountry("England"); 214 | address.setCountrycode("GB"); 215 | break; 216 | case 3: 217 | address.setCountry("Italy"); 218 | address.setCountrycode("IT"); 219 | break; 220 | default: 221 | System.err.println("generateCountry ->" + result); 222 | break; 223 | } 224 | 225 | return null; 226 | } 227 | 228 | private static Integer buildChildren() { 229 | return numberGenerator(0,5); 230 | } 231 | 232 | private static int numberGenerator(int min, int range) { 233 | return (int) Math.floor(Math.random()*range+min); 234 | } 235 | 236 | private static double doubleGenerator(double min, double max) { 237 | return min + (max - min) * new Random().nextDouble(); 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /java/backend/src/main/java/net/xeraa/backend/PersonRepository.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.backend; 2 | 3 | import org.springframework.data.repository.CrudRepository; 4 | 5 | public interface PersonRepository extends CrudRepository { 6 | 7 | Iterable findByNameLike(String name); 8 | } 9 | -------------------------------------------------------------------------------- /java/backend/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | # Give the project a nice name 2 | spring.application.name: backend 3 | 4 | # Set the port 5 | server.port: ${SERVER_PORT:8081} 6 | 7 | # Enable all the actuator endpoints for HTTP (keep them under the base path) and JMX 8 | management.endpoints: 9 | web: 10 | base-path: / 11 | exposure.include: "*" 12 | jmx.exposure.include: "*" 13 | 14 | spring.jpa.hibernate.ddl-auto: create 15 | spring.datasource.url: jdbc:mysql://${DATABASE_SERVER:localhost}:${DATABASE_PORT:3306}/${DATABASE_NAME:person}?serverTimezone=UTC 16 | spring.datasource.username: ${DATABASE_USERNAME:root} 17 | spring.datasource.password: ${DATABASE_PASSWORD:changeme} 18 | -------------------------------------------------------------------------------- /java/backend/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ​ 5 | 6 | ​ 7 | 8 | 9 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | 19 | ${CONSOLE_LOG_PATTERN} 20 | utf8 21 | 22 | 23 | 24 | 25 | 26 | ${LOG_PATH:-.}/${LOG_FILE}.log 27 | 28 | ${LOG_FILE}.log.%d{yyyy-MM-dd}.gz 29 | 7 30 | 31 | 32 | ${CONSOLE_LOG_PATTERN} 33 | utf8 34 | 35 | 36 | ​ 37 | 38 | 39 | ${LOG_PATH:-.}/${LOG_FILE}.json 40 | 41 | ${LOG_FILE}.json.%d{yyyy-MM-dd}.gz 42 | 7 43 | 44 | 45 | ${springAppName:-} 46 | 47 | 48 | ​ 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /java/backend/src/main/resources/prenoms.csv: -------------------------------------------------------------------------------- 1 | Gabriel,male 2 | Arthur,male 3 | Louise,female 4 | Raphael,male 5 | Adam,male 6 | Chloe,female 7 | Paul,male 8 | Alexandre,male 9 | Louis,male 10 | Emma,female 11 | Antoine,male 12 | Maxime,male 13 | Alice,female 14 | Ines,female 15 | Sarah,female 16 | Jeanne,female 17 | Lucas,male 18 | Victor,male 19 | Mohamed,male 20 | Camille,female 21 | Juliette,female 22 | Lea,female 23 | Nathan,male 24 | Thomas,male 25 | Sacha,male 26 | Jules,male 27 | Eva,female 28 | Lina,female 29 | Hugo,male 30 | Manon,female 31 | Zoe,female 32 | Adrien,male 33 | Nina,female 34 | Jade,female 35 | Noah,male 36 | Anna,female 37 | Clement,male 38 | Gaspard,male 39 | Joseph,male 40 | Augustin,male 41 | Charlotte,female 42 | Anais,female 43 | Rayan,male 44 | Ethan,male 45 | Oscar,male 46 | Martin,male 47 | Samuel,male 48 | Yanis,male 49 | Baptiste,male 50 | Mathilde,female 51 | Romane,female 52 | Leo,male 53 | Clemence,female 54 | Enzo,male 55 | Lucie,female 56 | Rose,female 57 | Gabrielle,female 58 | Lou,female 59 | Simon,male 60 | Noe,male 61 | Clara,female 62 | Adele,female 63 | Sofia,female 64 | Maxence,male 65 | Marie,female 66 | Axel,male 67 | Lisa,female 68 | Victoire,female 69 | Josephine,female 70 | Theo,male 71 | Valentin,male 72 | Alexis,male 73 | Leonie,female 74 | Heloise,female 75 | Julia,female 76 | Ava,female 77 | Lola,female 78 | Liam,male 79 | Margaux,female 80 | Maya,female 81 | Quentin,male 82 | Valentine,female 83 | Charles,male 84 | Come,male 85 | Agathe,female 86 | Pierre,male 87 | Yasmine,female 88 | Aaron,male 89 | Alix,female 90 | Elisa,female 91 | Diane,female 92 | Mathis,male 93 | Mila,female 94 | Victoria,female 95 | Jean,male 96 | Apolline,female 97 | Benjamin,male 98 | Noemie,female 99 | Elise,female 100 | Gabin,male 101 | Camille,male 102 | Ismael,male 103 | Lena,female 104 | Olivia,female 105 | Pauline,female 106 | Sara,female 107 | Timothee,male 108 | Elsa,female 109 | Margot,female 110 | David,male 111 | Leon,male 112 | Capucine,female 113 | Tom,male 114 | Alicia,female 115 | Elias,male 116 | Salome,female 117 | Eliott,male 118 | Lucien,male 119 | Mael,male 120 | Romain,male 121 | Isaac,male 122 | Noam,male 123 | Emilie,female 124 | Robin,male 125 | Youssef,male 126 | Aya,female 127 | Ibrahim,male 128 | Nael,male 129 | Felix,male 130 | Laura,female 131 | Nour,female 132 | Tristan,male 133 | Iris,female 134 | Matthieu,male 135 | Ruben,male 136 | Joshua,male 137 | Mariam,female 138 | Esther,female 139 | Fatoumata,female 140 | Lily,female 141 | Romy,female 142 | Violette,female 143 | William,male 144 | Garance,female 145 | Julie,female 146 | Nicolas,male 147 | Ulysse,male 148 | Octave,male 149 | Constance,female 150 | Eleonore,female 151 | Amine,male 152 | Leonard,male 153 | Luna,female 154 | Suzanne,female 155 | Daniel,male 156 | Daphne,female 157 | Julien,male 158 | Stella,female 159 | Ambre,female 160 | Evan,male 161 | Malo,male 162 | Marius,male 163 | Nolan,male 164 | Roman,male 165 | Samy,male 166 | Faustine,female 167 | Gustave,male 168 | Mehdi,male 169 | Sophie,female 170 | Yacine,male 171 | Eden,female 172 | Gregoire,male 173 | Sasha,female 174 | Antonin,male 175 | Aurelien,male 176 | Ayoub,male 177 | Eloise,female 178 | Solal,male 179 | Achille,male 180 | Ines,female 181 | Oceane,female 182 | Sophia,female 183 | Anatole,male 184 | Edouard,male 185 | Elie,male 186 | Lila,female 187 | Mathias,male 188 | Rafael,male 189 | Emmanuel,male 190 | Justine,female 191 | Lenny,male 192 | Amir,male 193 | Hadrien,male 194 | Lena,female 195 | Lyna,female 196 | Milo,male 197 | Noham,male 198 | Theodore,male 199 | Aminata,female 200 | Armand,male 201 | Celeste,female 202 | Edgar,male 203 | Mathieu,male 204 | Noa,female 205 | Rayane,male 206 | Thibault,male 207 | Auguste,male 208 | Madeleine,female 209 | Matheo,male 210 | Ahmed,male 211 | Aicha,female 212 | Anis,male 213 | Assia,female 214 | Basile,male 215 | Marion,female 216 | Vadim,male 217 | Ali,male 218 | Esteban,male 219 | Manel,female 220 | Melissa,female 221 | Myriam,female 222 | Sixtine,female 223 | Vincent,male 224 | Lise,female 225 | Roxane,female 226 | Wassim,male 227 | Yassine,male 228 | Clementine,female 229 | Inaya,female 230 | Melina,female 231 | Mia,female 232 | Moussa,male 233 | Abel,male 234 | Anouk,female 235 | Cesar,male 236 | Emile,male 237 | Kenza,female 238 | Lilia,female 239 | Luca,male 240 | Maelys,female 241 | Celia,female 242 | Charlie,female 243 | Chiara,female 244 | Ella,female 245 | Eugenie,female 246 | Guillaume,male 247 | Leopold,male 248 | Theophile,male 249 | Titouan,male 250 | Arsene,male 251 | Diego,male 252 | Hector,male 253 | Ilyes,male 254 | Issa,male 255 | Joachim,male 256 | Lilou,female 257 | Max,male 258 | Paloma,female 259 | Zakaria,male 260 | Corentin,male 261 | Estelle,female 262 | Hanna,female 263 | Imane,female 264 | Jasmine,female 265 | Noa,male 266 | Sofiane,male 267 | Stanislas,male 268 | Zelie,female 269 | Alex,male 270 | Amaury,male 271 | Amelie,female 272 | Fatima,female 273 | Florian,male 274 | Lorenzo,male 275 | Louna,female 276 | Mamadou,male 277 | Marguerite,female 278 | Nathanael,male 279 | Adem,male 280 | Albane,female 281 | Alma,female 282 | Blanche,female 283 | Erwan,male 284 | Eve,female 285 | Marin,male 286 | Sami,male 287 | Younes,male 288 | Amina,female 289 | Berenice,female 290 | Hortense,female 291 | Isaure,female 292 | Jacques,male 293 | Maria,female 294 | Maryam,female 295 | Nassim,male 296 | Salma,female 297 | Selma,female 298 | Thais,female 299 | Hippolyte,male 300 | Livia,female 301 | Marc,male 302 | Naomi,female 303 | Ninon,female 304 | Alban,male 305 | Alexandra,female 306 | Awa,female 307 | Dina,female 308 | Djibril,male 309 | Hawa,female 310 | Ibrahima,male 311 | Ilian,male 312 | Kais,male 313 | Kylian,male 314 | Maelle,female 315 | Maissa,female 316 | Maximilien,male 317 | Nino,male 318 | Tess,female 319 | Amira,female 320 | Anaelle,female 321 | Dylan,male 322 | Elisabeth,female 323 | Elliot,male 324 | Leane,female 325 | Loic,male 326 | Marine,female 327 | Penelope,female 328 | Tiago,male 329 | Timeo,male 330 | Timothe,male 331 | Virgile,male 332 | Yann,male 333 | Alexia,female 334 | Anas,male 335 | Angele,female 336 | Anthony,male 337 | Bintou,female 338 | Charline,female 339 | Claire,female 340 | Colombe,female 341 | Farah,female 342 | Luc,male 343 | Matteo,male 344 | Milan,male 345 | Nora,female 346 | Oumou,female 347 | Ousmane,male 348 | Pablo,male 349 | Rachel,female 350 | Souleymane,male 351 | Andrea,male 352 | Celine,female 353 | Coline,female 354 | Dimitri,male 355 | Elena,female 356 | Eliot,male 357 | Giulia,female 358 | Hannah,female 359 | Henri,male 360 | Khadija,female 361 | Leandre,male 362 | Lucile,female 363 | Marcus,male 364 | Matthias,male 365 | Romeo,male 366 | Sirine,female 367 | Ana,female 368 | Aurore,female 369 | Axelle,female 370 | Charlie,male 371 | Kevin,male 372 | Lassana,male 373 | Marwa,female 374 | Mathys,male 375 | Mohammed,male 376 | Nahel,male 377 | Rebecca,female 378 | Aymen,male 379 | Brune,female 380 | Cleo,female 381 | Dorian,male 382 | Elina,female 383 | Emy,female 384 | Ethel,female 385 | Fanta,female 386 | Francois,male 387 | Gael,male 388 | Helene,female 389 | Jonas,male 390 | Louane,female 391 | Maia,female 392 | Marceau,male 393 | Matteo,male 394 | Morgane,female 395 | Rafael,male 396 | Zacharie,male 397 | Abdoulaye,male 398 | Amandine,female 399 | Aurele,male 400 | Bianca,female 401 | Eden,male 402 | Gaston,male 403 | Hamza,male 404 | Hana,female 405 | Kenzo,male 406 | Mina,female 407 | Nael,male 408 | Nils,male 409 | Olivier,male 410 | Raphael,male 411 | Raphaelle,female 412 | Sasha,male 413 | Syrine,female 414 | Alienor,female 415 | Alya,female 416 | Amadou,male 417 | Ambroise,male 418 | Ariane,female 419 | Camelia,female 420 | Cassandre,female 421 | Clotilde,female 422 | Emmy,female 423 | Eric,male 424 | Etienne,male 425 | Ilyas,male 426 | Ismail,male 427 | Judith,female 428 | Justin,male 429 | Karim,male 430 | Lana,female 431 | Lara,female 432 | Lison,female 433 | Louisa,female 434 | Neil,male 435 | Omar,male 436 | Philippine,female 437 | Remi,male 438 | Ryan,male 439 | Solene,female 440 | Aliya,female 441 | Andre,male 442 | Anton,male 443 | Astrid,female 444 | Aymeric,male 445 | Damien,male 446 | eleonore,female 447 | Elliott,male 448 | Elya,female 449 | Flora,female 450 | Jenna,female 451 | Jessica,female 452 | Johan,male 453 | Julian,male 454 | Kelly,female 455 | Laure,female 456 | Luka,male 457 | Maimouna,female 458 | Mariama,female 459 | Naim,male 460 | Owen,male 461 | Sandro,male 462 | Wael,male 463 | Walid,male 464 | Aissatou,female 465 | Aksel,male 466 | Balthazar,male 467 | Caroline,female 468 | Elea,female 469 | Elia,female 470 | Ewen,male 471 | Gauthier,male 472 | Idriss,male 473 | James,male 474 | Leonore,female 475 | Louison,female 476 | Maxine,female 477 | Michel,male 478 | Oumar,male 479 | Tessa,female 480 | Adama,male 481 | Aissata,female 482 | Ashley,female 483 | Assa,female 484 | Audrey,female 485 | Bastien,male 486 | Bilel,male 487 | Colette,female 488 | Constantin,male 489 | Elyes,male 490 | Hanae,female 491 | Leonardo,male 492 | Maeva,female 493 | Mayeul,male 494 | Mya,female 495 | Nayla,female 496 | Norah,female 497 | Rania,female 498 | Sam,male 499 | Shirel,female 500 | Sibylle,female 501 | Tara,female 502 | Wael,male 503 | Yohan,male 504 | Youssouf,male 505 | Adel,male 506 | Alan,male 507 | Anae,female 508 | Anastasia,female 509 | Augustine,female 510 | Calixte,male 511 | Coumba,female 512 | Diana,female 513 | Domitille,female 514 | Eloi,male 515 | Ernest,male 516 | Ewan,male 517 | Ferdinand,male 518 | Ilan,male 519 | Ivan,male 520 | Jana,female 521 | Jonathan,male 522 | Lili,female 523 | Loane,female 524 | Ludivine,female 525 | Melissa,female 526 | Nine,female 527 | Philomene,female 528 | Robinson,male 529 | Rosalie,female 530 | Sonia,female 531 | Thelma,female 532 | Alia,female 533 | Angelina,female 534 | Boubacar,male 535 | Bryan,male 536 | Christian,male 537 | Christophe,male 538 | Clarisse,female 539 | Enora,female 540 | Fares,male 541 | Georges,male 542 | Jad,male 543 | Kamil,male 544 | Lilya,female 545 | Lino,male 546 | Marcel,male 547 | Mateo,male 548 | Mateo,male 549 | Maylis,female 550 | Melvil,male 551 | Mona,female 552 | Mouhamed,male 553 | Nada,female 554 | Naelle,female 555 | Nell,female 556 | Niels,male 557 | Sacha,female 558 | Serena,female 559 | Soline,female 560 | Talia,female 561 | Wendy,female 562 | Yara,female 563 | Aliyah,female 564 | Alyssa,female 565 | Andrea,female 566 | Andy,male 567 | Anissa,female 568 | Bilal,male 569 | Carla,female 570 | Charly,male 571 | Chris,male 572 | Cleophee,female 573 | Colin,male 574 | Dan,male 575 | elea,female 576 | Elio,male 577 | Elouan,male 578 | Ema,female 579 | Emmanuelle,female 580 | Emna,female 581 | Eugene,male 582 | Felicie,female 583 | Fleur,female 584 | Florent,male 585 | Gaetan,male 586 | Isabelle,female 587 | Jeremy,male 588 | June,female 589 | Kevin,male 590 | Kiara,female 591 | Leila,female 592 | Lilas,female 593 | Lilian,male 594 | Louka,male 595 | Mahamadou,male 596 | Malak,female 597 | Malik,male 598 | Marilou,female 599 | Marwane,male 600 | Maud,female 601 | Maxim,male 602 | Melina,female 603 | Naomie,female 604 | Nawel,female 605 | Oren,male 606 | Rita,female 607 | Sekou,male 608 | Shana,female 609 | Soren,male 610 | Valentina,female 611 | Zachary,male 612 | Abdoul,male 613 | Aboubacar,male 614 | Alessio,male 615 | Amelia,female 616 | Andreas,male 617 | Anouck,female 618 | Asma,female 619 | Aurelie,female 620 | Camil,male 621 | Dalia,female 622 | Darius,male 623 | Eyal,male 624 | Fanny,female 625 | Fatou,female 626 | Florence,female 627 | Gabriela,female 628 | Hassan,male 629 | Ilyana,female 630 | Imran,male 631 | Ismael,male 632 | Jeremie,male 633 | Jordan,male 634 | Kenan,male 635 | Lior,male 636 | Liv,female 637 | Loan,male 638 | Lukas,male 639 | Marianne,female 640 | Marley,male 641 | Meline,female 642 | Mellina,female 643 | Nahil,male 644 | Pia,female 645 | Prune,female 646 | Rami,male 647 | Tali,female 648 | Youcef,male 649 | Zadig,male 650 | Zahra,female 651 | Adriana,female 652 | Alistair,male 653 | Alix,male 654 | Amelia,female 655 | Armel,male 656 | Bertille,female 657 | Billie,female 658 | Chaima,female 659 | Cheick,male 660 | Clelia,female 661 | Cyprien,male 662 | elise,female 663 | eloise,female 664 | Elyas,male 665 | Emilia,female 666 | Emilien,male 667 | Eulalie,female 668 | Flore,female 669 | Gabriella,female 670 | Gloria,female 671 | Isis,female 672 | Iyad,male 673 | Lou,male 674 | Lya,female 675 | Lyam,male 676 | Lylia,female 677 | Mahaut,female 678 | Mariame,female 679 | Marwan,male 680 | Melchior,male 681 | Mickael,male 682 | Milena,female 683 | Morgan,male 684 | Naila,female 685 | Neila,female 686 | Nikita,male 687 | Ornella,female 688 | Philippe,male 689 | Rokia,female 690 | Selyan,male 691 | Sienna,female 692 | Teo,male 693 | Thibaut,male 694 | Younes,male 695 | Adrian,male 696 | Aida,female 697 | Aidan,male 698 | Alassane,male 699 | Albert,male 700 | Alessandro,male 701 | Aline,female 702 | Alisha,female 703 | Alois,male 704 | Alycia,female 705 | Amel,female 706 | Angelo,male 707 | Ariel,male 708 | Athenais,female 709 | Avril,female 710 | Benoit,male 711 | Blandine,female 712 | Cecile,female 713 | Celia,female 714 | Christopher,male 715 | Cloe,female 716 | Clovis,male 717 | Damian,male 718 | Dario,male 719 | elie,male 720 | Ellie,female 721 | Fatim,female 722 | Filip,male 723 | Gaetan,male 724 | Grace,female 725 | Honore,male 726 | Ilias,male 727 | Ilyan,male 728 | Ilyes,male 729 | Irene,female 730 | Issam,male 731 | Iyed,male 732 | Khadidja,female 733 | Kim,female 734 | Lahna,female 735 | Lamine,male 736 | Leandro,male 737 | Lia,female 738 | Line,female 739 | Lucia,female 740 | Lucy,female 741 | Lyes,male 742 | Maeva,female 743 | Mahault,female 744 | Mario,male 745 | Marvin,male 746 | Mathilda,female 747 | Mayssa,female 748 | Melvin,male 749 | Nesrine,female 750 | Nolann,male 751 | Nolhan,male 752 | Noor,female 753 | Nourane,female 754 | Roxanne,female 755 | Ryad,male 756 | Sabrina,female 757 | Sana,female 758 | Sean,male 759 | Simeon,male 760 | Tony,male 761 | Yael,female 762 | Yael,female 763 | Yoann,male 764 | Abraham,male 765 | Alyah,female 766 | Alyssia,female 767 | Amy,female 768 | Ania,female 769 | Anne,female 770 | Anselme,male 771 | Ari,male 772 | Arnaud,male 773 | Assya,female 774 | Bakary,male 775 | Cameron,male 776 | Candice,female 777 | Christine,female 778 | Clea,female 779 | Cyril,male 780 | Daria,female 781 | elisa,female 782 | Elly,female 783 | Elodie,female 784 | Emeline,female 785 | emile,male 786 | emilie,female 787 | Emily,female 788 | Emir,male 789 | eva,female 790 | Fares,male 791 | Flavie,female 792 | Gautier,male 793 | Guilhem,male 794 | Hafsa,female 795 | Hayden,male 796 | Hermine,female 797 | Ian,male 798 | Idrissa,male 799 | Imene,female 800 | Islam,male 801 | Janna,female 802 | Jean-Baptiste,male 803 | Jibril,male 804 | Joan,male 805 | Johanna,female 806 | Katia,female 807 | Kenny,male 808 | Kilian,male 809 | Laetitia,female 810 | Lazare,male 811 | Leina,female 812 | Leyna,female 813 | Lindsay,female 814 | Liza,female 815 | Luce,female 816 | Mael,male 817 | Mahe,male 818 | Malia,female 819 | Marco,male 820 | Marlon,male 821 | Matias,male 822 | Matis,male 823 | Meriem,female 824 | Merlin,male 825 | Milhan,male 826 | Moustapha,male 827 | Nelia,female 828 | Neyla,female 829 | Noan,male 830 | Noelie,female 831 | Paola,female 832 | Rahma,female 833 | Raoul,male 834 | Rodrigue,male 835 | Ronan,male 836 | Salimata,female 837 | Samba,male 838 | Sebastien,male 839 | Sidonie,female 840 | Silas,male 841 | Sohane,female 842 | Souleyman,male 843 | Stefan,male 844 | Tania,female 845 | Tasnim,female 846 | Tiffany,female 847 | Vanessa,female 848 | Vasco,male 849 | Viktor,male 850 | Yani,male 851 | Yannis,male 852 | Yse,female 853 | Zakariya,male 854 | Aleksandra,female 855 | Alessia,female 856 | Alfred,male 857 | Allan,male 858 | Amalia,female 859 | Amara,male 860 | Ambrine,female 861 | Andrea,female 862 | Angela,female 863 | Angelina,female 864 | Annaelle,female 865 | Anya,female 866 | Aristide,male 867 | Artus,male 868 | Aubin,male 869 | Auriane,female 870 | Aziz,male 871 | Ben,male 872 | Binta,female 873 | Celestin,male 874 | Celestine,female 875 | Celian,male 876 | Chahine,male 877 | Djeneba,female 878 | Elena,female 879 | Elior,male 880 | Elyne,female 881 | Erin,female 882 | Evann,male 883 | eve,female 884 | Ezio,male 885 | Fadi,male 886 | Harry,male 887 | Hedi,male 888 | Helena,female 889 | Henry,male 890 | Hiba,female 891 | Hind,female 892 | Imrane,male 893 | Jaden,male 894 | Jason,male 895 | Jeremy,male 896 | Joanna,female 897 | Kadiatou,female 898 | Karl,male 899 | Kayna,female 900 | Khalil,male 901 | Killian,male 902 | Laetitia,female 903 | Lauren,female 904 | Laurent,male 905 | Leana,female 906 | Leonard,male 907 | Linda,female 908 | Linoi,female 909 | Liora,female 910 | Lisandro,male 911 | Lital,female 912 | Lois,male 913 | Lou-Ann,female 914 | Louay,male 915 | Louca,male 916 | Lydia,female 917 | Lyla,female 918 | Mae,female 919 | Maelie,female 920 | Maelyne,female 921 | Maeva,female 922 | Mahmoud,male 923 | Mailys,female 924 | Maiwenn,female 925 | Manelle,female 926 | Marko,male 927 | Matilde,female 928 | Mattia,male 929 | Melody,female 930 | Mendel,male 931 | Meryam,female 932 | Mohamed-Amine,male 933 | Nabil,male 934 | Nadia,female 935 | Natalia,female 936 | Nelly,female 937 | Nicole,female 938 | Noura,female 939 | Oliver,male 940 | Olympe,female 941 | Pacome,male 942 | Pedro,male 943 | Ranya,female 944 | Safa,female 945 | Salim,male 946 | Samia,female 947 | Santiago,male 948 | Scarlett,female 949 | Selena,female 950 | Soumaya,female 951 | Tamara,female 952 | Thea,female 953 | Tiphaine,female 954 | Tomas,male 955 | Vianney,male 956 | Viviane,female 957 | Warren,male 958 | Willy,male 959 | Xavier,male 960 | Yohann,male 961 | Yossef,male 962 | Ysee,female 963 | Abdallah,male 964 | Abigail,female 965 | Aby,female 966 | Adame,male 967 | Adil,male 968 | Agnes,female 969 | Ahmad,male 970 | Aisha,female 971 | Alba,female 972 | Alexander,male 973 | Alima,female 974 | Alina,female 975 | Alissa,female 976 | Alizee,female 977 | Allegra,female 978 | Amanda,female 979 | Aris,male 980 | Arman,male 981 | Ayline,female 982 | Aymane,male 983 | Barbara,female 984 | Blessing,female 985 | Bonnie,female 986 | Brahim,male 987 | Brieuc,male 988 | Camilia,female 989 | Carl,male 990 | Cassandra,female 991 | Cassie,female 992 | Cecilia,female 993 | Chayma,female 994 | Clarence,male 995 | Clothilde,female 996 | Dani,male 997 | Daouda,male 998 | Darine,female 999 | Deborah,female 1000 | Deborah,female 1001 | El,male 1002 | Ela,female 1003 | Eleanore,female 1004 | Eliana,female 1005 | elias,male 1006 | Elizabeth,female 1007 | elodie,female 1008 | Elyssa,female 1009 | Enguerrand,male 1010 | Erwann,male 1011 | Fabio,male 1012 | Fatouma,female 1013 | Fiona,female 1014 | Fode,male 1015 | Goundo,female 1016 | Grace,female 1017 | Haroun,male 1018 | Haya,female 1019 | Hillel,male 1020 | Idris,male 1021 | Iliana,female 1022 | Ilona,female 1023 | Imad,male 1024 | Irina,female 1025 | Isra,female 1026 | Jayden,male 1027 | Jimmy,male 1028 | Joakim,male 1029 | Joana,female 1030 | Johann,male 1031 | John,male 1032 | Jonah,male 1033 | Joris,male 1034 | Josh,male 1035 | Juan,male 1036 | Kadidiatou,female 1037 | Kelyan,male 1038 | Keren,female 1039 | Leny,male 1040 | Leo,male 1041 | Loris,male 1042 | Louison,male 1043 | Lyana,female 1044 | Maceo,male 1045 | Maissane,female 1046 | Malick,male 1047 | Marina,female 1048 | Mark,male 1049 | Matheo,male 1050 | Matthew,male 1051 | May,female 1052 | Melanie,female 1053 | Melodie,female 1054 | Menahem,male 1055 | Michael,male 1056 | Mickael,male 1057 | Miguel,male 1058 | Minh,male 1059 | Mohammad,male 1060 | Mustapha,male 1061 | Nadir,male 1062 | Nail,male 1063 | Naima,female 1064 | Nathaniel,male 1065 | Nayel,male 1066 | Nelson,male 1067 | Nikola,male 1068 | Nohan,male 1069 | Nola,female 1070 | Oriane,female 1071 | Paolo,male 1072 | Patrick,male 1073 | Philemon,male 1074 | Quitterie,female 1075 | Rosa,female 1076 | Saad,male 1077 | Safia,female 1078 | Samson,male 1079 | Sekou,male 1080 | Selene,female 1081 | Sharon,female 1082 | Shelly,female 1083 | Sherine,female 1084 | Siloe,female 1085 | Stephanie,female 1086 | Sven,male 1087 | Swann,male 1088 | Sybille,female 1089 | Tesnime,female 1090 | Thalia,female 1091 | Thibaud,male 1092 | Tim,male 1093 | Tommy,male 1094 | Vladimir,male 1095 | Wissem,male 1096 | Yannick,male 1097 | Yasmina,female 1098 | Yassin,male 1099 | Yoan,male 1100 | Yousra,female 1101 | Yuri,male 1102 | Zephyr,male 1103 | Abd,male 1104 | Aicha,female 1105 | Aidan,male 1106 | Aiden,male 1107 | Aimee,female 1108 | Aina,female 1109 | Aissa,female 1110 | Aissatou,female 1111 | Akram,male 1112 | Alec,male 1113 | Alone,male 1114 | Aloys,male 1115 | Alpha,male 1116 | Anabelle,female 1117 | Anass,male 1118 | Ange,male 1119 | Angel,female 1120 | Angeline,female 1121 | Anita,female 1122 | Annabelle,female 1123 | Antonia,female 1124 | Arie,male 1125 | Arielle,female 1126 | Arij,female 1127 | Armance,female 1128 | Armelle,female 1129 | Arwa,female 1130 | Asia,female 1131 | Asmaa,female 1132 | Aurel,male 1133 | Aydan,male 1134 | Aylan,male 1135 | Aylin,female 1136 | Ayman,male 1137 | Barnabe,male 1138 | Barthelemy,male 1139 | Bella,female 1140 | Bradley,male 1141 | Camila,female 1142 | Cassiopee,female 1143 | Castille,female 1144 | Cerise,female 1145 | Chahinez,female 1146 | Charlene,female 1147 | Charlize,female 1148 | Cheikh,male 1149 | Coralie,female 1150 | Cosima,female 1151 | Cyrielle,female 1152 | Cyrine,female 1153 | Dalla,female 1154 | Daphnee,female 1155 | Darren,male 1156 | Dayane,male 1157 | Dov,male 1158 | Driss,male 1159 | Eddy,male 1160 | Eglantine,female 1161 | eline,female 1162 | Elissa,female 1163 | Eloane,female 1164 | eloi,male 1165 | Eya,female 1166 | Eytan,male 1167 | Fantine,female 1168 | Farouk,male 1169 | Fatma,female 1170 | Feryel,female 1171 | Franck,male 1172 | Franklin,male 1173 | Gaelle,female 1174 | Gary,male 1175 | George,male 1176 | Georgia,female 1177 | Gianni,male 1178 | Gwenaelle,female 1179 | Hadja,female 1180 | Hadriel,male 1181 | Hajar,female 1182 | Halima,female 1183 | Hania,female 1184 | Harouna,male 1185 | Hatouma,female 1186 | Hedi,male 1187 | Helena,female 1188 | Ilhan,male 1189 | Iliane,male 1190 | Ilyane,male 1191 | Ilyess,male 1192 | Ines,female 1193 | Inza,male 1194 | Issiaka,male 1195 | Jacob,male 1196 | Jacqueline,female 1197 | Joanne,female 1198 | Joe,male 1199 | Josue,male 1200 | Joyce,female 1201 | Juliana,female 1202 | Kadidia,female 1203 | Kayla,female 1204 | Keira,female 1205 | Kenzi,male 1206 | Kenzy,male 1207 | Keyla,female 1208 | Kim,male 1209 | Laurine,female 1210 | Leandro,male 1211 | Leontine,female 1212 | Leopoldine,female 1213 | Leticia,female 1214 | Levana,female 1215 | Leyla,female 1216 | Liana,female 1217 | Lilie,female 1218 | Lili-Rose,female 1219 | Lilly,female 1220 | Lily-Rose,female 1221 | Lois,female 1222 | Loise,female 1223 | Lou-Anne,female 1224 | Lounis,male 1225 | Lubin,male 1226 | Lucille,female 1227 | Ludmila,female 1228 | Mae,male 1229 | Mamoudou,male 1230 | Marie-Ange,female 1231 | Matisse,male 1232 | Matys,male 1233 | Maxens,male 1234 | Mayar,female 1235 | Meryem,female 1236 | Milla,female 1237 | Mira,female 1238 | Miya,female 1239 | Monica,female 1240 | Naia,female 1241 | Nala,female 1242 | Natacha,female 1243 | Nelya,female 1244 | Niame,female 1245 | Niouma,female 1246 | Nizar,male 1247 | Noam,male 1248 | Nouha,male 1249 | Olga,female 1250 | Ophelie,female 1251 | Paula,female 1252 | Peter,male 1253 | Priscille,female 1254 | Prosper,male 1255 | Prudence,female 1256 | Rebecca,female 1257 | Reda,male 1258 | Reda,male 1259 | Sadio,female 1260 | Said,male 1261 | Selena,female 1262 | Shai,male 1263 | Shaima,female 1264 | Shaina,female 1265 | Shannon,female 1266 | Shayan,male 1267 | Shayna,female 1268 | Sherine,female 1269 | Shirine,female 1270 | Sihem,female 1271 | Sohan,male 1272 | Stephane,male 1273 | Steven,male 1274 | Swan,male 1275 | Talya,female 1276 | Tancrede,male 1277 | Tanya,female 1278 | Tao,male 1279 | Tatiana,female 1280 | Tea,female 1281 | Terence,male 1282 | Theo,male 1283 | Theophane,male 1284 | Tiana,female 1285 | Tidiane,male 1286 | Tina,female 1287 | Tsipora,female 1288 | Tyron,male 1289 | Vera,female 1290 | Viktoria,female 1291 | Yahya,male 1292 | Yamina,female 1293 | Ylan,male 1294 | Yoel,male 1295 | Yuna,female 1296 | Yvan,male 1297 | Zacharia,male 1298 | Zakary,male 1299 | -------------------------------------------------------------------------------- /java/build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | ext { 3 | springBootVersion = '3.5.0' 4 | logbackEncoderVersion = '6.2' 5 | javaEcsLoggingVersion = '1.7.0' 6 | } 7 | repositories { 8 | mavenCentral() 9 | maven { url 'https://repo.spring.io/milestone' } 10 | } 11 | dependencies { 12 | classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") 13 | } 14 | } 15 | 16 | 17 | subprojects { 18 | apply plugin: 'java' 19 | apply plugin: 'eclipse' 20 | apply plugin: 'idea' 21 | 22 | sourceCompatibility = 1.8 23 | 24 | repositories { 25 | mavenCentral() 26 | maven { url 'https://repo.spring.io/milestone' } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3' 3 | services: 4 | 5 | mysql: 6 | image: mysql:5 7 | command: --default-authentication-plugin=mysql_native_password 8 | restart: always 9 | environment: 10 | - MYSQL_ROOT_PASSWORD=$ELASTIC_PASSWORD 11 | - MYSQL_USER=elastic 12 | - MYSQL_PASSWORD=$ELASTIC_PASSWORD 13 | - MYSQL_DATABASE=person 14 | container_name: mysql 15 | networks: ['stack'] 16 | ports: ['3306:3306'] 17 | restart: on-failure 18 | 19 | networks: 20 | stack: {} 21 | -------------------------------------------------------------------------------- /java/frontend/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'org.springframework.boot' 2 | apply plugin: 'io.spring.dependency-management' 3 | 4 | 5 | jar { 6 | baseName = 'frontend' 7 | version = '1.0' 8 | } 9 | 10 | 11 | dependencies { 12 | compile( 13 | 'org.springframework.boot:spring-boot-starter-web', 14 | 'org.springframework.boot:spring-boot-devtools', 15 | 'org.springframework.boot:spring-boot-starter-thymeleaf', 16 | "co.elastic.logging:logback-ecs-encoder:${javaEcsLoggingVersion}", 17 | 'org.springframework.boot:spring-boot-starter-actuator', 18 | 'org.jolokia:jolokia-core' 19 | ) 20 | } 21 | 22 | 23 | bootJar { 24 | launchScript() 25 | } 26 | -------------------------------------------------------------------------------- /java/frontend/src/main/java/net/xeraa/frontend/FrontendApplication.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.frontend; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.web.client.RestTemplate; 7 | 8 | @SpringBootApplication 9 | public class FrontendApplication { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(FrontendApplication.class, args); 13 | } 14 | 15 | @Bean 16 | public RestTemplate getRestTemplate() { 17 | return new RestTemplate(); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /java/frontend/src/main/java/net/xeraa/frontend/FrontendController.java: -------------------------------------------------------------------------------- 1 | package net.xeraa.frontend; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.slf4j.MDC; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.http.HttpMethod; 9 | import org.springframework.http.ResponseEntity; 10 | import org.springframework.stereotype.Controller; 11 | import org.springframework.ui.Model; 12 | import org.springframework.web.bind.annotation.RequestMapping; 13 | import org.springframework.web.bind.annotation.RequestParam; 14 | import org.springframework.web.client.RestTemplate; 15 | 16 | import java.util.List; 17 | import java.util.Random; 18 | 19 | @Controller 20 | public class FrontendController { 21 | 22 | private static final Logger log = LoggerFactory.getLogger(FrontendController.class); 23 | 24 | @Autowired 25 | private RestTemplate restTemplate; 26 | 27 | private Random random = new Random(); 28 | 29 | @Value("${APP_BACKEND:#{'http://localhost:8081'}}") 30 | private String backendUrl; 31 | 32 | @Value("${APP_FRONTEND:#{'http://localhost:8080'}}") 33 | private String frontendUrl; 34 | 35 | @RequestMapping("/generate") 36 | public void generate(@RequestParam(value="size", required=false, defaultValue="5") Integer size, Model model) { 37 | String callUrl = backendUrl + "/add"; 38 | log.info("Calling {}", callUrl); 39 | 40 | if (size > 10){ 41 | log.warn("Trying to add {} people. This is probably a bit too much and would overload the server", size); 42 | size = 10; 43 | log.info("Change the number of people to be added to {}", size); 44 | } 45 | 46 | for (int i = 0; i < size; i++) { 47 | restTemplate.getForObject(callUrl, String.class); 48 | } 49 | 50 | model.addAttribute("size", size); 51 | log.info("Added {} people", size); 52 | } 53 | 54 | @RequestMapping("/add") 55 | public void add(@RequestParam String name, Model model) { 56 | String callUrl = backendUrl + "/add?name={name}"; 57 | log.info("Calling {}", callUrl); 58 | 59 | restTemplate.exchange(callUrl, 60 | HttpMethod.GET, 61 | null, 62 | String.class, 63 | name 64 | ).getBody(); 65 | 66 | model.addAttribute("size", 1); 67 | model.addAttribute("name", name); 68 | MDC.put("name", name); 69 | log.info("Added 1 person with name {}", name); 70 | } 71 | 72 | @RequestMapping("/search") 73 | public void search(@RequestParam(value="q", required=false, defaultValue="") String q, Model model) { 74 | 75 | String callUrl; 76 | List persons; 77 | if (q.isEmpty()) { 78 | callUrl = backendUrl + "/all"; 79 | log.info("Calling {}", callUrl); 80 | persons = restTemplate.getForObject(callUrl, List.class); 81 | } else { 82 | callUrl = backendUrl + "/search?q={q}"; 83 | log.info("Calling {}", callUrl); 84 | persons = restTemplate.exchange(callUrl, 85 | HttpMethod.GET, 86 | null, 87 | List.class, 88 | q 89 | ).getBody(); 90 | MDC.put("name", q); 91 | log.info("Searched for {}", q); 92 | } 93 | 94 | model.addAttribute("size", persons.size()); 95 | model.addAttribute("persons", persons); 96 | } 97 | 98 | @RequestMapping("/good") 99 | public void good(@RequestParam(value="name", required=false, defaultValue="World") String name, Model model) { 100 | model.addAttribute("name", name); 101 | if(!"World".equals(name)) { 102 | MDC.put("name", name); 103 | } 104 | log.info("Calling something good"); 105 | MDC.clear(); 106 | } 107 | 108 | @RequestMapping("/bad") 109 | public void bad() { 110 | log.info("Calling something bad"); 111 | throw new RuntimeException("My bad, something went wrong..."); 112 | } 113 | 114 | @RequestMapping("/null") 115 | public void nullpointer() { 116 | log.info("Calling something null"); 117 | throw new NullPointerException("This is indeed null..."); 118 | } 119 | 120 | @RequestMapping("/call") 121 | public void call(Model model) throws InterruptedException { 122 | String callUrl = backendUrl + "/slow"; 123 | int millis = this.random.nextInt(2000); 124 | Thread.sleep(millis); 125 | //this.tracer.addTag("random-sleep-millis", String.valueOf(millis)); 126 | model.addAttribute("returnValue", restTemplate.getForObject(callUrl, String.class)); 127 | log.info("Calling {} with a delay of {} ms", callUrl, millis); 128 | } 129 | 130 | @RequestMapping("/call-bad") 131 | public void callBad() throws InterruptedException { 132 | String callUrl = frontendUrl + "/bad"; 133 | int millis = this.random.nextInt(2000); 134 | Thread.sleep(millis); 135 | //this.tracer.addTag("random-sleep-millis", String.valueOf(millis)); 136 | restTemplate.getForObject(callUrl, String.class); 137 | log.info("Calling {} with a delay of {} ms", callUrl, millis); 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | # Give the project a nice name 2 | spring.application.name: frontend 3 | 4 | # Set the port 5 | server.port: ${SERVER_PORT:8080} 6 | 7 | # Enable all the actuator endpoints for HTTP (keep them under the base path) and JMX 8 | management.endpoints: 9 | web: 10 | base-path: / 11 | exposure.include: "*" 12 | jmx.exposure.include: "*" 13 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ​ 5 | 6 | ​ 7 | 8 | 9 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | 19 | ${CONSOLE_LOG_PATTERN} 20 | utf8 21 | 22 | 23 | 24 | 25 | 26 | ${LOG_PATH:-.}/${LOG_FILE}.log 27 | 28 | ${LOG_FILE}.log.%d{yyyy-MM-dd}.gz 29 | 7 30 | 31 | 32 | ${CONSOLE_LOG_PATTERN} 33 | utf8 34 | 35 | 36 | ​ 37 | 38 | 39 | ${LOG_PATH:-.}/${LOG_FILE}.json 40 | 41 | ${LOG_FILE}.json.%d{yyyy-MM-dd}.gz 42 | 7 43 | 44 | 45 | ${springAppName:-} 46 | 47 | 48 | ​ 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/static/elastic-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xeraa/microservice-monitoring/3e5e914bde83b8ff55773f83b2b1de3c6b0ab82f/java/frontend/src/main/resources/static/favicon.ico -------------------------------------------------------------------------------- /java/frontend/src/main/resources/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Welcome! 5 | 6 | 7 | 8 | 9 | 10 | 11 |

Welcome to the Elastic log, metric, and trace demo!

12 |

Things to try out:

13 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/static/style.css: -------------------------------------------------------------------------------- 1 | html { 2 | background-color: #e6e9e9; 3 | background-image: -webkit-linear-gradient(270deg,rgb(230,233,233) 0%,rgb(216,221,221) 100%); 4 | background-image: linear-gradient(270deg,rgb(230,233,233) 0%,rgb(216,221,221) 100%); 5 | -webkit-font-smoothing: antialiased; 6 | } 7 | body { 8 | margin: 0 auto; 9 | padding: 2em 2em 4em; 10 | max-width: 900px; 11 | font-family: Helvetica, Arial, sans-serif; 12 | font-size: 1.2em; 13 | line-height: 1.5em; 14 | color: #545454; 15 | background-color: #ffffff; 16 | box-shadow: 0 0 2px rgba(0, 0, 0, 0.06); 17 | } 18 | .logo { 19 | vertical-align: top; 20 | border: 0; 21 | } 22 | h1 { 23 | font-size: 36px; 24 | line-height: 49px; 25 | display: block; 26 | -webkit-font-smoothing: antialiased; 27 | font-weight: 3500; 28 | } 29 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/templates/add.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 200 OK 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | Back to main 13 | 14 | 15 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/templates/call.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 200 OK 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | Back to main 13 | 14 | 15 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/templates/generate.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 200 OK 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | Back to main 13 | 14 | 15 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/templates/good.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 200 OK 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | Back to main 13 | 14 | 15 | -------------------------------------------------------------------------------- /java/frontend/src/main/resources/templates/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 200 OK 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
NameChildrenCountryCity
Joe1FranceFrance
30 | Back to main 31 | 32 | 33 | -------------------------------------------------------------------------------- /java/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xeraa/microservice-monitoring/3e5e914bde83b8ff55773f83b2b1de3c6b0ab82f/java/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue May 16 16:41:37 EEST 2017 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-all.zip 7 | -------------------------------------------------------------------------------- /java/gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn ( ) { 37 | echo "$*" 38 | } 39 | 40 | die ( ) { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save ( ) { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /java/settings.gradle: -------------------------------------------------------------------------------- 1 | include 'backend', 'frontend' 2 | -------------------------------------------------------------------------------- /lightsail/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | 3 | inventory = ./inventory 4 | 5 | # Set the user globally 6 | remote_user = ubuntu 7 | -------------------------------------------------------------------------------- /lightsail/configure_all.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | vars: 11 | kibana_basic_auth: "{{ attendee_user }}:{{ attendee_password }}" 12 | 13 | 14 | pre_tasks: 15 | - name: Install Python2 to make Ansible work 16 | raw: sudo apt-get update && sudo apt-get -y install python-minimal 17 | 18 | 19 | post_tasks: 20 | - include_tasks: include_event.yml 21 | vars: 22 | application: configure_all 23 | 24 | 25 | tasks: 26 | 27 | 28 | # System 29 | - name: Update and upgrade apt packages 30 | apt: upgrade=dist 31 | 32 | - name: Install NTP to avoid time drift and PIP to manage Python dependencies plus its build tools 33 | apt: 34 | name: [ 'ntp', 'ntpdate', 'python3-pip', 'build-essential', 'libssl-dev', 'libffi-dev' ] 35 | 36 | - name: Install the JRE 37 | apt: name=openjdk-8-jre-headless install_recommends=no 38 | 39 | - name: Install the pyOpenSSL library, so Ansible can use it to check TLS certificates 40 | pip: name=pyopenssl 41 | 42 | 43 | # Add David 44 | - name: Add David's user 45 | user: 46 | name: david 47 | groups: sudo 48 | shell: /bin/bash 49 | state: present 50 | 51 | - name: Placing key for David 52 | authorized_key: 53 | user: david 54 | key: "{{ lookup('file', './files/david.pub') }}" 55 | 56 | - name: Allow David to log in via SSH 57 | lineinfile: 58 | path: /etc/ssh/sshd_config 59 | regexp: '^AllowUsers' 60 | line: 'AllowUsers ubuntu david' 61 | state: present 62 | 63 | - name: Allow passwordless sudo 64 | lineinfile: 65 | path: /etc/sudoers 66 | state: present 67 | regexp: '^%sudo' 68 | line: '%sudo ALL=(ALL) NOPASSWD: ALL' 69 | validate: 'visudo -cf %s' 70 | 71 | - name: Restart SSH 72 | service: name=ssh state=restarted 73 | 74 | 75 | # Global Elasticsearch configuration — this should run early on, so it is used for all later steps 76 | - name: Register a global index template 77 | uri: 78 | url: "{{elasticsearch_host}}_template/template_global" 79 | body_format: json 80 | method: PUT 81 | user: "{{ elasticsearch_user }}" 82 | password: "{{ elasticsearch_password }}" 83 | force_basic_auth: true 84 | body: 85 | template: "*" 86 | settings: 87 | number_of_shards: 1 88 | number_of_replicas: 0 89 | refresh_interval: 2s 90 | status_code: 91 | - 201 92 | - 200 93 | run_once: true 94 | 95 | 96 | # Beats 97 | - name: Set the Elasticsearch password for Beats 98 | lineinfile: 99 | dest: /tmp/cred 100 | line: "{{ elasticsearch_password }}" 101 | state: present 102 | create: yes 103 | mode: 0600 104 | 105 | - name: Get the Beats 106 | apt: deb={{ elastic_download }}/downloads/beats/{{ item }}/{{ item }}-{{ elastic_version }}-amd64.deb 107 | loop: 108 | - auditbeat 109 | - filebeat 110 | - metricbeat 111 | - packetbeat 112 | 113 | - name: Change the Beats configuration 114 | template: "src=templates/{{ item }}.yml dest=/etc/{{ item }}/{{ item }}.yml" 115 | loop: 116 | - auditbeat 117 | - filebeat 118 | - metricbeat 119 | - packetbeat 120 | 121 | - name: Create the Beats keystores 122 | command: "{{ item }} keystore create --force" 123 | loop: 124 | - auditbeat 125 | - filebeat 126 | - metricbeat 127 | - packetbeat 128 | 129 | - name: Set the password in the Beats keystore files 130 | shell: cat /tmp/cred | {{ item }} keystore add ES_PWD --stdin --force 131 | loop: 132 | - auditbeat 133 | - filebeat 134 | - metricbeat 135 | - packetbeat 136 | 137 | - name: Remove the password file 138 | file: 139 | path: /tmp/cred 140 | state: absent 141 | 142 | - name: Restart and make sure the Beats autostart 143 | service: name={{ item }} state=restarted enabled=yes 144 | loop: 145 | - auditbeat 146 | - filebeat 147 | - metricbeat 148 | - packetbeat 149 | 150 | - name: Wait if the Beats are actually running 151 | pause: minutes=1 152 | 153 | - name: Get the state of all services and check the status of Auditbeat 154 | service_facts: ~ 155 | failed_when: ansible_facts.services.auditbeat.state != "running" 156 | 157 | - name: Get the state of all services and check the status of Filebeat 158 | service_facts: ~ 159 | failed_when: ansible_facts.services.filebeat.state != "running" 160 | 161 | - name: Get the state of all services and check the status of Metricbeat 162 | service_facts: ~ 163 | failed_when: ansible_facts.services.metricbeat.state != "running" 164 | 165 | - name: Get the state of all services and check the status of Packetbeat 166 | service_facts: ~ 167 | failed_when: ansible_facts.services.packetbeat.state != "running" 168 | 169 | 170 | # TLS 171 | - name: Add the certbot repository 172 | apt_repository: repo="ppa:certbot/certbot" 173 | 174 | - name: Install certbot and update the cache for the new PPA 175 | apt: name=python-certbot-nginx update_cache=yes 176 | 177 | - name: Add the hostname to the certificates to create 178 | set_fact: 179 | certificates: 180 | - "{{ inventory_hostname }}" 181 | 182 | - name: Add more domains to the frontend certificate 183 | set_fact: 184 | certificates: 185 | - "{{ inventory_hostname }}" 186 | - "{{ domain }}" 187 | - "www.{{ domain }}" 188 | when: inventory_hostname_short == "frontend" 189 | 190 | - name: Add more domains to the backend certificate 191 | set_fact: 192 | certificates: 193 | - "{{ inventory_hostname }}" 194 | - "kibana.{{ domain }}" 195 | - "dashboard.{{ domain }}" 196 | when: inventory_hostname_short == "backend" 197 | 198 | - name: Just to be extra sure, explicitly stop nginx (if it is installed) 199 | service: name=nginx state=stopped 200 | ignore_errors: yes 201 | 202 | - name: Create the certificate 203 | command: > 204 | certbot certonly --non-interactive --standalone 205 | --agree-tos --email admin@{{ domain }} 206 | -d {{ certificates | join(',') }} 207 | creates=/etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem 208 | 209 | - name: Generate strong dhparams, but only if the file doesn't exist 210 | command: openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048 creates=/etc/ssl/certs/dhparam.pem 211 | 212 | 213 | # APM 214 | - name: Fetch the APM agent 215 | get_url: 216 | url: "https://search.maven.org/remotecontent?filepath=co/elastic/apm/elastic-apm-agent/{{ apm_java_version }}/elastic-apm-agent-{{ apm_java_version }}.jar" 217 | dest: "/opt/elastic-apm-agent-{{ apm_java_version }}.jar" 218 | mode: 0444 219 | 220 | 221 | # nginx 222 | - name: Install nginx 223 | apt: name=nginx 224 | 225 | - name: Set a global TLS configuration 226 | template: src=templates/tls.conf dest=/etc/nginx/tls.conf 227 | 228 | - name: Change the nginx configuration 229 | template: src=templates/nginx.conf dest=/etc/nginx/sites-available/default 230 | 231 | - name: Restart nginx and make sure it autostarts 232 | service: name=nginx state=restarted enabled=yes 233 | 234 | - name: Check HTTP 235 | uri: 236 | url: "http://{{ inventory_hostname }}" 237 | follow_redirects: none 238 | status_code: 301 239 | register: response 240 | retries: 3 241 | delay: 2 242 | delegate_to: 127.0.0.1 243 | become: false 244 | 245 | - name: Fail if HTTP is not being redirected to HTTPS 246 | fail: 247 | when: response.status != 301 248 | 249 | - name: Check HTTPS 250 | openssl_certificate: 251 | path: /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem 252 | provider: assertonly 253 | issuer: 254 | O: Let's Encrypt 255 | has_expired: false 256 | subject_alt_name: 257 | - "DNS:{{ inventory_hostname }}" 258 | 259 | - name: Check HTTPS apex 260 | openssl_certificate: 261 | path: /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem 262 | provider: assertonly 263 | issuer: 264 | O: Let's Encrypt 265 | has_expired: false 266 | subject_alt_name: 267 | - "DNS:{{ domain }}" 268 | when: inventory_hostname_short == "frontend" 269 | -------------------------------------------------------------------------------- /lightsail/configure_backend.yml: -------------------------------------------------------------------------------- 1 | - hosts: backend 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: configure_backend 14 | 15 | 16 | tasks: 17 | 18 | 19 | # MySQL 20 | - name: Install the DEB packages required for Ansible's MySQL modules 21 | apt: 22 | name: [ 'python3-dev', 'libmysqlclient-dev' ] 23 | 24 | - name: Install the Python package required for Ansible's MySQL modules 25 | pip: name=mysqlclient 26 | 27 | - name: Install MySQL 28 | apt: name=mysql-server 29 | 30 | - name: Removes all anonymous user accounts 31 | mysql_user: 32 | name: "" 33 | host_all: yes 34 | state: absent 35 | 36 | - name: Create database user and password with all database privileges 37 | mysql_user: 38 | name: "{{ mysql_user }}" 39 | password: "{{ mysql_password }}" 40 | priv: "*.*:ALL" 41 | host: "%" 42 | state: present 43 | 44 | - name: Create a new database for the application 45 | mysql_db: 46 | name: "{{ mysql_database }}" 47 | state: present 48 | 49 | - name: Bind MySQL to all interfaces 50 | ini_file: dest=/etc/mysql/mysql.conf.d/mysqld.cnf 51 | section=mysqld 52 | option=bind-address 53 | value="0.0.0.0" 54 | 55 | - name: Restart MySQL and make sure it autostarts 56 | service: name=mysql state=restarted enabled=yes 57 | 58 | 59 | # Heartbeat 60 | - name: Set the Elasticsearch password for Heartbeat and APM 61 | lineinfile: 62 | dest: /tmp/cred 63 | line: "{{ elasticsearch_password }}" 64 | state: present 65 | create: yes 66 | mode: 0600 67 | 68 | - name: Install Heartbeat 69 | apt: deb={{ elastic_download }}/downloads/beats/heartbeat/heartbeat-{{ elastic_version }}-amd64.deb 70 | 71 | - name: Change the Heartbeat configuration 72 | template: src=templates/heartbeat.yml dest=/etc/heartbeat/heartbeat.yml 73 | 74 | - name: Create the Heartbeat keystore 75 | command: heartbeat keystore create --force 76 | 77 | - name: Set the password in the Heartbeat keystore file 78 | shell: cat /tmp/cred | heartbeat keystore add ES_PWD --stdin --force 79 | 80 | - name: Remove the password file 81 | file: 82 | path: /tmp/cred 83 | state: absent 84 | 85 | - name: Restart Heartbeat and make sure it autostarts 86 | service: name=heartbeat-elastic state=restarted enabled=yes 87 | 88 | - name: Wait if Heartbeat is actually running 89 | pause: minutes=1 90 | 91 | - name: Get the state of all services and check the status of Heartbeat 92 | service_facts: ~ 93 | failed_when: ansible_facts.services["heartbeat-elastic"].state != "running" 94 | 95 | 96 | # Dashboard user 97 | - name: Create a role for a user to only view the dashboards 98 | uri: 99 | url: "{{elasticsearch_host}}.security-6/doc/role-read_dashboard" 100 | body_format: json 101 | method: PUT 102 | user: "{{ elasticsearch_user }}" 103 | password: "{{ elasticsearch_password }}" 104 | force_basic_auth: true 105 | body: "{{ lookup('file','security_role_dashboard.json') }}" 106 | status_code: 107 | - 201 108 | - 200 109 | 110 | - name: Create the dashboard user 111 | uri: 112 | url: "{{elasticsearch_host}}_xpack/security/user/{{ attendee_user }}" 113 | body_format: json 114 | method: POST 115 | user: "{{ elasticsearch_user }}" 116 | password: "{{ elasticsearch_password }}" 117 | force_basic_auth: true 118 | body: "{{ lookup('template','security_user_dashboard.json') }}" 119 | status_code: 120 | - 201 121 | - 200 122 | 123 | 124 | # Watcher 125 | - name: Add an example Watch from a local file 126 | uri: 127 | url: "{{elasticsearch_host}}_xpack/watcher/watch/heapsize" 128 | body_format: json 129 | method: PUT 130 | user: "{{ elasticsearch_user }}" 131 | password: "{{ elasticsearch_password }}" 132 | force_basic_auth: true 133 | body: "{{ lookup('file','alerting_heapsize.json') }}" 134 | status_code: 135 | - 201 136 | - 200 137 | 138 | 139 | # ApacheBench 140 | - name: Install ApacheBench 141 | apt: 142 | name: [ 'apache2-utils', 'parallel' ] 143 | 144 | - name: Add a list of URLs to benchmark 145 | template: src=templates/urls.txt dest=/home/ubuntu/urls.txt owner=ubuntu group=ubuntu mode=0644 146 | 147 | - name: Add a quick ApacheBench script 148 | copy: 149 | src: files/ab.sh 150 | dest: /home/ubuntu/ab.sh 151 | owner: ubuntu 152 | group: ubuntu 153 | mode: 0755 154 | 155 | - name: Generate some load at the full hour, so ML can learn something 156 | cron: 157 | name: Generate some load every hour 158 | minute: "0" 159 | hour: "*" 160 | job: > 161 | (echo "https://{{ domain }}/good"; echo "https://{{ domain }}/good?name=bot") | parallel 'ab -n 10 -c 2 {}' && 162 | sleep 15s && 163 | (echo "https://{{ domain }}/search?q=Philipp"; echo "https://{{ domain }}/null") | parallel 'ab -n 6 -c 2 {}' && 164 | sleep 45s && 165 | (echo "https://{{ domain }}/"; echo "https://{{ domain }}/good?name=bot") | parallel 'ab -n 20 -c 4 {}' 166 | -------------------------------------------------------------------------------- /lightsail/deploy_backend.yml: -------------------------------------------------------------------------------- 1 | - hosts: backend 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: deploy_backend 14 | 15 | 16 | tasks: 17 | 18 | # Check that we can reach MySQL 19 | - name: Check MySQL is accessible before deploying the application 20 | wait_for: 21 | host: "{{ mysql_server }}" 22 | port: 3306 23 | state: started 24 | delay: 0 25 | timeout: 3 26 | 27 | - include_tasks: include_deploy_boot.yml 28 | vars: 29 | application: backend 30 | -------------------------------------------------------------------------------- /lightsail/deploy_frontend.yml: -------------------------------------------------------------------------------- 1 | - hosts: frontend 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: deploy_frontend 14 | 15 | 16 | tasks: 17 | 18 | - include_tasks: include_deploy_boot.yml 19 | vars: 20 | application: frontend 21 | -------------------------------------------------------------------------------- /lightsail/files/ab.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat urls.txt | parallel "ab -n 1000 -c 30 {}" 4 | -------------------------------------------------------------------------------- /lightsail/files/alerting_heapsize.json: -------------------------------------------------------------------------------- 1 | { 2 | "trigger": { 3 | "schedule": { 4 | "interval": "1m" 5 | } 6 | }, 7 | "input": { 8 | "search": { 9 | "request": { 10 | "search_type": "query_then_fetch", 11 | "indices": [ 12 | "metricbeat-*" 13 | ], 14 | "types": [], 15 | "body": { 16 | "size": 0, 17 | "query": { 18 | "bool": { 19 | "filter": { 20 | "range": { 21 | "@timestamp": { 22 | "gte": "{{ctx.trigger.scheduled_time}}||-5m", 23 | "lte": "{{ctx.trigger.scheduled_time}}", 24 | "format": "strict_date_optional_time||epoch_millis" 25 | } 26 | } 27 | } 28 | } 29 | }, 30 | "aggs": { 31 | "bucketAgg": { 32 | "terms": { 33 | "field": "beat.name", 34 | "size": 5, 35 | "order": { 36 | "metricAgg": "desc" 37 | } 38 | }, 39 | "aggs": { 40 | "metricAgg": { 41 | "max": { 42 | "field": "jolokia.metrics.memory.heap_usage.used" 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } 51 | }, 52 | "condition": { 53 | "script": { 54 | "source": "ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['metricAgg'].value > params.threshold) { return true; } } return false;", 55 | "lang": "painless", 56 | "params": { 57 | "threshold": 50000000 58 | } 59 | } 60 | }, 61 | "transform": { 62 | "script": { 63 | "source": "HashMap result = new HashMap(); ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; ArrayList filteredHits = new ArrayList(); for (int i = 0; i < arr.length; i++) { HashMap filteredHit = new HashMap(); filteredHit.key = arr[i].key; filteredHit.value = arr[i]['metricAgg'].value; if (filteredHit.value > params.threshold) { filteredHits.add(filteredHit); } } result.results = filteredHits; return result;", 64 | "lang": "painless", 65 | "params": { 66 | "threshold": 50000000 67 | } 68 | } 69 | }, 70 | "actions": { 71 | "logging_1": { 72 | "logging": { 73 | "level": "info", 74 | "text": "Heap is too large" 75 | } 76 | } 77 | }, 78 | "metadata": { 79 | "name": "Heap", 80 | "watcherui": { 81 | "trigger_interval_unit": "m", 82 | "agg_type": "max", 83 | "time_field": "@timestamp", 84 | "trigger_interval_size": 1, 85 | "term_size": 5, 86 | "time_window_unit": "m", 87 | "threshold_comparator": ">", 88 | "term_field": "beat.name", 89 | "index": [ 90 | "metricbeat-*" 91 | ], 92 | "time_window_size": 5, 93 | "threshold": 50000000, 94 | "agg_field": "jolokia.metrics.memory.heap_usage.used" 95 | }, 96 | "xpack": { 97 | "type": "threshold" 98 | } 99 | }, 100 | "status": { 101 | "state": { 102 | "active": true, 103 | "timestamp": "2017-09-29T13:31:00.309Z" 104 | }, 105 | "actions": { 106 | "logging_1": { 107 | "ack": { 108 | "timestamp": "2017-09-29T13:32:00.822Z", 109 | "state": "ackable" 110 | }, 111 | "last_execution": { 112 | "timestamp": "2017-09-29T13:40:03.537Z", 113 | "successful": true 114 | }, 115 | "last_successful_execution": { 116 | "timestamp": "2017-09-29T13:40:03.537Z", 117 | "successful": true 118 | } 119 | } 120 | }, 121 | "version": -1, 122 | "last_checked": "2017-09-29T13:40:03.537Z", 123 | "last_met_condition": "2017-09-29T13:40:03.537Z" 124 | } 125 | } -------------------------------------------------------------------------------- /lightsail/files/david.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAI/wHIPpPnO5VmlMuhAHftOGmjSofuPjIfIk7EiynE5ZErQcqN1+Za7MFUzV7FeBwJwc/XdNDzRBFGbWvixG/J2gVcpYEELVDHyBwJqzNEACs1eEFt2eODBCFzx+B3D74dnbP/UsK7LyUYjy6Bz9Wz4n13SHqH69hiRvYovyEbSGBMoQL27IOdo10XfLFqlRZ0s5gfjpMBbozSZXMgrxBnmQVxVtuLMM3sw+yZjn3IWzPZgMfQGZzxCOXxLzywWXcjcNcJvkGh3kbLC70bR3hqgLSmXgs84K/rXpjUy+o93msmwt81CZzShbJ4ub6Wu9eVw7j/xQVD/GPtboAiECH david@pilato.fr -------------------------------------------------------------------------------- /lightsail/files/security_role_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": [], 3 | "indices": [ 4 | { 5 | "names": [ 6 | "apm-*", 7 | "metricbeat-*", 8 | "packetbeat-*", 9 | "filebeat-*", 10 | "heartbeat-*", 11 | "auditbeat-*" 12 | ], 13 | "privileges": [ 14 | "read" 15 | ], 16 | "field_security": { 17 | "grant": [ 18 | "*" 19 | ] 20 | } 21 | } 22 | ], 23 | "run_as": [], 24 | "metadata": {}, 25 | "type": "role" 26 | } 27 | -------------------------------------------------------------------------------- /lightsail/include_deploy_boot.yml: -------------------------------------------------------------------------------- 1 | - name: Package the JAR 2 | local_action: shell cd ../java/{{ application }}/; gradle build 3 | become: false 4 | changed_when: true 5 | 6 | - name: Get the current timestamp 7 | set_fact: 8 | current_timestamp: "{{ ansible_date_time.epoch }}" 9 | 10 | - name: Make sure that the logging directory exists 11 | file: 12 | path: /var/log/apps/ 13 | state: directory 14 | owner: ubuntu 15 | group: ubuntu 16 | mode: 0755 17 | 18 | - name: Deploy the executable uber JAR 19 | copy: 20 | src: "../java/{{ application }}/build/libs/{{ application }}.jar" 21 | dest: "/opt/{{ application }}_{{ current_timestamp }}.jar" 22 | owner: ubuntu 23 | group: ubuntu 24 | mode: 0500 25 | 26 | - name: Provide a conf file to run it as a service 27 | template: 28 | src: templates/{{ application }}.conf 29 | dest: "/opt/{{ application }}_{{ current_timestamp }}.conf" 30 | owner: root 31 | group: root 32 | mode: 0400 33 | 34 | - name: Check if there is a previous version 35 | stat: 36 | path: "/opt/{{ application }}.jar" 37 | register: symlink 38 | 39 | - name: Link to current version 40 | file: 41 | src: "/opt/{{ application }}_{{ current_timestamp }}.jar" 42 | dest: "/opt/{{ application }}.jar" 43 | owner: ubuntu 44 | group: ubuntu 45 | state: link 46 | 47 | - name: Link the JAR so it can be managed as a service through init.d 48 | file: 49 | src: "/opt/{{ application }}.jar" 50 | dest: "/etc/init.d/{{ application }}" 51 | state: link 52 | owner: ubuntu 53 | group: ubuntu 54 | 55 | - name: Restart the JAR, make sure it autostarts, and reload the configuration 56 | systemd: name="{{ application }}" state=restarted enabled=yes daemon_reload=yes 57 | 58 | - name: Wait for the Java application to start up 59 | pause: seconds=30 60 | 61 | - name: Check if the service is available 62 | uri: url="https://{{ inventory_hostname }}/health" 63 | register: response 64 | ignore_errors: yes 65 | 66 | - name: Remove old JARs (keeping the five latest good ones) 67 | shell: "ls -tr /opt/{{ application }}_*.jar | head -n -5 | xargs rm -f" 68 | register: remove_output 69 | 70 | - name: Remove old configs (keeping the five latest good ones) 71 | shell: "ls -tr /opt/{{ application }}_*.conf | head -n -5 | xargs rm -f" 72 | -------------------------------------------------------------------------------- /lightsail/include_event.yml: -------------------------------------------------------------------------------- 1 | - name: Get the local user 2 | command: whoami 3 | register: local_username 4 | delegate_to: 127.0.0.1 5 | become: false 6 | changed_when: false 7 | 8 | - name: Store the playbook run event in Elasticsearch so it can be used as an annotation 9 | uri: 10 | url: "{{elasticsearch_host}}events/deployment" 11 | body_format: json 12 | method: POST 13 | user: "{{ elasticsearch_user }}" 14 | password: "{{ elasticsearch_password }}" 15 | force_basic_auth: true 16 | body: 17 | "@timestamp": "{{ ansible_date_time.iso8601 }}" 18 | application: "{{ application }}" 19 | system: ansible 20 | host: "{{ inventory_hostname }}" 21 | user: "{{ local_username.stdout }}" 22 | status_code: 201 23 | -------------------------------------------------------------------------------- /lightsail/inventory: -------------------------------------------------------------------------------- 1 | [all] 2 | backend.xeraa.wtf ansible_python_interpreter=/usr/bin/python3 3 | frontend.xeraa.wtf ansible_python_interpreter=/usr/bin/python3 4 | 5 | 6 | [backend] 7 | backend.xeraa.wtf 8 | 9 | 10 | [frontend] 11 | frontend.xeraa.wtf 12 | -------------------------------------------------------------------------------- /lightsail/restart_frontend.yml: -------------------------------------------------------------------------------- 1 | - hosts: frontend 2 | remote_user: ubuntu 3 | become: true 4 | gather_facts: yes 5 | 6 | 7 | vars_files: 8 | - variables.yml 9 | 10 | 11 | post_tasks: 12 | - include_tasks: include_event.yml 13 | vars: 14 | application: restart_frontend 15 | 16 | 17 | tasks: 18 | 19 | - name: Stop the frontend application 20 | systemd: name="frontend" state=stopped 21 | 22 | - name: Wait while the application is down 23 | pause: seconds=30 24 | 25 | - name: Start the frontend application again 26 | systemd: name="frontend" state=started 27 | 28 | - name: Wait for the Java application to start up 29 | pause: seconds=30 30 | 31 | - name: Check if the service is available 32 | uri: url="https://{{ inventory_hostname }}/health" 33 | register: response 34 | ignore_errors: yes 35 | -------------------------------------------------------------------------------- /lightsail/templates/auditbeat.yml: -------------------------------------------------------------------------------- 1 | auditbeat.modules: 2 | 3 | 4 | - module: system 5 | datasets: 6 | - host 7 | - user 8 | - package 9 | - login 10 | period: 1m 11 | user.detect_password_changes: true 12 | 13 | 14 | - module: system 15 | datasets: 16 | - process 17 | - socket 18 | period: 2s 19 | 20 | 21 | - module: file_integrity 22 | paths: 23 | - /opt/ 24 | scan_at_start: true 25 | scan_rate_per_sec: 50 MiB 26 | file.max_file_size: 100 MiB 27 | file.hash_types: [sha1] 28 | 29 | 30 | name: "{{ inventory_hostname }}" 31 | tags: ["{{ env }}", "lightsail"] 32 | 33 | 34 | processors: 35 | - add_cloud_metadata: ~ 36 | - add_host_metadata: ~ 37 | 38 | 39 | xpack.monitoring.enabled: true 40 | 41 | 42 | output.elasticsearch: 43 | hosts: ["{{ elasticsearch_host }}"] 44 | username: "{{ elasticsearch_user }}" 45 | password: "${ES_PWD}" 46 | 47 | 48 | setup: 49 | kibana: 50 | host: "{{ kibana_host }}" 51 | username: "{{ elasticsearch_user }}" 52 | password: "${ES_PWD}" 53 | dashboards.enabled: true 54 | -------------------------------------------------------------------------------- /lightsail/templates/backend.conf: -------------------------------------------------------------------------------- 1 | JAVA_OPTS="-Xmx384m\ 2 | -DLOG_PATH=/var/log/apps\ 3 | -DSERVER_PORT=8080\ 4 | -DAPP_BACKEND={{ backend_server }}\ 5 | -DAPP_FRONTEND={{ frontend_server }}\ 6 | -DDATABASE_SERVER={{ mysql_server }}\ 7 | -DDATABASE_USERNAME={{ mysql_user }}\ 8 | -DDATABASE_PASSWORD={{mysql_password }}\ 9 | -DDATABASE_NAME={{ mysql_database }}\ 10 | -javaagent:/opt/elastic-apm-agent-{{ apm_java_version }}.jar\ 11 | -Delastic.apm.service_name=backend\ 12 | -Delastic.apm.application_packages=net.xeraa.backend\ 13 | -Delastic.apm.stack_trace_limit=150\ 14 | -Delastic.apm.trace_methods=net.xeraa.*#*\ 15 | -Delastic.apm.environment={{ env }}\ 16 | -Delastic.apm.sample_rate=1.0\ 17 | -Delastic.apm.capture_body=all\ 18 | -Delastic.apm.span_frames_min_duration=-1\ 19 | -Delastic.apm.server_urls={{ apm_server }}\ 20 | -Delastic.apm.ignore_urls=/health,/metrics*,/jolokia\ 21 | -Delastic.apm.log_file=/var/log/apps/apm-backend\ 22 | -Delastic.apm.enable_log_correlation=true\ 23 | -Delastic.apm.secret_token={{ apm_secret }}" 24 | -------------------------------------------------------------------------------- /lightsail/templates/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.inputs: 2 | 3 | # Collect the JSON log files from the Spring Boot apps 4 | - type: log 5 | paths: 6 | - /var/log/apps/*.json 7 | fields_under_root: true 8 | fields: 9 | application: java 10 | json.keys_under_root: true 11 | json.add_error_key: true 12 | 13 | 14 | filebeat.modules: 15 | - module: auditd 16 | - module: nginx 17 | - module: system 18 | {% if inventory_hostname_short == 'backend' %} 19 | - module: mysql 20 | {% endif %} 21 | 22 | 23 | name: "{{ inventory_hostname }}" 24 | tags: ["{{ env }}", "lightsail"] 25 | 26 | 27 | processors: 28 | - add_cloud_metadata: ~ 29 | - add_host_metadata: ~ 30 | 31 | 32 | xpack.monitoring.enabled: true 33 | 34 | 35 | output.elasticsearch: 36 | hosts: ["{{ elasticsearch_host }}"] 37 | username: "{{ elasticsearch_user }}" 38 | password: "${ES_PWD}" 39 | 40 | 41 | setup: 42 | kibana: 43 | host: "{{ kibana_host }}" 44 | username: "{{ elasticsearch_user }}" 45 | password: "${ES_PWD}" 46 | dashboards.enabled: true 47 | -------------------------------------------------------------------------------- /lightsail/templates/frontend.conf: -------------------------------------------------------------------------------- 1 | JAVA_OPTS="-Xmx384m\ 2 | -DLOG_PATH=/var/log/apps\ 3 | -DSERVER_PORT=8080\ 4 | -DAPP_BACKEND={{ backend_server }}\ 5 | -DAPP_FRONTEND={{ frontend_server }}\ 6 | -javaagent:/opt/elastic-apm-agent-{{ apm_java_version }}.jar\ 7 | -Delastic.apm.service_name=frontend\ 8 | -Delastic.apm.application_packages=net.xeraa.frontend\ 9 | -Delastic.apm.stack_trace_limit=150\ 10 | -Delastic.apm.trace_methods=net.xeraa.*#*\ 11 | -Delastic.apm.environment={{ env }}\ 12 | -Delastic.apm.sample_rate=1.0\ 13 | -Delastic.apm.capture_body=all\ 14 | -Delastic.apm.span_frames_min_duration=-1\ 15 | -Delastic.apm.server_urls={{ apm_server }}\ 16 | -Delastic.apm.ignore_urls=/health,/metrics*,/jolokia\ 17 | -Delastic.apm.log_file=/var/log/apps/apm-frontend\ 18 | -Delastic.apm.enable_log_correlation=true\ 19 | -Delastic.apm.secret_token={{ apm_secret }}" 20 | -------------------------------------------------------------------------------- /lightsail/templates/heartbeat.yml: -------------------------------------------------------------------------------- 1 | heartbeat.monitors: 2 | 3 | - type: http 4 | urls: ["https://{{ domain }}/health"] 5 | schedule: "@every 10s" 6 | timeout: 3s 7 | check.response.status: 200 8 | 9 | - type: http 10 | urls: ["https://backend.{{ domain }}/health"] 11 | schedule: "@every 10s" 12 | timeout: 3s 13 | check.response.status: 200 14 | 15 | - type: http 16 | urls: ["https://kibana.{{ domain }}"] 17 | schedule: "@every 10s" 18 | timeout: 3s 19 | check.response.status: 200 20 | 21 | - type: http 22 | urls: ["{{ elasticsearch_host }}"] 23 | username: {{ elasticsearch_user }} 24 | password: "${ES_PWD}" 25 | schedule: "@every 10s" 26 | timeout: 3s 27 | check.response.status: 200 28 | 29 | - type: http 30 | urls: ["{{ apm_server }}/healthcheck"] 31 | schedule: "@every 10s" 32 | timeout: 3s 33 | check.response.status: 200 34 | 35 | - type: tcp 36 | hosts: ["{{ mysql_server }}:3306"] 37 | schedule: "@every 10s" 38 | timeout: 3s 39 | name: mysql 40 | 41 | 42 | name: "{{ inventory_hostname }}" 43 | tags: ["{{ env }}", "lightsail"] 44 | 45 | 46 | processors: 47 | - add_cloud_metadata: ~ 48 | - add_host_metadata: ~ 49 | 50 | 51 | xpack.monitoring.enabled: true 52 | 53 | 54 | output.elasticsearch: 55 | hosts: ["{{ elasticsearch_host }}"] 56 | username: "{{ elasticsearch_user }}" 57 | password: "${ES_PWD}" 58 | 59 | 60 | # No setup since Uptime Monitoring is part of Kibana 61 | -------------------------------------------------------------------------------- /lightsail/templates/metricbeat.yml: -------------------------------------------------------------------------------- 1 | metricbeat.modules: 2 | 3 | - module: system 4 | metricsets: 5 | - cpu 6 | - load 7 | - core 8 | - diskio 9 | - filesystem 10 | - fsstat 11 | - memory 12 | - network 13 | - process 14 | - process_summary 15 | - socket 16 | enabled: true 17 | period: 10s 18 | processes: ['.*'] 19 | cgroups: true 20 | process.include_top_n: 21 | enabled: true 22 | by_cpu: 20 23 | by_memory: 20 24 | 25 | - module: nginx 26 | metricsets: ["stubstatus"] 27 | enabled: true 28 | period: 10s 29 | hosts: ["https://{{ inventory_hostname }}"] 30 | 31 | {% if inventory_hostname_short == 'backend' %} 32 | - module: mysql 33 | metricsets: ["status"] 34 | hosts: ["tcp({{ mysql_server }}:3306)/"] 35 | username: "{{ mysql_user }}" 36 | password: "{{ mysql_password }}" 37 | {% endif %} 38 | 39 | - module: http 40 | metricsets: ["json"] 41 | period: 10s 42 | hosts: ["localhost:8080"] 43 | namespace: health 44 | path: /health 45 | method: GET 46 | 47 | {% for metric in ('http.server.requests', 'process.files.max', 'jvm.gc.memory.promoted', 'tomcat.cache.hit', 48 | 'jvm.memory.committed', 'system.load.average.1m', 'tomcat.cache.access', 'jvm.memory.used', 49 | 'jvm.gc.max.data.size', 'system.cpu.count', 'logback.events', 'tomcat.global.sent', 50 | 'jvm.buffer.memory.used', 'tomcat.sessions.created', 'jvm.memory.max', 'jvm.threads.daemon', 51 | 'system.cpu.usage', 'jvm.gc.memory.allocated', 'tomcat.global.request.max', 52 | 'tomcat.global.request', 'tomcat.sessions.expired', 'jvm.threads.live', 'jvm.threads.peak', 53 | 'tomcat.global.received', 'process.uptime', 'tomcat.sessions.rejected', 'process.cpu.usage', 54 | 'jvm.gc.pause', 'tomcat.threads.config.max', 'jvm.classes.loaded', 'jvm.classes.unloaded', 55 | 'tomcat.global.error', 'tomcat.sessions.active.current', 'tomcat.sessions.alive.max', 56 | 'jvm.gc.live.data.size', 'tomcat.servlet.request.max', 'tomcat.threads.current', 57 | 'tomcat.servlet.request', 'process.files.open', 'jvm.buffer.count', 'jvm.buffer.total.capacity', 58 | 'tomcat.sessions.active.max', 'tomcat.threads.busy', 'process.start.time', 'tomcat.servlet.error') %} 59 | - module: http 60 | metricsets: ["json"] 61 | period: 10s 62 | hosts: ["localhost:8080"] 63 | namespace: metrics 64 | path: "/metrics/{{ metric }}" 65 | method: GET 66 | dedot.enabled: true 67 | {% endfor %} 68 | 69 | - module: jolokia 70 | metricsets: ["jmx"] 71 | hosts: ["localhost:8080"] 72 | namespace: metrics 73 | jmx.mappings: 74 | - mbean: "java.lang:type=Runtime" 75 | attributes: 76 | - attr: Uptime 77 | field: uptime 78 | - mbean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" 79 | attributes: 80 | - attr: CollectionTime 81 | field: gc.cms_collection_time 82 | event: gc 83 | - attr: CollectionCount 84 | field: gc.cms_collection_count 85 | event: gc 86 | - mbean: "java.lang:type=Memory" 87 | attributes: 88 | - attr: HeapMemoryUsage 89 | field: memory.heap_usage 90 | event: heap 91 | - attr: NonHeapMemoryUsage 92 | field: memory.non_heap_usage 93 | event: heap 94 | - mbean: "java.lang:type=Threading" 95 | attributes: 96 | - attr: ThreadCount 97 | field: thread.count 98 | event: thread 99 | - attr: DaemonThreadCount 100 | field: thread.daemon 101 | event: thread 102 | jmx.instance: "{{ inventory_hostname }}" 103 | 104 | 105 | name: "{{ inventory_hostname }}" 106 | tags: ["{{ env }}", "lightsail"] 107 | 108 | 109 | processors: 110 | - add_cloud_metadata: ~ 111 | - add_host_metadata: ~ 112 | 113 | 114 | xpack.monitoring.enabled: true 115 | 116 | 117 | output.elasticsearch: 118 | hosts: ["{{ elasticsearch_host }}"] 119 | username: "{{ elasticsearch_user }}" 120 | password: "${ES_PWD}" 121 | 122 | 123 | setup: 124 | kibana: 125 | host: "{{ kibana_host }}" 126 | username: "{{ elasticsearch_user }}" 127 | password: "${ES_PWD}" 128 | dashboards.enabled: true 129 | -------------------------------------------------------------------------------- /lightsail/templates/nginx.conf: -------------------------------------------------------------------------------- 1 | # Don't send the nginx version number in error pages and server header 2 | server_tokens off; 3 | 4 | # Don't allow the page to render inside a frame of an iframe 5 | add_header X-Frame-Options DENY; 6 | 7 | # Disable sniffing for user supplied content 8 | add_header X-Content-Type-Options nosniff; 9 | 10 | # Add the HSTS header for all subdomains 11 | add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; 12 | 13 | 14 | # Redirect HTTP to HTTPS 15 | server { 16 | listen 80; 17 | server_name _; 18 | 19 | location / { 20 | return 301 https://$host$request_uri; 21 | } 22 | } 23 | 24 | 25 | # Set up HTTPS 26 | server { 27 | charset utf-8; 28 | listen 443 ssl; 29 | server_name _; 30 | 31 | ssl_certificate /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem; 32 | ssl_certificate_key /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem; 33 | include /etc/nginx/tls.conf; 34 | 35 | # Proxy HTTPS to HTTP on port 8080 to the Java application 36 | location / { 37 | proxy_pass http://127.0.0.1:8080; 38 | proxy_redirect off; 39 | proxy_set_header Host $host; 40 | proxy_set_header X-Real-IP $remote_addr; 41 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 42 | proxy_set_header X-Forwarded-Host $server_name; 43 | } 44 | 45 | # Enable the stub status module for Metricbeat 46 | location /server-status { 47 | stub_status on; 48 | access_log off; 49 | } 50 | } 51 | 52 | 53 | {% if inventory_hostname_short == 'backend' %} 54 | # On the backend instance proxy Kibana dashboard mode to have a nice URL 55 | server { 56 | charset utf-8; 57 | listen 443 ssl; 58 | server_name dashboard.{{ domain }}; 59 | 60 | ssl_certificate /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem; 61 | ssl_certificate_key /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem; 62 | include /etc/nginx/tls.conf; 63 | 64 | location / { 65 | proxy_pass {{ kibana_host }}; 66 | proxy_set_header Host $host; 67 | proxy_set_header X-Real-IP $remote_addr; 68 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 69 | proxy_set_header X-Found-Cluster {{ kibana_id }}; 70 | proxy_set_header Authorization "Basic {{ kibana_basic_auth | b64encode }}"; 71 | } 72 | } 73 | 74 | # On the backend instance proxy Kibana to have a nice URL 75 | server { 76 | charset utf-8; 77 | listen 443 ssl; 78 | server_name kibana.{{ domain }}; 79 | 80 | ssl_certificate /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem; 81 | ssl_certificate_key /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem; 82 | include /etc/nginx/tls.conf; 83 | 84 | location / { 85 | proxy_pass {{ kibana_host }}; 86 | proxy_set_header Host $host; 87 | proxy_set_header X-Real-IP $remote_addr; 88 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 89 | proxy_set_header X-Found-Cluster {{ kibana_id }}; 90 | } 91 | } 92 | {% endif %} 93 | -------------------------------------------------------------------------------- /lightsail/templates/packetbeat.yml: -------------------------------------------------------------------------------- 1 | packetbeat.interfaces.device: any 2 | 3 | 4 | packetbeat.flows: 5 | timeout: 30s 6 | period: 10s 7 | 8 | 9 | packetbeat.protocols: 10 | 11 | - type: icmp 12 | enabled: true 13 | 14 | - type: dns 15 | ports: [53] 16 | 17 | - type: http 18 | ports: [80, 8080] 19 | real_ip_header: "X-Forwarded-For" 20 | 21 | {% if inventory_hostname_short == 'backend' %} 22 | - type: mysql 23 | ports: [3306] 24 | {% endif %} 25 | 26 | - type: tls 27 | ports: [443] 28 | 29 | 30 | packetbeat.procs: 31 | enabled: true 32 | monitored: 33 | 34 | - process: nginx 35 | cmdline_grep: nginx 36 | - process: java 37 | cmdline_grep: java 38 | 39 | {% if inventory_hostname_short == 'backend' %} 40 | - process: mysql 41 | cmdline_grep: mysql 42 | {% endif %} 43 | 44 | 45 | name: "{{ inventory_hostname }}" 46 | tags: ["{{ env }}", "lightsail"] 47 | 48 | 49 | processors: 50 | 51 | - add_cloud_metadata: ~ 52 | - add_host_metadata: ~ 53 | 54 | - drop_event: 55 | when: 56 | or: 57 | # Exclude pinging metrics via REST and JMX 58 | - contains.path: "/metrics/" 59 | - equals.path: "/jolokia" 60 | # Exclude pinging health 61 | - equals.path: "/health" 62 | # Exclude nginx status 63 | - equals.path: "/server-status" 64 | 65 | 66 | xpack.monitoring.enabled: true 67 | 68 | 69 | output.elasticsearch: 70 | hosts: ["{{ elasticsearch_host }}"] 71 | username: "{{ elasticsearch_user }}" 72 | password: "${ES_PWD}" 73 | 74 | 75 | setup: 76 | kibana: 77 | host: "{{ kibana_host }}" 78 | username: "{{ elasticsearch_user }}" 79 | password: "${ES_PWD}" 80 | dashboards.enabled: true 81 | -------------------------------------------------------------------------------- /lightsail/templates/security_user_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "username": "{{ attendee_user }}", 3 | "password": "{{ attendee_password }}", 4 | "roles": [ 5 | "read_dashboard", 6 | "kibana_dashboard_only_user" 7 | ], 8 | "full_name": "Dashboard User", 9 | "email": "dashboard@{{ domain }}", 10 | "metadata": {}, 11 | "enabled": true 12 | } 13 | -------------------------------------------------------------------------------- /lightsail/templates/tls.conf: -------------------------------------------------------------------------------- 1 | ssl_dhparam /etc/ssl/certs/dhparam.pem; 2 | ssl_protocols TLSv1.1 TLSv1.2; 3 | ssl_prefer_server_ciphers on; 4 | ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; 5 | ssl_ecdh_curve secp384r1; 6 | ssl_session_cache shared:SSL:10m; 7 | ssl_session_tickets off; 8 | ssl_stapling on; 9 | ssl_stapling_verify on; 10 | resolver 172.16.0.23 8.8.8.8 valid=300s; 11 | resolver_timeout 5s; 12 | -------------------------------------------------------------------------------- /lightsail/templates/urls.txt: -------------------------------------------------------------------------------- 1 | https://{{ domain }}/ 2 | https://{{ domain }}/good 3 | https://{{ domain }}/generate 4 | https://{{ domain }}/good?name=Philipp 5 | https://{{ domain }}/add?name=Philipp 6 | https://{{ domain }}/search?q=Philipp 7 | https://{{ domain }}/bad 8 | https://{{ domain }}/null 9 | https://{{ domain }}/search -------------------------------------------------------------------------------- /lightsail/terraform.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | # Credentials are defined in the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY 3 | region = "${var.region}" 4 | } 5 | 6 | 7 | # Create the SSH key pair 8 | resource "aws_lightsail_key_pair" "microservice_monitoring_key_pair" { 9 | name = "microservice_monitoring_key_pair" 10 | public_key = "${file("~/.ssh/id_rsa.pub")}" 11 | } 12 | 13 | 14 | # Create the backend instance and its DNS entry 15 | resource "aws_lightsail_instance" "backend" { 16 | name = "backend" 17 | availability_zone = "${var.region}b" 18 | blueprint_id = "${var.operating_system}" 19 | bundle_id = "${var.size}" 20 | key_pair_name = "microservice_monitoring_key_pair" 21 | depends_on = ["aws_lightsail_key_pair.microservice_monitoring_key_pair"] 22 | } 23 | resource "aws_route53_record" "backend" { 24 | zone_id = "${var.zone_id}" 25 | name = "backend.${var.domain}" 26 | type = "A" 27 | ttl = "60" 28 | records = ["${aws_lightsail_instance.backend.public_ip_address}"] 29 | } 30 | resource "aws_route53_record" "kibana" { 31 | zone_id = "${var.zone_id}" 32 | name = "kibana.${var.domain}" 33 | type = "A" 34 | alias { 35 | name = "backend.${var.domain}" 36 | zone_id = "${var.zone_id}" 37 | evaluate_target_health = false 38 | } 39 | depends_on = ["aws_route53_record.backend"] 40 | } 41 | resource "aws_route53_record" "dashboard" { 42 | zone_id = "${var.zone_id}" 43 | name = "dashboard.${var.domain}" 44 | type = "A" 45 | alias { 46 | name = "backend.${var.domain}" 47 | zone_id = "${var.zone_id}" 48 | evaluate_target_health = false 49 | } 50 | depends_on = ["aws_route53_record.backend"] 51 | } 52 | 53 | 54 | # Create the frontend instance and its DNS entries 55 | resource "aws_lightsail_instance" "frontend" { 56 | name = "frontend" 57 | availability_zone = "${var.region}a" 58 | blueprint_id = "${var.operating_system}" 59 | bundle_id = "${var.size}" 60 | key_pair_name = "microservice_monitoring_key_pair" 61 | depends_on = ["aws_lightsail_key_pair.microservice_monitoring_key_pair"] 62 | } 63 | resource "aws_route53_record" "frontend" { 64 | zone_id = "${var.zone_id}" 65 | name = "frontend.${var.domain}" 66 | type = "A" 67 | ttl = "60" 68 | records = ["${aws_lightsail_instance.frontend.public_ip_address}"] 69 | } 70 | resource "aws_route53_record" "apex" { 71 | zone_id = "${var.zone_id}" 72 | name = "${var.domain}" 73 | type = "A" 74 | alias { 75 | name = "frontend.${var.domain}" 76 | zone_id = "${var.zone_id}" 77 | evaluate_target_health = false 78 | } 79 | depends_on = ["aws_route53_record.frontend"] 80 | } 81 | resource "aws_route53_record" "www" { 82 | zone_id = "${var.zone_id}" 83 | name = "www.${var.domain}" 84 | type = "A" 85 | alias { 86 | name = "frontend.${var.domain}" 87 | zone_id = "${var.zone_id}" 88 | evaluate_target_health = false 89 | } 90 | depends_on = ["aws_route53_record.frontend"] 91 | } 92 | -------------------------------------------------------------------------------- /lightsail/variables.tf: -------------------------------------------------------------------------------- 1 | # Default instance size 2 | # Options: nano_1_0, micro_1_0, small_1_0, medium_1_0, large_1_0 3 | # Override: -var 'size=your-size' 4 | variable "size" { 5 | default = "small_1_0" 6 | } 7 | 8 | 9 | # Default AWS region 10 | # Options: eu-central-1, eu-west-1, eu-west-2, us-east-1, us-east-2, us-west-2 for Lightsail 11 | # Override: -var 'region=your-region' 12 | variable "region" { 13 | default = "eu-west-1" 14 | } 15 | 16 | 17 | # Default domain 18 | # Options: You need to use your own domain that you've registered in Route53. 19 | # Override: -var 'domain=your-domain.com' 20 | variable "domain" { 21 | default = "xeraa.wtf" 22 | } 23 | 24 | 25 | # Zone ID of the domain, no default 26 | # Options: You should provide the Zone ID of the domain in the environment variable TF_VAR_zone_id 27 | # Override: -var 'zone_id=XXXXXXXXXXXXX' 28 | variable "zone_id" {} 29 | 30 | 31 | # Operating system on AWS Lightsail 32 | # Options: Only change this at your own risk; it will probably break things. 33 | # Override: -var 'operating_system=ubuntu_16_04' 34 | variable "operating_system" { 35 | default = "ubuntu_18_04" 36 | } 37 | -------------------------------------------------------------------------------- /lightsail/variables.yml: -------------------------------------------------------------------------------- 1 | # Version to install 2 | elastic_version: 7.2.0 3 | elastic_download: https://artifacts.elastic.co 4 | apm_java_version: 1.7.0 5 | 6 | 7 | # Elastic Cloud credentials 8 | elasticsearch_host: "{{ lookup('env','ELASTICSEARCH_HOST') }}" 9 | elasticsearch_user: "{{ lookup('env','ELASTICSEARCH_USER') }}" 10 | elasticsearch_password: "{{ lookup('env','ELASTICSEARCH_PASSWORD') }}" 11 | kibana_host: "{{ lookup('env','KIBANA_HOST') }}" 12 | kibana_id: "{{ lookup('env','KIBANA_ID') }}" 13 | 14 | 15 | # Setup of the infrastructure 16 | env: production 17 | domain: xeraa.wtf 18 | backend_server: https://backend.{{ domain }} 19 | frontend_server: https://frontend.{{ domain }} 20 | apm_server: "{{ lookup('env','APM_HOST') }}" 21 | apm_secret: "{{ lookup('env','APM_TOKEN') }}" 22 | 23 | 24 | # MySQL config 25 | mysql_server: localhost 26 | mysql_user: "{{ lookup('env','ELASTICSEARCH_USER') }}" 27 | mysql_password: "{{ lookup('env','ELASTICSEARCH_PASSWORD') }}" 28 | mysql_database: person 29 | 30 | 31 | # Credentials for Kibana dashboard-only mode 32 | attendee_user: dashboard 33 | attendee_password: secret 34 | -------------------------------------------------------------------------------- /local/.env: -------------------------------------------------------------------------------- 1 | ELASTIC_VERSION=7.1.0 2 | ELASTIC_SECURITY=true 3 | ELASTIC_PASSWORD=changeme 4 | ELASTIC_APM_AGENT_VERSION=1.6.1 5 | # Add -d to the next line if you want to debug the wait_for script 6 | DEBUG_WAIT_FOR= 7 | -------------------------------------------------------------------------------- /local/config/apm-server.yml: -------------------------------------------------------------------------------- 1 | apm-server: 2 | host: "0.0.0.0:8200" 3 | 4 | output.elasticsearch: 5 | hosts: ["elasticsearch:9200"] 6 | username: "${elasticsearch.username}" 7 | password: "${elasticsearch.password}" 8 | 9 | indices: 10 | - index: "apm-%{[observer.version]}-sourcemap" 11 | when.contains: 12 | processor.event: "sourcemap" 13 | 14 | - index: "apm-%{[observer.version]}-error-%{+yyyy.MM.dd}" 15 | when.contains: 16 | processor.event: "error" 17 | 18 | - index: "apm-%{[observer.version]}-transaction-%{+yyyy.MM.dd}" 19 | when.contains: 20 | processor.event: "transaction" 21 | 22 | - index: "apm-%{[observer.version]}-span-%{+yyyy.MM.dd}" 23 | when.contains: 24 | processor.event: "span" 25 | 26 | - index: "apm-%{[observer.version]}-metric-%{+yyyy.MM.dd}" 27 | when.contains: 28 | processor.event: "metric" 29 | 30 | - index: "apm-%{[observer.version]}-onboarding-%{+yyyy.MM.dd}" 31 | when.contains: 32 | processor.event: "onboarding" 33 | 34 | xpack.monitoring.enabled: true 35 | -------------------------------------------------------------------------------- /local/config/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.config: 2 | modules: 3 | path: ${path.config}/modules.d/*.yml 4 | reload.enabled: false 5 | 6 | filebeat.inputs: 7 | - type: log 8 | paths: 9 | - /mnt/backend-log/backend.json 10 | fields_under_root: true 11 | fields: 12 | application: java 13 | layer: backend 14 | json.keys_under_root: true 15 | json.add_error_key: true 16 | - type: log 17 | paths: 18 | - /mnt/frontend-log/frontend.json 19 | fields_under_root: true 20 | fields: 21 | application: java 22 | layer: frontend 23 | json.keys_under_root: true 24 | json.add_error_key: true 25 | 26 | # Module: mysql 27 | # Docs: https://www.elastic.co/guide/en/beats/filebeat/7.1/filebeat-module-mysql.html 28 | 29 | filebeat.modules: 30 | - module: system 31 | - module: mysql 32 | error: 33 | enabled: true 34 | var.paths: ["/mnt/mysql-log/error.log*"] 35 | slowlog: 36 | enabled: true 37 | var.paths: ["/mnt/mysql-log/mysql-slow.log*"] 38 | 39 | processors: 40 | - add_cloud_metadata: ~ 41 | - add_docker_metadata: ~ 42 | 43 | output.elasticsearch: 44 | hosts: ['${ELASTICSEARCH_HOSTS:elasticsearch:9200}'] 45 | username: '${ELASTICSEARCH_USERNAME:}' 46 | password: '${ELASTICSEARCH_PASSWORD:}' 47 | 48 | xpack.monitoring.enabled: true 49 | 50 | setup.kibana: 51 | host: "${KIBANA_HOSTS:kibana:5601}" 52 | username: '${ELASTICSEARCH_USERNAME:}' 53 | password: '${ELASTICSEARCH_PASSWORD:}' 54 | setup.dashboards.enabled: true 55 | setup.dashboards.retry.enabled: true 56 | setup.dashboards.retry.interval: 5s 57 | -------------------------------------------------------------------------------- /local/config/heartbeat.yml: -------------------------------------------------------------------------------- 1 | heartbeat.monitors: 2 | - type: icmp 3 | schedule: '@every 5s' 4 | hosts: 5 | - elasticsearch 6 | - kibana 7 | - mysql 8 | - java-backend 9 | - java-frontend 10 | - type: http 11 | schedule: '@every 5s' 12 | urls: 13 | - 'http://${ELASTICSEARCH_USERNAME:elastic}:${ELASTICSEARCH_PASSWORD:}@elasticsearch:9200' 14 | - 'http://kibana:5601' 15 | - 'http://java-backend:8081/jolokia' 16 | - 'http://java-frontend:8080' 17 | - type: tcp 18 | enabled: true 19 | schedule: '@every 5s' 20 | hosts: ["tcp://mysql:3306"] 21 | 22 | processors: 23 | - add_cloud_metadata: ~ 24 | - add_docker_metadata: ~ 25 | 26 | output.elasticsearch: 27 | hosts: ['${ELASTICSEARCH_HOSTS:elasticsearch:9200}'] 28 | username: '${ELASTICSEARCH_USERNAME:}' 29 | password: '${ELASTICSEARCH_PASSWORD:}' 30 | 31 | xpack.monitoring.enabled: true 32 | 33 | #setup.kibana: 34 | # host: "${KIBANA_HOSTS:kibana:5601}" 35 | # username: '${ELASTICSEARCH_USERNAME:}' 36 | # password: '${ELASTICSEARCH_PASSWORD:}' 37 | #setup.dashboards.enabled: true 38 | #setup.dashboards.retry.enabled: true 39 | #setup.dashboards.retry.interval: 5s 40 | -------------------------------------------------------------------------------- /local/config/metricbeat.yml: -------------------------------------------------------------------------------- 1 | metricbeat.modules: 2 | 3 | - module: system 4 | metricsets: 5 | - cpu 6 | - load 7 | - core 8 | - diskio 9 | - filesystem 10 | - fsstat 11 | - memory 12 | - network 13 | - process 14 | - socket 15 | enabled: true 16 | period: 10s 17 | processes: ['.*'] 18 | cgroups: true 19 | 20 | - module: jolokia 21 | metricsets: ["jmx"] 22 | hosts: ["localhost:8080", "localhost:8081"] 23 | username: admin 24 | password: secret 25 | namespace: "metrics" 26 | jmx.mappings: 27 | - mbean: "java.lang:type=Runtime" 28 | attributes: 29 | - attr: Uptime 30 | field: uptime 31 | - mbean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" 32 | attributes: 33 | - attr: CollectionTime 34 | field: gc.cms_collection_time 35 | - attr: CollectionCount 36 | field: gc.cms_collection_count 37 | - mbean: "java.lang:type=Memory" 38 | attributes: 39 | - attr: HeapMemoryUsage 40 | field: memory.heap_usage 41 | - attr: NonHeapMemoryUsage 42 | field: memory.non_heap_usage 43 | jmx.instance: "{{ inventory_hostname }}" 44 | 45 | - module: docker 46 | metricsets: ["cpu", "info", "memory", "network", "diskio", "container"] 47 | hosts: ["unix:///var/run/docker.sock"] 48 | enabled: true 49 | period: 10s 50 | 51 | processors: 52 | - add_cloud_metadata: ~ 53 | - add_docker_metadata: ~ 54 | 55 | output.elasticsearch: 56 | hosts: ['${ELASTICSEARCH_HOSTS:elasticsearch:9200}'] 57 | username: '${ELASTICSEARCH_USERNAME:}' 58 | password: '${ELASTICSEARCH_PASSWORD:}' 59 | 60 | xpack.monitoring.enabled: true 61 | 62 | setup.kibana: 63 | host: "${KIBANA_HOSTS:kibana:5601}" 64 | username: '${ELASTICSEARCH_USERNAME:}' 65 | password: '${ELASTICSEARCH_PASSWORD:}' 66 | setup.dashboards.enabled: true 67 | setup.dashboards.retry.enabled: true 68 | setup.dashboards.retry.interval: 5s 69 | -------------------------------------------------------------------------------- /local/config/packetbeat.yml: -------------------------------------------------------------------------------- 1 | packetbeat.interfaces.device: any 2 | 3 | packetbeat.flows: 4 | timeout: 30s 5 | period: 10s 6 | 7 | packetbeat.protocols.dns: 8 | ports: [53] 9 | include_authorities: true 10 | include_additionals: true 11 | 12 | packetbeat.protocols.http: 13 | ports: [80, 5601, 9200, 8080, 8081, 5000, 8002] 14 | 15 | packetbeat.protocols.memcache: 16 | ports: [11211] 17 | 18 | packetbeat.protocols.mysql: 19 | ports: [3306] 20 | 21 | packetbeat.protocols.pgsql: 22 | ports: [5432] 23 | 24 | packetbeat.protocols.redis: 25 | ports: [6379] 26 | 27 | packetbeat.protocols.thrift: 28 | ports: [9090] 29 | 30 | packetbeat.protocols.mongodb: 31 | ports: [27017] 32 | 33 | packetbeat.protocols.cassandra: 34 | ports: [9042] 35 | 36 | processors: 37 | - add_cloud_metadata: ~ 38 | - add_docker_metadata: ~ 39 | 40 | output.elasticsearch: 41 | hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' 42 | username: '${ELASTICSEARCH_USERNAME:}' 43 | password: '${ELASTICSEARCH_PASSWORD:}' 44 | 45 | xpack.monitoring.enabled: true 46 | 47 | setup.kibana: 48 | host: "${KIBANA_HOSTS:kibana:5601}" 49 | username: '${ELASTICSEARCH_USERNAME:}' 50 | password: '${ELASTICSEARCH_PASSWORD:}' 51 | setup.dashboards.enabled: true 52 | setup.dashboards.retry.enabled: true 53 | setup.dashboards.retry.interval: 5s 54 | -------------------------------------------------------------------------------- /local/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3' 3 | services: 4 | 5 | elasticsearch: 6 | image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTIC_VERSION 7 | container_name: elasticsearch 8 | environment: 9 | - cluster.name=microservice-monitoring 10 | - bootstrap.memory_lock=true 11 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 12 | - discovery.type=single-node 13 | - cluster.routing.allocation.disk.threshold_enabled=false 14 | - ELASTIC_PASSWORD=$ELASTIC_PASSWORD 15 | - xpack.security.enabled=$ELASTIC_SECURITY 16 | - xpack.monitoring.collection.enabled=true 17 | ulimits: 18 | memlock: 19 | soft: -1 20 | hard: -1 21 | ports: 22 | - 9200:9200 23 | healthcheck: 24 | test: ["CMD", "curl","-s" ,"-f", "http://elastic:$ELASTIC_PASSWORD@localhost:9200/_cat/health"] 25 | networks: ['stack'] 26 | 27 | kibana: 28 | image: docker.elastic.co/kibana/kibana:$ELASTIC_VERSION 29 | environment: 30 | - ELASTICSEARCH_USERNAME=elastic 31 | - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD 32 | container_name: kibana 33 | volumes: 34 | - ./scripts:/mnt/scripts:ro 35 | command: sh -c '/mnt/scripts/wait-for $DEBUG_WAIT_FOR http://elastic:$ELASTIC_PASSWORD@elasticsearch:9200 && /mnt/scripts/kibana.sh' 36 | ports: ['127.0.0.1:5601:5601'] 37 | healthcheck: 38 | test: ["CMD", "curl", "-s", "-f", "http://elastic:$ELASTIC_PASSWORD@localhost:5601/api/status"] 39 | retries: 6 40 | restart: on-failure 41 | networks: ['stack'] 42 | depends_on: ['elasticsearch'] 43 | 44 | filebeat: 45 | image: docker.elastic.co/beats/filebeat:$ELASTIC_VERSION 46 | user: root 47 | environment: 48 | - ELASTICSEARCH_USERNAME=elastic 49 | - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD 50 | container_name: filebeat 51 | # If the host system has logs at "/var/log", mount them at "/mnt/log" 52 | # inside the container, where Filebeat can find them. 53 | volumes: 54 | - ./scripts:/mnt/scripts:ro 55 | - /var/run/docker.sock:/var/run/docker.sock:ro 56 | - /var/log:/mnt/log:ro 57 | - backend-log:/mnt/backend-log:ro 58 | - frontend-log:/mnt/frontend-log:ro 59 | - mysql-log:/mnt/mysql-log:ro 60 | # Provide a custom Filebeat configuration 61 | - ./config/filebeat.yml:/usr/share/filebeat/filebeat.yml 62 | command: sh -c '/mnt/scripts/wait-for $DEBUG_WAIT_FOR http://elastic:$ELASTIC_PASSWORD@kibana:5601/api/status && filebeat -e' 63 | networks: ['stack'] 64 | # depends_on: ['kibana'] 65 | restart: on-failure 66 | 67 | heartbeat: 68 | image: docker.elastic.co/beats/heartbeat:$ELASTIC_VERSION 69 | user: root 70 | environment: 71 | - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 72 | - ELASTICSEARCH_USERNAME=elastic 73 | - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD 74 | - KIBANA_HOSTS=http://kibana:5601 75 | container_name: heartbeat 76 | volumes: 77 | - ./scripts:/mnt/scripts:ro 78 | - /var/run/docker.sock:/var/run/docker.sock:ro 79 | # Provide a custom Heartbeat configuration 80 | - ./config/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml 81 | command: sh -c '/mnt/scripts/wait-for $DEBUG_WAIT_FOR http://elastic:$ELASTIC_PASSWORD@kibana:5601/api/status && heartbeat -e' 82 | networks: ['stack'] 83 | depends_on: ['kibana'] 84 | restart: on-failure 85 | 86 | metricbeat: 87 | image: docker.elastic.co/beats/metricbeat:$ELASTIC_VERSION 88 | user: root 89 | environment: 90 | - ELASTICSEARCH_USERNAME=elastic 91 | - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD 92 | container_name: metricbeat 93 | networks: ['stack'] 94 | # The commented sections below enable Metricbeat to monitor the Docker host, 95 | # rather than the Metricbeat container. It's problematic with Docker for 96 | # Windows, however, since "/proc", "/sys", etc. don't exist on Windows. 97 | volumes: 98 | - ./scripts:/mnt/scripts:ro 99 | - /var/run/docker.sock:/var/run/docker.sock:ro 100 | - /proc:/hostfs/proc:ro 101 | - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro 102 | - /:/hostfs:ro 103 | # Provide a custom Metricbeat configuration 104 | - ./config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml 105 | command: sh -c '/mnt/scripts/wait-for $DEBUG_WAIT_FOR http://elastic:$ELASTIC_PASSWORD@kibana:5601/api/status && metricbeat -e' 106 | networks: ['stack'] 107 | depends_on: ['kibana'] 108 | restart: on-failure 109 | 110 | packetbeat: 111 | image: docker.elastic.co/beats/packetbeat:$ELASTIC_VERSION 112 | user: root 113 | environment: 114 | # Since we did that, Packetbeat is not part of the "stack" Docker network 115 | # that the other containers are connected to, and thus can't resolve the 116 | # hostname "elasticsearch". Instead, we'll tell it to find Elasticsearch 117 | # on "localhost", which is the Docker host machine in this context. 118 | - ELASTICSEARCH_HOSTS=localhost:9200 119 | - ELASTICSEARCH_USERNAME=elastic 120 | - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD 121 | - KIBANA_HOSTS=localhost:5601 122 | container_name: packetbeat 123 | volumes: 124 | - ./scripts:/mnt/scripts:ro 125 | - /var/run/docker.sock:/var/run/docker.sock:ro 126 | # Provide a custom Packetbeat configuration 127 | - ./config/packetbeat.yml:/usr/share/packetbeat/packetbeat.yml 128 | # Packetbeat needs some elevated privileges capture network traffic. 129 | # We'll grant them with POSIX capabilities. 130 | cap_add: ['NET_RAW', 'NET_ADMIN'] 131 | # Use "host mode" networking to allow Packetbeat to capture traffic from 132 | # real network interface on the host, rather than being isolated to the 133 | # container's virtual interface. 134 | command: sh -c '/mnt/scripts/wait-for $DEBUG_WAIT_FOR http://elastic:$ELASTIC_PASSWORD@localhost:5601/api/status && packetbeat -e' 135 | network_mode: host 136 | depends_on: ['kibana'] 137 | restart: on-failure 138 | 139 | apm-server: 140 | image: docker.elastic.co/apm/apm-server:$ELASTIC_VERSION 141 | environment: 142 | - ELASTICSEARCH_USERNAME=elastic 143 | - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD 144 | container_name: apm-server 145 | volumes: 146 | - ./scripts:/mnt/scripts:ro 147 | - ./config/apm-server.yml:/usr/share/apm-server/apm-server.yml 148 | command: sh -c '/mnt/scripts/wait-for $DEBUG_WAIT_FOR http://elastic:$ELASTIC_PASSWORD@elasticsearch:9200 && /mnt/scripts/apm.sh' 149 | ports: ['8200:8200'] 150 | networks: ['stack'] 151 | depends_on: ['elasticsearch'] 152 | restart: on-failure 153 | 154 | mysql: 155 | image: mysql:5 156 | command: --default-authentication-plugin=mysql_native_password 157 | restart: always 158 | environment: 159 | - MYSQL_ROOT_PASSWORD=$ELASTIC_PASSWORD 160 | - MYSQL_USER=elastic 161 | - MYSQL_PASSWORD=$ELASTIC_PASSWORD 162 | - MYSQL_DATABASE=person 163 | container_name: mysql 164 | volumes: 165 | - mysql-log:/var/log/mysql 166 | networks: ['stack'] 167 | ports: ['3306:3306'] 168 | restart: on-failure 169 | 170 | apm-agent-download: 171 | image: alpine:latest 172 | container_name: apm-agent-download 173 | command: sh -c 'cd /root/apm-agent ; wget https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_APM_AGENT_VERSION/elastic-apm-agent-$ELASTIC_APM_AGENT_VERSION.jar' 174 | volumes: 175 | - apm-agent:/root/apm-agent 176 | 177 | java-backend: 178 | image: openjdk:8-stretch 179 | restart: always 180 | environment: 181 | - DATABASE_SERVER=mysql 182 | - DATABASE_USERNAME=elastic 183 | - DATABASE_PASSWORD=$ELASTIC_PASSWORD 184 | - DATABASE_NAME=person 185 | - DATABASE_PORT=3306 186 | volumes: 187 | - backend-log:/mnt/project 188 | - apm-agent:/mnt/project/apm-agent:ro 189 | - ../java/backend/build/libs:/mnt/project/app:ro 190 | container_name: java-backend 191 | command: sh -c 'cd /mnt/project ; java -jar -javaagent:/mnt/project/apm-agent/elastic-apm-agent-$ELASTIC_APM_AGENT_VERSION.jar -Delastic.apm.service_name=backend -Delastic.apm.server_urls=http://apm-server:8200 -Delastic.apm.application_packages=net.xeraa.backend /mnt/project/app/backend.jar' 192 | ports: ['8081:8081'] 193 | networks: ['stack'] 194 | 195 | java-frontend: 196 | image: openjdk:8-stretch 197 | restart: always 198 | environment: 199 | - APP_BACKEND=http://java-backend:8081 200 | volumes: 201 | - frontend-log:/mnt/project 202 | - apm-agent:/mnt/project/apm-agent:ro 203 | - ../java/frontend/build/libs:/mnt/project/app:ro 204 | container_name: java-frontend 205 | command: sh -c 'cd /mnt/project ; java -jar -javaagent:/mnt/project/apm-agent/elastic-apm-agent-$ELASTIC_APM_AGENT_VERSION.jar -Delastic.apm.service_name=frontend -Delastic.apm.server_urls=http://apm-server:8200 -Delastic.apm.application_packages=net.xeraa.frontend /mnt/project/app/frontend.jar' 206 | ports: ['8080:8080'] 207 | networks: ['stack'] 208 | 209 | networks: 210 | stack: {} 211 | 212 | volumes: 213 | backend-log: {} 214 | frontend-log: {} 215 | mysql-log: {} 216 | apm-agent: {} 217 | 218 | 219 | -------------------------------------------------------------------------------- /local/scripts/apm.sh: -------------------------------------------------------------------------------- 1 | apm-server keystore create 2 | echo "$ELASTICSEARCH_USERNAME" | apm-server keystore add elasticsearch.username --stdin 3 | echo "$ELASTICSEARCH_PASSWORD" | apm-server keystore add elasticsearch.password --stdin 4 | apm-server -e 5 | -------------------------------------------------------------------------------- /local/scripts/kibana.sh: -------------------------------------------------------------------------------- 1 | bin/kibana-keystore create 2 | echo "$ELASTICSEARCH_USERNAME" | bin/kibana-keystore add elasticsearch.username --stdin 3 | echo "$ELASTICSEARCH_PASSWORD" | bin/kibana-keystore add elasticsearch.password --stdin 4 | bin/kibana 5 | -------------------------------------------------------------------------------- /local/scripts/wait-for: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | CODE=200 4 | DEBUG=0 5 | 6 | echoerr() { 7 | if [ "$DEBUG" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi 8 | } 9 | 10 | usage() { 11 | exitcode="$1" 12 | cat << USAGE >&2 13 | Usage: 14 | $cmdname url [-t timeout] [-- command args] 15 | -d | --debug Debug mode 16 | -c CODE | --http_code Expected http code (200 by default) 17 | -- COMMAND ARGS Execute command with args after the test finishes 18 | USAGE 19 | exit "$exitcode" 20 | } 21 | 22 | wait_for() { 23 | ret=0 24 | while [ "$ret" != "$CODE" ]; do 25 | ret=$(curl -s -o /dev/null -w '%{http_code}' $URL) 26 | if [ "$DEBUG" -ne 0 ]; then echo "Waiting for $URL to return $CODE... Got $ret"; fi 27 | sleep 1 28 | done 29 | exit 0 30 | } 31 | 32 | while [ $# -gt 0 ] 33 | do 34 | case "$1" in 35 | http*://* ) 36 | URL=$1 37 | shift 1 38 | ;; 39 | -d | --debug) 40 | DEBUG=1 41 | shift 1 42 | ;; 43 | -c) 44 | CODE="$2" 45 | if [ "$CODE" = "" ]; then break; fi 46 | shift 2 47 | ;; 48 | --http_code=*) 49 | CODE="${1#*=}" 50 | shift 1 51 | ;; 52 | --) 53 | shift 54 | break 55 | ;; 56 | --help) 57 | usage 0 58 | ;; 59 | *) 60 | echoerr "Unknown argument: $1" 61 | usage 1 62 | ;; 63 | esac 64 | done 65 | 66 | if [ "$URL" = "" ]; then 67 | echoerr "Error: you need to provide a url to test." 68 | usage 2 69 | fi 70 | 71 | wait_for "$@" 72 | 73 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Microservice Monitoring 2 | 3 | Monitor logs, metrics, pings, and traces of your distributed (micro-) services. There are also [slides](https://speakerdeck.com/xeraa/360-degrees-monitoring-of-your-microservices) walking you through the features of this repository. 4 | 5 | 6 | 7 | ## Features 8 | 9 | * **X-Pack Monitoring**: Start the overview page to show the systems we are using for monitoring. 10 | * **Metricbeat System**: Show the *[Metricbeat System] Overview* dashboard in Kibana and then switch to *[Metricbeat System] Host overview*. 11 | * **Infrastructure UI**: As an alternative view show the Infrastructure UI and explain how this will scale much better. 12 | * **Packetbeat**: Show the *[Packetbeat] Overview*, *[Packetbeat] Flows*, *[Packetbeat] MySQL performance*, and *[Packetbeat] HTTP* dashboard, let attendees access the various URLs and see the corresponding graphs. In *Discover* you can point out the `proc` enrichment for nginx, Java, and MySQL. Optionally show the *[Packetbeat] TLS Sessions* and *[Packetbeat] DNS Tunneling* dashboards as well. 13 | * **Filebeat modules**: Show the *[Filebeat Nginx] Access and error logs*, *[Filebeat MySQL] Overview* (might need a wider time window to see anything useful), *[Filebeat System] Syslog dashboard*, *[Filebeat System] SSH login attempts*. 14 | * **Filebeat**: Let attendees hit */good* with a parameter and point out the MDC logging under `json.name` and the context view for one log message. Let attendees hit */bad* and */null* to show the stacktrace both in the JSON log file and in Kibana by filtering down on `application:java` and `json.severity: ERROR`. Also point out the cloud `meta.*` and `host.*` information. And show the `json.stack_hash`, which you can use for visualizations too. 15 | 16 | ![](img/stacktraces.png) 17 | 18 | * **Auditbeat**: Show changes to the */opt/* folder with the *[Auditbeat File Integrity] Overview* dashboard. 19 | * **Heartbeat**: Run Heartbeat and show the *Heartbeat HTTP monitoring* dashboard in Kibana, then stop and start the frontend application with `ansible-playbook restart_frontend.yml` or do it manually and see the change. 20 | * **Metricbeat**: Show the *[Metricbeat Nginx] Overview* and *[Metricbeat MySQL] Overview* dashboards. 21 | * **Metricbeat HTTP**: Show */health* and */metrics* with cURL (credentials are `admin` and `secret`). Then collect the same information with Metricbeat's HTTP module and show it in Kibana's Discover tab. 22 | * **Metricbeat JMX**: Display the same */health* and */metrics* data and its collection through JMX. 23 | * **Visual Builder**: Build a more advanced visualization with the Time Series Visual Builder, for example to show the heap usage in percent by calculating the average of `jolokia.metrics.memory.heap_usage.used` divided by the max of `jolokia.metrics.memory.heap_usage.max`. 24 | 25 | ![](img/heap-usage.png) 26 | 27 | * **Annotations**: Include the deployment *events* as an annotations. 28 | 29 | ![](img/heap-usage-annotated.png) 30 | 31 | * **APM**: Show the traces so far, point out the MySQL queries (currently on the backend instance only), and where things are slow or throwing errors. If there is not enough activity on the instances, call `./ab.sh` on the monitor instance. Also show errors and metrics. 32 | * **Kibana Dashboard Mode**: Point attendees to the Kibana instance to let them play around on their own. 33 | 34 | 35 | 36 | ## Setup 37 | 38 | If the network connection is decent, show it on [Amazon Lightsail](https://amazonlightsail.com). Otherwise fall back to the local setup and have all the dependencies downloaded in advance. 39 | 40 | 41 | 42 | ### Lightsail 43 | 44 | Make sure you have run this before the demo, because some steps take time and require a decent internet connection. 45 | 46 | 1. Make sure you have your AWS account set up, access key created, and added as environment variables in `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. Protip: Use [https://github.com/sorah/envchain](https://github.com/sorah/envchain) to keep your environment variables safe. 47 | 1. Create the Elastic Cloud instance with the same version as specified in *variables.yml*'s `elastic_version`, enable Kibana, enable APM, and set the environment variables with the values for `ELASTICSEARCH_HOST`, `ELASTICSEARCH_USER`, `ELASTICSEARCH_PASSWORD`, `KIBANA_HOST`, `KIBANA_ID`, `APM_HOST`, and `APM_TOKEN`. 48 | 1. Change into the *lightsail/* directory. 49 | 1. Change the settings to a domain you have registered under Route53 in *inventory*, *variables.tf*, and *variables.yml*. Set the Hosted Zone for that domain and export the Zone ID under the environment variable `TF_VAR_zone_id`. If you haven't created the Hosted Zone yet, you should set it up in the AWS Console first and then set the environment variable. 50 | 1. If you haven't installed the AWS plugin for Terraform, get it with `terraform init` first. Then create the keypair, DNS settings, and instances with `terraform apply`. 51 | 1. Open HTTPS on the network configuration on both instances (waiting for this [Terraform issue](https://github.com/terraform-providers/terraform-provider-aws/issues/700) to automate these steps). 52 | 1. Apply the base configuration to all instances with `ansible-playbook configure_all.yml`. 53 | 1. Apply the instance specific configuration with `ansible-playbook configure_backend.yml`. 54 | 1. Deploy the JARs with `ansible-playbook deploy_backend.yml` and `ansible-playbook deploy_frontend.yml` (Ansible is also building them). 55 | 56 | When you are done, remove the instances, DNS settings, and key with `terraform destroy`. 57 | 58 | 59 | 60 | ### Workshop 61 | 62 | Very similar to the Lightsail setup above. The main difference is that everything is running on one instance and you need to open the port 5601 for Kibana (Elasticsearch, APM,... are only accessible on localhost) and 88 if you want to include the PHP examples. 63 | 64 | Credentials: 65 | 66 | * SSH: `ssh elastic-admin@workshop-.xeraa.wtf` elastic-admin / secret 67 | * Elasticsearch: `http://localhost:9200` admin / secret 68 | * Kibana: `http://workshop-.xeraa.wtf:5601` admin / secret 69 | 70 | 71 | 72 | ### Local 73 | 74 | Make sure you have run this before the demo, because some steps take time and require a decent internet connection. 75 | 76 | 1. Change into the *local/* directory. 77 | 1. Run `docker-compose up`, which will bring up Elasticsearch, Kibana, and all the Beats. 78 | 1. 79 | 1. 80 | 1. Run the Java applications from their directories with `./gradle bootRun`. 81 | 1. 82 | 83 | When you are done, stop the Java applications and remove the Docker setup with `docker-compose down -v`. 84 | 85 | 86 | 87 | ## Todo 88 | 89 | * Test and document the Docker-Compose flow 90 | * Redo the story since the content changed a bit over time 91 | * https://tech.paulcz.net/blog/spring-into-kubernetes-part-1/ 92 | * https://codecentric.github.io/chaos-monkey-spring-boot/ 93 | * Micrometer / http://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-metrics.html 94 | * Kubernetes 95 | * Functionbeat (though there are no CloudWatch metrics for Lightsail) 96 | * https://github.com/elastic/examples/blob/master/Alerting/Sample%20Watches/errors_in_logs/watch.json 97 | * Better Watch syntax as in https://github.com/elastic/examples/blob/master/Alerting/Sample%20Watches/unexpected_account_activity/watch.json 98 | -------------------------------------------------------------------------------- /workshop/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | 3 | inventory = ./inventory 4 | 5 | # Set the user globally 6 | remote_user = ubuntu 7 | -------------------------------------------------------------------------------- /workshop/configure_all.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | pre_tasks: 11 | - name: Install Python2 to make Ansible work 12 | raw: sudo apt-get update && sudo apt-get -y install python-minimal 13 | 14 | 15 | post_tasks: 16 | - include_tasks: include_event.yml 17 | vars: 18 | application: configure_all 19 | 20 | 21 | tasks: 22 | 23 | 24 | # System 25 | - name: Update and upgrade apt packages 26 | apt: upgrade=dist 27 | 28 | - name: Install NTP to avoid time drift and PIP to manage Python dependencies plus its build tools and dev tools 29 | apt: 30 | name: [ 'ntp', 'ntpdate', 'python3-pip', 'build-essential', 'libssl-dev', 'libffi-dev', 'git', 'whois', 'unzip' ] 31 | 32 | - name: Install the JDK 33 | apt: name=openjdk-8-jdk install_recommends=no 34 | 35 | 36 | # Allow SSH logins 37 | - name: Add a group for developers 38 | group: 39 | name: developers 40 | state: present 41 | 42 | - name: Add a regular user 43 | user: 44 | name: elastic-user 45 | password: "{{ssh_password }}" 46 | groups: developers 47 | shell: /bin/bash 48 | state: present 49 | 50 | - name: Create a file for that specific user only readable by them 51 | template: src=templates/secret.txt dest=/home/elastic-user/secret.txt owner=elastic-user mode=0600 52 | 53 | - name: Add a root user 54 | user: 55 | name: elastic-admin 56 | password: "{{ssh_password }}" 57 | groups: sudo 58 | shell: /bin/bash 59 | state: present 60 | 61 | - name: Allow passwordless sudo 62 | lineinfile: 63 | path: /etc/sudoers 64 | state: present 65 | regexp: '^%sudo' 66 | line: '%sudo ALL=(ALL) NOPASSWD: ALL' 67 | validate: 'visudo -cf %s' 68 | 69 | - name: Allow password based authentication for SSH 70 | lineinfile: 71 | path: /etc/ssh/sshd_config 72 | regexp: '^PasswordAuthentication' 73 | line: 'PasswordAuthentication yes' 74 | state: present 75 | 76 | - name: Allow our users to log in via SSH 77 | lineinfile: 78 | path: /etc/ssh/sshd_config 79 | regexp: '^AllowUsers' 80 | line: 'AllowUsers ubuntu elastic-user elastic-admin' 81 | state: present 82 | 83 | - name: Restart SSH 84 | service: name=ssh state=restarted 85 | 86 | 87 | # osquery 88 | - name: Add osquery's repository key 89 | apt_key: 90 | keyserver: keyserver.ubuntu.com 91 | id: 1484120AC4E9F8A1A577AEEE97A80C63C9D8B80B 92 | state: present 93 | 94 | - name: Add osquery's server repository 95 | apt_repository: 96 | repo: "deb [arch=amd64] https://osquery-packages.s3.amazonaws.com/deb deb main" 97 | state: present 98 | 99 | - name: Install osquery 100 | apt: name=osquery update_cache=true 101 | 102 | - name: Stop osquery since it sometimes need a few seconds between stop and start to work properly 103 | service: name=osqueryd state=stopped 104 | 105 | - name: Allow osquery to access the rsyslog.conf 106 | blockinfile: 107 | path: /etc/rsyslog.conf 108 | block: | 109 | template( 110 | name="OsqueryCsvFormat" 111 | type="string" 112 | string="%timestamp:::date-rfc3339,csv%,%hostname:::csv%,%syslogseverity:::csv%,%syslogfacility-text:::csv%,%syslogtag:::csv%,%msg:::csv%\n" 113 | ) 114 | *.* action(type="ompipe" Pipe="/var/osquery/syslog_pipe" template="OsqueryCsvFormat") 115 | 116 | - name: Restart rsyslog 117 | service: name=rsyslog state=restarted 118 | 119 | - name: Provide the base configuration for osquery 120 | template: src=templates/osquery.conf dest=/etc/osquery/osquery.conf 121 | 122 | - name: Change the interval of the osquery queries to 5min so we can see results quicker 123 | replace: 124 | path: "/usr/share/osquery/packs/{{ item }}.conf" 125 | regexp: '"interval" : .*' 126 | replace: '"interval" : "300",' 127 | loop: 128 | - osquery-monitoring 129 | - incident-response 130 | - it-compliance 131 | - vuln-management 132 | 133 | - name: Restart osquery and make sure it autostarts 134 | service: name=osqueryd state=restarted enabled=yes 135 | 136 | 137 | # Elasticsearch 138 | - name: Install Elasticsearch 139 | apt: deb={{ elastic_download }}/downloads/elasticsearch/elasticsearch-{{ elastic_version }}.deb 140 | 141 | - name: Stop Elasticsearch 142 | service: name=elasticsearch state=stopped 143 | 144 | - name: Change Elasticsearch's minimum memory usage 145 | lineinfile: 146 | dest: /etc/elasticsearch/jvm.options 147 | regexp: "^-Xms" 148 | line: "-Xms768m" 149 | state: present 150 | 151 | - name: Change Elasticsearch's maximum memory usage 152 | lineinfile: 153 | dest: /etc/elasticsearch/jvm.options 154 | regexp: "^-Xmx" 155 | line: "-Xmx768m" 156 | state: present 157 | 158 | - name: Do not swap 159 | lineinfile: 160 | dest: /etc/elasticsearch/elasticsearch.yml 161 | regexp: "bootstrap.memory_lock" 162 | line: "bootstrap.memory_lock: true" 163 | state: present 164 | 165 | - name: Enable monitoring of the Elastic Stack components 166 | lineinfile: 167 | dest: /etc/elasticsearch/elasticsearch.yml 168 | regexp: "xpack.monitoring.collection.enabled" 169 | line: "xpack.monitoring.collection.enabled: true" 170 | state: present 171 | 172 | - name: Enable security 173 | lineinfile: 174 | dest: /etc/elasticsearch/elasticsearch.yml 175 | regexp: "xpack.security.enabled" 176 | line: "xpack.security.enabled: true" 177 | state: present 178 | 179 | - name: Collect the list of installed Elasticsearch plugins 180 | command: /usr/share/elasticsearch/bin/elasticsearch-plugin list 181 | register: elasticsearch_plugins 182 | 183 | - name: Add the ingest-geoip plugin to Elasticsearch 184 | command: /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip --batch 185 | when: "'ingest-geoip' not in elasticsearch_plugins.stdout" 186 | 187 | - name: Add the ingest-user-agent plugin to Elasticsearch 188 | command: /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-user-agent --batch 189 | when: "'ingest-user-agent' not in elasticsearch_plugins.stdout" 190 | 191 | - name: Remove a user for Elasticsearch (just in case we want to make any changes) 192 | command: /usr/share/elasticsearch/bin/elasticsearch-users userdel {{ elasticsearch_user }} 193 | ignore_errors: true 194 | 195 | - name: Create a user for Elasticsearch 196 | command: /usr/share/elasticsearch/bin/elasticsearch-users useradd {{ elasticsearch_user }} -p {{ elasticsearch_password }} -r superuser 197 | 198 | - name: Restart Elasticsearch and make sure it autostarts 199 | service: name=elasticsearch state=restarted enabled=yes 200 | 201 | - name: Wait for Elasticsearch to become available 202 | wait_for: 203 | port: 9200 204 | delay: 5 205 | 206 | - name: Active the 30 day X-Pack trial 207 | uri: 208 | url: "{{elasticsearch_host}}_xpack/license/start_trial?acknowledge=true" 209 | method: POST 210 | user: "{{ elasticsearch_user }}" 211 | password: "{{ elasticsearch_password }}" 212 | force_basic_auth: true 213 | status_code: 214 | - 200 215 | - 403 #Trial was already activated 216 | 217 | - name: Register a global index template 218 | uri: 219 | url: "{{elasticsearch_host}}_template/template_global" 220 | body_format: json 221 | method: PUT 222 | user: "{{ elasticsearch_user }}" 223 | password: "{{ elasticsearch_password }}" 224 | body: 225 | template: "*" 226 | settings: 227 | number_of_shards: 1 228 | number_of_replicas: 0 229 | refresh_interval: 2s 230 | status_code: 231 | - 200 232 | - 201 233 | 234 | 235 | # Kibana 236 | - name: Install Kibana 237 | apt: deb={{ elastic_download }}/downloads/kibana/kibana-{{ elastic_version }}-amd64.deb 238 | 239 | - name: Stop Kibana 240 | service: name=kibana state=stopped 241 | 242 | - name: Make Kibana available on all network interfaces 243 | lineinfile: 244 | dest: /etc/kibana/kibana.yml 245 | regexp: '^server.host' 246 | line: 'server.host: "0.0.0.0"' 247 | 248 | - name: Create the Kibana logging directory 249 | file: 250 | path: /var/log/kibana/ 251 | state: directory 252 | owner: kibana 253 | group: kibana 254 | 255 | - name: Enable persistent Kibana logs 256 | lineinfile: 257 | dest: /etc/kibana/kibana.yml 258 | regexp: '^logging.dest' 259 | line: 'logging.dest: /var/log/kibana/kibana.log' 260 | 261 | - name: Set the username for the Elasticsearch user 262 | lineinfile: 263 | dest: /etc/kibana/kibana.yml 264 | regexp: "^elasticsearch.username" 265 | line: "elasticsearch.username: {{ elasticsearch_user }}" 266 | state: present 267 | 268 | - name: Set the password for the Elasticsearch user 269 | lineinfile: 270 | dest: /etc/kibana/kibana.yml 271 | regexp: "^elasticsearch.password" 272 | line: "elasticsearch.password: {{ elasticsearch_password }}" 273 | state: present 274 | 275 | - name: Restart Kibana and make sure it autostarts 276 | service: name=kibana state=restarted enabled=yes 277 | 278 | - name: Wait for Kibana to become available, since later steps depend on this 279 | wait_for: 280 | port: 5601 281 | delay: 5 282 | 283 | - name: Check if Kibana is available locally and the firewall rules are correct 284 | local_action: uri url="http://{{ inventory_hostname }}:5601" 285 | become: false 286 | 287 | 288 | # Beats 289 | - name: Set the Elasticsearch password for Beats and APM 290 | lineinfile: 291 | dest: /tmp/cred 292 | line: "{{ elasticsearch_password }}" 293 | state: present 294 | create: yes 295 | mode: 0600 296 | 297 | - name: Get the Beats 298 | apt: deb={{ elastic_download }}/downloads/beats/{{ item }}/{{ item }}-{{ elastic_version }}-amd64.deb 299 | loop: 300 | - auditbeat 301 | - filebeat 302 | - heartbeat 303 | - metricbeat 304 | - packetbeat 305 | 306 | - name: Change the Beats configuration 307 | template: "src=templates/{{ item }}.yml dest=/etc/{{ item }}/{{ item }}.yml" 308 | loop: 309 | - auditbeat 310 | - filebeat 311 | - heartbeat 312 | - metricbeat 313 | - packetbeat 314 | 315 | - name: Create the Beats keystores 316 | command: "{{ item }} keystore create --force" 317 | loop: 318 | - auditbeat 319 | - filebeat 320 | - heartbeat 321 | - metricbeat 322 | - packetbeat 323 | 324 | - name: Set the password in the Beats keystore files 325 | shell: cat /tmp/cred | {{ item }} keystore add ES_PWD --stdin --force 326 | loop: 327 | - auditbeat 328 | - filebeat 329 | - heartbeat 330 | - metricbeat 331 | - packetbeat 332 | 333 | - name: Restart and make sure the Beats autostart 334 | service: name={{ item }} state=restarted enabled=yes 335 | loop: 336 | - auditbeat 337 | - filebeat 338 | - heartbeat-elastic 339 | - metricbeat 340 | - packetbeat 341 | 342 | - name: Wait if the Beats are actually running 343 | pause: minutes=1 344 | 345 | - name: Get the state of all services and check the status of Auditbeat 346 | service_facts: ~ 347 | failed_when: ansible_facts.services.auditbeat.state != "running" 348 | 349 | - name: Get the state of all services and check the status of Filebeat 350 | service_facts: ~ 351 | failed_when: ansible_facts.services.filebeat.state != "running" 352 | 353 | - name: Get the state of all services and check the status of Heartbeat 354 | service_facts: ~ 355 | failed_when: ansible_facts.services["heartbeat-elastic"].state != "running" 356 | 357 | - name: Get the state of all services and check the status of Metricbeat 358 | service_facts: ~ 359 | failed_when: ansible_facts.services.metricbeat.state != "running" 360 | 361 | - name: Get the state of all services and check the status of Packetbeat 362 | service_facts: ~ 363 | failed_when: ansible_facts.services.packetbeat.state != "running" 364 | 365 | 366 | # APM 367 | - name: Install APM 368 | apt: deb={{ elastic_download }}/downloads/apm-server/apm-server-{{ elastic_version }}-amd64.deb 369 | 370 | - name: Change the APM configuration 371 | template: src=templates/apm-server.yml dest=/etc/apm-server/apm-server.yml 372 | 373 | - name: Create the APM keystore 374 | command: apm-server keystore create --force 375 | 376 | - name: Set the secret for APM 377 | lineinfile: 378 | dest: /tmp/sec 379 | line: "{{ apm_secret }}" 380 | state: present 381 | create: yes 382 | mode: 0600 383 | 384 | - name: Set the password in the APM keystore file 385 | shell: cat /tmp/cred | apm-server keystore add ES_PWD --stdin --force 386 | 387 | - name: Set the secret in the APM keystore file 388 | shell: cat /tmp/sec | apm-server keystore add APM_SEC --stdin --force 389 | 390 | - name: Remove the password file 391 | file: 392 | path: /tmp/cred 393 | state: absent 394 | 395 | - name: Remove the secret file 396 | file: 397 | path: /tmp/sec 398 | state: absent 399 | 400 | - name: Restart APM and make sure it autostarts 401 | service: name=apm-server state=restarted enabled=yes 402 | 403 | - name: Fetch the APM agent 404 | get_url: 405 | url: "https://search.maven.org/remotecontent?filepath=co/elastic/apm/elastic-apm-agent/{{ apm_java_version }}/elastic-apm-agent-{{ apm_java_version }}.jar" 406 | dest: "/opt/elastic-apm-agent-{{ apm_java_version }}.jar" 407 | mode: 0444 408 | 409 | - name: Wait if APM is actually running 410 | pause: minutes=1 411 | 412 | - name: Get the state of all services and check the status of APM 413 | service_facts: ~ 414 | failed_when: ansible_facts.services["apm-server"].state != "running" 415 | 416 | 417 | # Set a default index pattern 418 | - name: Set Metricbeat as the default index pattern 419 | uri: 420 | url: "{{kibana_host}}/api/kibana/settings/defaultIndex" 421 | body_format: json 422 | method: POST 423 | force_basic_auth: true 424 | user: "{{ elasticsearch_user }}" 425 | password: "{{ elasticsearch_password }}" 426 | headers: 427 | kbn-xsrf: kibana 428 | body: 429 | value: "metricbeat-*" 430 | 431 | 432 | # Watcher 433 | - name: Add an example Watch from a local file 434 | uri: 435 | url: "{{elasticsearch_host}}_xpack/watcher/watch/heapsize" 436 | body_format: json 437 | method: PUT 438 | user: "{{ elasticsearch_user }}" 439 | password: "{{ elasticsearch_password }}" 440 | body: "{{ lookup('file','alerting_heapsize.json') }}" 441 | status_code: 442 | - 201 443 | - 200 444 | 445 | 446 | # nginx 447 | - name: Install nginx 448 | apt: name=nginx 449 | 450 | - name: Change the nginx configuration 451 | template: src=templates/nginx.conf dest=/etc/nginx/sites-available/default 452 | 453 | - name: Restart nginx and make sure it autostarts 454 | service: name=nginx state=restarted enabled=yes 455 | 456 | 457 | # MySQL 458 | - name: Install the DEB packages required for Ansible's MySQL modules 459 | apt: 460 | name: [ 'python3-dev', 'libmysqlclient-dev' ] 461 | 462 | - name: Install the Python package required for Ansible's MySQL modules 463 | pip: name=mysqlclient 464 | 465 | - name: Install MySQL 466 | apt: name=mysql-server 467 | 468 | - name: Removes all anonymous user accounts 469 | mysql_user: 470 | name: "" 471 | host_all: yes 472 | state: absent 473 | 474 | - name: Create database user and password for MySQL with all database privileges 475 | mysql_user: 476 | name: "{{ mysql_user }}" 477 | password: "{{ mysql_password }}" 478 | priv: "*.*:ALL" 479 | host: "%" 480 | state: present 481 | 482 | - name: Create a new database for Java 483 | mysql_db: 484 | name: "{{ mysql_database }}" 485 | state: present 486 | 487 | - name: Create a new database for SilverStripe 488 | mysql_db: 489 | name: "{{ silverstripe_database }}" 490 | state: present 491 | 492 | - name: Bind MySQL to all interfaces 493 | ini_file: dest=/etc/mysql/mysql.conf.d/mysqld.cnf 494 | section=mysqld 495 | option=bind-address 496 | value="0.0.0.0" 497 | 498 | - name: Restart MySQL and make sure it autostarts 499 | service: name=mysql state=restarted enabled=yes 500 | 501 | 502 | # ApacheBench 503 | - name: Install ApacheBench 504 | apt: 505 | name: [ 'apache2-utils', 'parallel' ] 506 | 507 | - name: Add a list of URLs to benchmark 508 | template: src=templates/urls.txt dest=/home/ubuntu/urls.txt owner=ubuntu group=ubuntu mode=0644 509 | 510 | - name: Add a quick ApacheBench script 511 | copy: 512 | src: files/ab.sh 513 | dest: /home/ubuntu/ab.sh 514 | owner: ubuntu 515 | group: ubuntu 516 | mode: 0755 517 | 518 | 519 | # Gradle 520 | - name: Extract Gradle 521 | unarchive: 522 | src: "https://services.gradle.org/distributions/gradle-{{ gradle_version }}-all.zip" 523 | dest: /usr/local/ 524 | remote_src: yes 525 | 526 | - name: Set Gradle home 527 | lineinfile: 528 | path: /etc/profile 529 | regexp: 'GRADLE_HOME' 530 | line: 'export GRADLE_HOME=/usr/local/gradle-{{ gradle_version }}/' 531 | state: present 532 | 533 | - name: Set Gradle path 534 | lineinfile: 535 | path: /etc/profile 536 | regexp: 'PATH' 537 | line: 'export PATH=${GRADLE_HOME}bin:${PATH}' 538 | state: present 539 | -------------------------------------------------------------------------------- /workshop/configure_php.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: configure_php 14 | 15 | 16 | tasks: 17 | 18 | # PHP 19 | - name: Install PHP 20 | apt: 21 | name: [ 'php-cli', 'php-curl', 'php-fpm', 'php-gd', 'php-intl', 'php-mbstring', 'php-mysql', 'php-tidy', 'php-xdebug', 'php-xml' ] 22 | 23 | - name: Configure PHP 24 | lineinfile: 25 | dest: "/etc/php/{{ php_version }}/fpm/php.ini" 26 | line: "{{ item.key }} = {{ item.value }}" 27 | regexp: "^\\s*(;\\s*)?{{ item.key }}" 28 | with_items: 29 | - { key: 'display_errors', value: 'On' } 30 | - { key: 'date.timezone', value: 'Europe/Vienna' } 31 | 32 | - name: Enable the PHP status page 33 | lineinfile: 34 | dest: "/etc/php/{{ php_version }}/fpm/pool.d/www.conf" 35 | regexp: '^;pm.status_path' 36 | line: pm.status_path = /status 37 | 38 | - name: Restart PHP and make sure it autostarts 39 | service: "name=php{{ php_version }}-fpm state=restarted enabled=yes" 40 | 41 | - name: Check if Composer is installed 42 | stat: path=/usr/local/bin/composer 43 | register: composer_bin 44 | 45 | - name: Download Composer installer 46 | get_url: 47 | url: https://getcomposer.org/installer 48 | dest: /tmp/composer-installer.php 49 | mode: 0755 50 | when: not composer_bin.stat.exists 51 | 52 | - name: Run Composer installer 53 | command: > 54 | php composer-installer.php --install-dir=/usr/local/bin --filename=composer 55 | chdir=/tmp 56 | when: not composer_bin.stat.exists 57 | 58 | 59 | # SilverStripe 60 | - name: Check if SilverStripe is installed 61 | stat: path="{{ silverstripe_directory }}" 62 | register: silverstripe_dir 63 | 64 | - name: Create SilverStripe and install it with all dependencies if it doesn't yet exist 65 | composer: 66 | command: create-project 67 | arguments: "silverstripe/installer ./silverstripe {{ silverstripe_version }}" 68 | working_dir: /var/www/html 69 | prefer_dist: yes 70 | when: not silverstripe_dir.stat.exists 71 | 72 | - name: Make sure the logs folder exists 73 | file: 74 | path: "{{ silverstripe_directory }}/logs" 75 | state: directory 76 | 77 | - name: Fix the permissions for SilverStripe 78 | file: 79 | path: "{{ silverstripe_directory }}" 80 | owner: www-data 81 | group: www-data 82 | mode: 0755 83 | state: directory 84 | recurse: yes 85 | 86 | - name: Set up SilverStripe 87 | template: 88 | src: templates/env 89 | dest: "{{ silverstripe_directory }}.env" 90 | owner: www-data 91 | group: www-data 92 | mode: 0755 93 | 94 | - name: Deploy some custom code 95 | copy: 96 | src: files/mysite/ 97 | dest: "{{ silverstripe_directory }}mysite/" 98 | owner: www-data 99 | group: www-data 100 | mode: 0755 101 | 102 | - name: Configure my site through a template 103 | template: 104 | src: templates/mysite.yml 105 | dest: "{{ silverstripe_directory }}mysite/_config/mysite.yml" 106 | owner: www-data 107 | group: www-data 108 | mode: 0755 109 | 110 | - name: Call flush on the page to finish the installation and apply any changes 111 | uri: url=http://{{ inventory_hostname }}:88?flush=all 112 | -------------------------------------------------------------------------------- /workshop/deploy_backend.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: deploy_backend 14 | 15 | 16 | tasks: 17 | 18 | - include_tasks: include_deploy_boot.yml 19 | vars: 20 | application: backend 21 | port: 8081 22 | -------------------------------------------------------------------------------- /workshop/deploy_bad.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: deploy_bad 14 | 15 | 16 | tasks: 17 | 18 | - name: Package the JAR 19 | local_action: shell cd ../java/bad/; gradle build 20 | become: false 21 | changed_when: true 22 | run_once: true 23 | 24 | - name: Deploy our bad.jar 25 | copy: 26 | src: ../java/bad/build/libs/Bad-1.0.jar 27 | dest: /opt/bad.jar 28 | owner: ubuntu 29 | group: ubuntu 30 | mode: 0500 31 | 32 | - name: Automatically run the bad.jar with a cron job 33 | cron: 34 | name: "Run /opt/bad.jar every 10 minutes" 35 | minute: "*/6" 36 | hour: "*" 37 | job: "java -Xmx400m -jar /opt/bad.jar" 38 | -------------------------------------------------------------------------------- /workshop/deploy_frontend.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: yes 4 | 5 | 6 | vars_files: 7 | - variables.yml 8 | 9 | 10 | post_tasks: 11 | - include_tasks: include_event.yml 12 | vars: 13 | application: deploy_frontend 14 | 15 | 16 | tasks: 17 | 18 | - include_tasks: include_deploy_boot.yml 19 | vars: 20 | application: frontend 21 | port: 8080 22 | -------------------------------------------------------------------------------- /workshop/files/ab.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat urls.txt | parallel "ab -n 300 -c 10 {}" 4 | -------------------------------------------------------------------------------- /workshop/files/alerting_heapsize.json: -------------------------------------------------------------------------------- 1 | { 2 | "trigger": { 3 | "schedule": { 4 | "interval": "1m" 5 | } 6 | }, 7 | "input": { 8 | "search": { 9 | "request": { 10 | "search_type": "query_then_fetch", 11 | "indices": [ 12 | "metricbeat-*" 13 | ], 14 | "types": [], 15 | "body": { 16 | "size": 0, 17 | "query": { 18 | "bool": { 19 | "filter": { 20 | "range": { 21 | "@timestamp": { 22 | "gte": "{{ctx.trigger.scheduled_time}}||-5m", 23 | "lte": "{{ctx.trigger.scheduled_time}}", 24 | "format": "strict_date_optional_time||epoch_millis" 25 | } 26 | } 27 | } 28 | } 29 | }, 30 | "aggs": { 31 | "bucketAgg": { 32 | "terms": { 33 | "field": "beat.name", 34 | "size": 5, 35 | "order": { 36 | "metricAgg": "desc" 37 | } 38 | }, 39 | "aggs": { 40 | "metricAgg": { 41 | "max": { 42 | "field": "jolokia.metrics.memory.heap_usage.used" 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } 51 | }, 52 | "condition": { 53 | "script": { 54 | "source": "ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; for (int i = 0; i < arr.length; i++) { if (arr[i]['metricAgg'].value > params.threshold) { return true; } } return false;", 55 | "lang": "painless", 56 | "params": { 57 | "threshold": 50000000 58 | } 59 | } 60 | }, 61 | "transform": { 62 | "script": { 63 | "source": "HashMap result = new HashMap(); ArrayList arr = ctx.payload.aggregations.bucketAgg.buckets; ArrayList filteredHits = new ArrayList(); for (int i = 0; i < arr.length; i++) { HashMap filteredHit = new HashMap(); filteredHit.key = arr[i].key; filteredHit.value = arr[i]['metricAgg'].value; if (filteredHit.value > params.threshold) { filteredHits.add(filteredHit); } } result.results = filteredHits; return result;", 64 | "lang": "painless", 65 | "params": { 66 | "threshold": 50000000 67 | } 68 | } 69 | }, 70 | "actions": { 71 | "logging_1": { 72 | "logging": { 73 | "level": "info", 74 | "text": "Heap is too large" 75 | } 76 | } 77 | }, 78 | "metadata": { 79 | "name": "Heap", 80 | "watcherui": { 81 | "trigger_interval_unit": "m", 82 | "agg_type": "max", 83 | "time_field": "@timestamp", 84 | "trigger_interval_size": 1, 85 | "term_size": 5, 86 | "time_window_unit": "m", 87 | "threshold_comparator": ">", 88 | "term_field": "beat.name", 89 | "index": [ 90 | "metricbeat-*" 91 | ], 92 | "time_window_size": 5, 93 | "threshold": 50000000, 94 | "agg_field": "jolokia.metrics.memory.heap_usage.used" 95 | }, 96 | "xpack": { 97 | "type": "threshold" 98 | } 99 | }, 100 | "status": { 101 | "state": { 102 | "active": true, 103 | "timestamp": "2017-09-29T13:31:00.309Z" 104 | }, 105 | "actions": { 106 | "logging_1": { 107 | "ack": { 108 | "timestamp": "2017-09-29T13:32:00.822Z", 109 | "state": "ackable" 110 | }, 111 | "last_execution": { 112 | "timestamp": "2017-09-29T13:40:03.537Z", 113 | "successful": true 114 | }, 115 | "last_successful_execution": { 116 | "timestamp": "2017-09-29T13:40:03.537Z", 117 | "successful": true 118 | } 119 | } 120 | }, 121 | "version": -1, 122 | "last_checked": "2017-09-29T13:40:03.537Z", 123 | "last_met_condition": "2017-09-29T13:40:03.537Z" 124 | } 125 | } -------------------------------------------------------------------------------- /workshop/files/mysite/_config/logging.yml: -------------------------------------------------------------------------------- 1 | SilverStripe\Core\Injector\Injector: 2 | Psr\Log\LoggerInterface: 3 | calls: 4 | LogFileHandler: [ pushHandler, [ %$LogFileHandler ] ] 5 | JsonFileHandler: [ pushHandler, [ %$JsonFileHandler ] ] 6 | 7 | LogFileHandler: 8 | class: Monolog\Handler\StreamHandler 9 | constructor: 10 | - "../logs/silverstripe.log" 11 | - "notice" 12 | 13 | JsonFileHandler: 14 | class: Monolog\Handler\StreamHandler 15 | constructor: 16 | - "../logs/silverstripe.json" 17 | - "notice" 18 | properties: 19 | Formatter: %$Monolog\Formatter\JsonFormatter 20 | -------------------------------------------------------------------------------- /workshop/files/mysite/_config/routes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | Name: mysiteroutes 3 | After: framework/_config/routes#coreroutes 4 | --- 5 | SilverStripe\Control\Director: 6 | rules: 7 | 'error//$Action/$ID/$Name': 'ErrorController' 8 | -------------------------------------------------------------------------------- /workshop/files/mysite/code/controllers/ErrorController.php: -------------------------------------------------------------------------------- 1 | '%$' . LoggerInterface::class 10 | ]; 11 | 12 | public $logger; 13 | 14 | private static $allowed_actions = [ 15 | 'index', 16 | 'server', 17 | 'client', 18 | 'exception', 19 | ]; 20 | 21 | public function index(){ 22 | $this->logger->warning('Something is causing a warning 🚧'); 23 | parent::init(); 24 | } 25 | 26 | public function server(){ 27 | user_error("Server error 😱", E_USER_WARNING); 28 | return; 29 | } 30 | 31 | public function client(){ 32 | $this->setResponse(new HTTPResponse()); 33 | $this->getResponse()->setStatusCode(400); 34 | $this->getResponse()->setBody('Invalid user interaction 🤯'); 35 | $this->logger->debug('Invalid user interaction 🤯'); 36 | return $this->getResponse(); 37 | } 38 | 39 | public function exception(){ 40 | throw new \LogicException('Welcome to exception land 🔥'); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /workshop/include_deploy_boot.yml: -------------------------------------------------------------------------------- 1 | - name: Clone the repository so we can build it locally 2 | git: 3 | repo: https://github.com/xeraa/microservice-monitoring.git 4 | dest: /opt/microservice-monitoring 5 | 6 | - name: Package the JAR 7 | shell: cd /opt/microservice-monitoring/java/{{ application }}/; /usr/local/gradle-{{ gradle_version }}/bin/gradle build 8 | changed_when: true 9 | 10 | - name: Get the current timestamp 11 | set_fact: 12 | current_timestamp: "{{ ansible_date_time.epoch }}" 13 | 14 | - name: Make sure that the logging directory exists 15 | file: 16 | path: /var/log/apps/ 17 | state: directory 18 | owner: ubuntu 19 | group: ubuntu 20 | mode: 0755 21 | 22 | - name: Deploy the executable uber JAR 23 | copy: 24 | src: "/opt/microservice-monitoring/java/{{ application }}/build/libs/{{ application }}.jar" 25 | dest: "/opt/{{ application }}_{{ current_timestamp }}.jar" 26 | owner: ubuntu 27 | group: ubuntu 28 | mode: 0500 29 | remote_src: yes 30 | 31 | - name: Provide a conf file to run it as a service 32 | template: 33 | src: templates/{{ application }}.conf 34 | dest: "/opt/{{ application }}_{{ current_timestamp }}.conf" 35 | owner: root 36 | group: root 37 | mode: 0400 38 | 39 | - name: Check if there is a previous version 40 | stat: 41 | path: "/opt/{{ application }}.jar" 42 | register: symlink 43 | 44 | - name: Link to current version 45 | file: 46 | src: "/opt/{{ application }}_{{ current_timestamp }}.jar" 47 | dest: "/opt/{{ application }}.jar" 48 | owner: ubuntu 49 | group: ubuntu 50 | state: link 51 | 52 | - name: Link the JAR so it can be managed as a service through init.d 53 | file: 54 | src: "/opt/{{ application }}.jar" 55 | dest: "/etc/init.d/{{ application }}" 56 | state: link 57 | owner: ubuntu 58 | group: ubuntu 59 | 60 | - name: Reload the configuration and make the service autostart 61 | systemd: name="{{ application }}" enabled=yes daemon_reload=yes 62 | ignore_errors: true 63 | 64 | - name: Restart the service 65 | systemd: name="{{ application }}" state=restarted 66 | 67 | - name: Wait for the Java application to start up 68 | pause: seconds=30 69 | 70 | - name: Check if the service is available 71 | uri: url="http://localhost:{{ port }}/health" 72 | register: response 73 | 74 | - name: Remove old JARs (keeping the five latest good ones) 75 | shell: "ls -tr /opt/{{ application }}_*.jar | head -n -5 | xargs rm -f" 76 | register: remove_output 77 | 78 | - name: Remove old configs (keeping the five latest good ones) 79 | shell: "ls -tr /opt/{{ application }}_*.conf | head -n -5 | xargs rm -f" 80 | -------------------------------------------------------------------------------- /workshop/include_event.yml: -------------------------------------------------------------------------------- 1 | - name: Get the local user 2 | command: whoami 3 | register: local_username 4 | delegate_to: 127.0.0.1 5 | become: false 6 | changed_when: false 7 | 8 | - name: Store the playbook run event in Elasticsearch so it can be used as an annotation 9 | uri: 10 | url: "{{elasticsearch_host}}events/deployment" 11 | body_format: json 12 | method: POST 13 | user: "{{ elasticsearch_user }}" 14 | password: "{{ elasticsearch_password }}" 15 | body: 16 | "@timestamp": "{{ ansible_date_time.iso8601 }}" 17 | application: "{{ application }}" 18 | system: ansible 19 | host: "{{ inventory_hostname }}" 20 | user: "{{ local_username.stdout }}" 21 | status_code: 201 22 | -------------------------------------------------------------------------------- /workshop/inventory: -------------------------------------------------------------------------------- 1 | [all] 2 | workshop-[0:0].xeraa.wtf ansible_python_interpreter=/usr/bin/python3 3 | 4 | 5 | # If there are a lot of hosts and you run into concurrency issues, try to split them up and run with --limit='group0' 6 | #[group0] 7 | #workshop-[0:4].xeraa.wtf ansible_python_interpreter=/usr/bin/python3 8 | -------------------------------------------------------------------------------- /workshop/templates/apm-server.yml: -------------------------------------------------------------------------------- 1 | apm-server: 2 | host: "0.0.0.0:8200" 3 | 4 | secret_token: "${APM_SEC}" 5 | 6 | 7 | setup.template.settings: 8 | index.number_of_shards: 1 9 | index.codec: best_compression 10 | 11 | 12 | output.elasticsearch: 13 | hosts: ["{{ elasticsearch_host }}"] 14 | username: "{{ elasticsearch_user }}" 15 | password: "${ES_PWD}" 16 | 17 | 18 | setup: 19 | kibana: 20 | host: "{{ kibana_host }}" 21 | username: "{{ elasticsearch_user }}" 22 | password: "${ES_PWD}" 23 | dashboards.enabled: true 24 | -------------------------------------------------------------------------------- /workshop/templates/auditbeat.yml: -------------------------------------------------------------------------------- 1 | auditbeat.modules: 2 | 3 | - module: auditd 4 | resolve_ids: true 5 | failure_mode: silent 6 | backlog_limit: 8196 7 | rate_limit: 0 8 | include_raw_message: false 9 | include_warnings: false 10 | audit_rules: | 11 | ## Define audit rules here. 12 | ## Create file watches (-w) or syscall audits (-a action,filter). 13 | ## * action can be either always or never. 14 | ## * filter specifies which kernel rule-matching filter is applied to the event, 15 | ## which can be one of the following: task, exit, user, and exclude. 16 | ## Several system calls can be grouped into one rule, each specified after the -S option. 17 | ## Add a keyword (-k) to the log event. 18 | 19 | # Things that affect identity 20 | -w /etc/group -p wa -k identity 21 | -w /etc/passwd -p wa -k identity 22 | -w /etc/gshadow -p wa -k identity 23 | -w /etc/shadow -p wa -k identity 24 | -w /etc/security/opasswd -p wa -k identity 25 | 26 | # Log read access to passwd from selected users (1001=developers) 27 | -a always,exit -F path=/etc/passwd -F perm=r -F uid=1001 -k developers-passwd-read 28 | 29 | # Log permission errors 30 | -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access 31 | 32 | # Log processes that call the connect system call for IPv4 and IPv6 33 | -a always,exit -F arch=b64 -S connect -F a0=2 -k connect 34 | -a always,exit -F arch=b64 -S connect -F a0=10 -k connect 35 | 36 | # Detect when an admin may be abusing power by looking in a user's home directory 37 | -a always,exit -F dir=/home -F uid=0 -F auid>=1000 -F auid!=4294967295 -C auid!=obj_uid -k power-abuse 38 | 39 | # Log all executed processes 40 | -a always,exit -S execve 41 | 42 | # Log all elevation of privileges 43 | -a always,exit -F arch=b64 -S setuid -F a0=0 -F exe=/usr/bin/su -k elevated-privs 44 | -a always,exit -F arch=b32 -S setuid -F a0=0 -F exe=/usr/bin/su -k elevated-privs 45 | -a always,exit -F arch=b64 -S setresuid -F a0=0 -F exe=/usr/bin/sudo -k elevated-privs 46 | -a always,exit -F arch=b32 -S setresuid -F a0=0 -F exe=/usr/bin/sudo -k elevated-privs 47 | -a always,exit -F arch=b64 -S execve -C uid!=euid -F euid=0 -k elevated-privs 48 | -a always,exit -F arch=b32 -S execve -C uid!=euid -F euid=0 -k elevated-privs 49 | 50 | 51 | - module: file_integrity 52 | paths: 53 | - /opt/ 54 | scan_at_start: true 55 | scan_rate_per_sec: 50 MiB 56 | file.max_file_size: 100 MiB 57 | file.hash_types: [sha1] 58 | 59 | 60 | name: "{{ inventory_hostname }}" 61 | tags: ["{{ env }}", "lightsail"] 62 | 63 | 64 | processors: 65 | - add_cloud_metadata: ~ 66 | - add_host_metadata: ~ 67 | 68 | 69 | xpack.monitoring.enabled: true 70 | 71 | 72 | output.elasticsearch: 73 | hosts: ["{{ elasticsearch_host }}"] 74 | username: "{{ elasticsearch_user }}" 75 | password: "${ES_PWD}" 76 | 77 | 78 | setup: 79 | kibana: 80 | host: "{{ kibana_host }}" 81 | username: "{{ elasticsearch_user }}" 82 | password: "${ES_PWD}" 83 | dashboards.enabled: true 84 | -------------------------------------------------------------------------------- /workshop/templates/backend.conf: -------------------------------------------------------------------------------- 1 | JAVA_OPTS="-Xmx384m\ 2 | -DLOG_PATH=/var/log/apps\ 3 | -DSERVER_PORT=8081\ 4 | -DAPP_BACKEND=http://{{ backend_server }}\ 5 | -DAPP_FRONTEND=http://{{ frontend_server }}\ 6 | -DDATABASE_USERNAME={{ mysql_user }}\ 7 | -DDATABASE_PASSWORD={{mysql_password }}\ 8 | -DDATABASE_NAME={{ mysql_database }}\ 9 | -javaagent:/opt/elastic-apm-agent-{{ apm_java_version }}.jar\ 10 | -Delastic.apm.service_name=backend\ 11 | -Delastic.apm.application_packages=net.xeraa.backend\ 12 | -Delastic.apm.stack_trace_limit=150\ 13 | -Delastic.apm.environment={{ env }}\ 14 | -Delastic.apm.sample_rate=1.0\ 15 | -Delastic.apm.capture_body=all\ 16 | -Delastic.apm.span_frames_min_duration=-1\ 17 | -Delastic.apm.server_urls={{ apm_server }}\ 18 | -Delastic.apm.ignore_urls=/health,/metrics*,/jolokia\ 19 | -Delastic.apm.log_file=/var/log/apps/apm-backend\ 20 | -Delastic.apm.enable_log_correlation=true\ 21 | -Delastic.apm.secret_token={{ apm_secret }}" 22 | -------------------------------------------------------------------------------- /workshop/templates/env: -------------------------------------------------------------------------------- 1 | ## Environment 2 | SS_ENVIRONMENT_TYPE="dev" 3 | SS_DEFAULT_ADMIN_USERNAME="{{ silverstripe_user }}" 4 | SS_DEFAULT_ADMIN_PASSWORD="{{ silverstripe_password }}" 5 | SS_BASE_URL="http://{{ inventory_hostname }}/" 6 | 7 | ## Database 8 | SS_DATABASE_CLASS="MySQLDatabase" 9 | SS_DATABASE_USERNAME="{{ mysql_user }}" 10 | SS_DATABASE_PASSWORD="{{ mysql_password }}" 11 | SS_DATABASE_SERVER="127.0.0.1" 12 | SS_DATABASE_NAME="{{ silverstripe_database }}" 13 | -------------------------------------------------------------------------------- /workshop/templates/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.inputs: 2 | 3 | # Collect the JSON log files from the Spring Boot apps 4 | - type: log 5 | paths: 6 | - /var/log/apps/*.json 7 | fields_under_root: true 8 | fields: 9 | application: java 10 | json.message_key: log 11 | 12 | 13 | filebeat.modules: 14 | - module: auditd 15 | - module: mysql 16 | - module: nginx 17 | - module: osquery 18 | - module: system 19 | 20 | 21 | name: "{{ inventory_hostname }}" 22 | tags: ["{{ env }}", "lightsail"] 23 | 24 | 25 | processors: 26 | - add_cloud_metadata: ~ 27 | - add_host_metadata: ~ 28 | 29 | 30 | xpack.monitoring.enabled: true 31 | 32 | 33 | output.elasticsearch: 34 | hosts: ["{{ elasticsearch_host }}"] 35 | username: "{{ elasticsearch_user }}" 36 | password: "${ES_PWD}" 37 | 38 | 39 | setup: 40 | kibana: 41 | host: "{{ kibana_host }}" 42 | username: "{{ elasticsearch_user }}" 43 | password: "${ES_PWD}" 44 | dashboards.enabled: true 45 | -------------------------------------------------------------------------------- /workshop/templates/frontend.conf: -------------------------------------------------------------------------------- 1 | JAVA_OPTS="-Xmx384m\ 2 | -DLOG_PATH=/var/log/apps\ 3 | -DSERVER_PORT=8080\ 4 | -DAPP_BACKEND=http://{{ backend_server }}\ 5 | -DAPP_FRONTEND=http://{{ frontend_server }}\ 6 | -javaagent:/opt/elastic-apm-agent-{{ apm_java_version }}.jar\ 7 | -Delastic.apm.service_name=frontend\ 8 | -Delastic.apm.application_packages=net.xeraa.frontend\ 9 | -Delastic.apm.stack_trace_limit=150\ 10 | -Delastic.apm.environment={{ env }}\ 11 | -Delastic.apm.sample_rate=1.0\ 12 | -Delastic.apm.capture_body=all\ 13 | -Delastic.apm.span_frames_min_duration=-1\ 14 | -Delastic.apm.server_urls={{ apm_server }}\ 15 | -Delastic.apm.ignore_urls=/health,/metrics*,/jolokia\ 16 | -Delastic.apm.log_file=/var/log/apps/apm-frontend\ 17 | -Delastic.apm.enable_log_correlation=true\ 18 | -Delastic.apm.secret_token={{ apm_secret }}" 19 | -------------------------------------------------------------------------------- /workshop/templates/heartbeat.yml: -------------------------------------------------------------------------------- 1 | heartbeat.monitors: 2 | 3 | - type: http 4 | urls: ["http://{{ frontend_server }}/health"] 5 | schedule: "@every 10s" 6 | timeout: 3s 7 | check.response.status: 200 8 | 9 | - type: http 10 | urls: ["http://{{ backend_server }}/health"] 11 | schedule: "@every 10s" 12 | timeout: 3s 13 | check.response.status: 200 14 | 15 | - type: http 16 | urls: ["{{ elasticsearch_host }}"] 17 | username: {{ elasticsearch_user }} 18 | password: {{ elasticsearch_password }} 19 | schedule: "@every 10s" 20 | timeout: 3s 21 | check.response.status: 200 22 | 23 | - type: tcp 24 | hosts: ["{{ mysql_server }}"] 25 | schedule: "@every 10s" 26 | timeout: 3s 27 | 28 | - type: tcp 29 | hosts: ["{{ apm_server }}"] 30 | schedule: "@every 10s" 31 | timeout: 3s 32 | 33 | 34 | name: "{{ inventory_hostname }}" 35 | tags: ["{{ env }}", "lightsail"] 36 | 37 | 38 | processors: 39 | - add_cloud_metadata: ~ 40 | - add_host_metadata: ~ 41 | 42 | 43 | xpack.monitoring.enabled: true 44 | 45 | 46 | output.elasticsearch: 47 | hosts: ["{{ elasticsearch_host }}"] 48 | username: "{{ elasticsearch_user }}" 49 | password: "${ES_PWD}" 50 | 51 | 52 | setup: 53 | kibana: 54 | host: "{{ kibana_host }}" 55 | username: "{{ elasticsearch_user }}" 56 | password: "${ES_PWD}" 57 | dashboards.enabled: true 58 | -------------------------------------------------------------------------------- /workshop/templates/metricbeat.yml: -------------------------------------------------------------------------------- 1 | metricbeat.modules: 2 | 3 | - module: system 4 | metricsets: 5 | - cpu 6 | - load 7 | - core 8 | - diskio 9 | - filesystem 10 | - fsstat 11 | - memory 12 | - network 13 | - process 14 | - process_summary 15 | - socket 16 | enabled: true 17 | period: 10s 18 | processes: ['.*'] 19 | cgroups: true 20 | process.include_top_n: 21 | enabled: true 22 | by_cpu: 20 23 | by_memory: 20 24 | 25 | - module: nginx 26 | metricsets: ["stubstatus"] 27 | enabled: true 28 | period: 10s 29 | hosts: ["{{ inventory_hostname }}"] 30 | 31 | - module: mysql 32 | metricsets: ["status"] 33 | hosts: ["tcp(localhost:3306)/"] 34 | username: "{{ mysql_user }}" 35 | password: "{{ mysql_password }}" 36 | 37 | - module: http 38 | metricsets: ["json"] 39 | period: 10s 40 | hosts: ["{{ backend_server }}"] 41 | namespace: health 42 | path: /health 43 | method: GET 44 | fields: 45 | service: backend 46 | 47 | {% for metric in ('http.server.requests', 'process.files.max', 'jvm.gc.memory.promoted', 'tomcat.cache.hit', 48 | 'jvm.memory.committed', 'system.load.average.1m', 'tomcat.cache.access', 'jvm.memory.used', 49 | 'jvm.gc.max.data.size', 'system.cpu.count', 'logback.events', 'tomcat.global.sent', 50 | 'jvm.buffer.memory.used', 'tomcat.sessions.created', 'jvm.memory.max', 'jvm.threads.daemon', 51 | 'system.cpu.usage', 'jvm.gc.memory.allocated', 'tomcat.global.request.max', 52 | 'tomcat.global.request', 'tomcat.sessions.expired', 'jvm.threads.live', 'jvm.threads.peak', 53 | 'tomcat.global.received', 'process.uptime', 'tomcat.sessions.rejected', 'process.cpu.usage', 54 | 'jvm.gc.pause', 'tomcat.threads.config.max', 'jvm.classes.loaded', 'jvm.classes.unloaded', 55 | 'tomcat.global.error', 'tomcat.sessions.active.current', 'tomcat.sessions.alive.max', 56 | 'jvm.gc.live.data.size', 'tomcat.servlet.request.max', 'tomcat.threads.current', 57 | 'tomcat.servlet.request', 'process.files.open', 'jvm.buffer.count', 'jvm.buffer.total.capacity', 58 | 'tomcat.sessions.active.max', 'tomcat.threads.busy', 'process.start.time', 'tomcat.servlet.error') %} 59 | - module: http 60 | metricsets: ["json"] 61 | period: 10s 62 | hosts: ["{{ backend_server }}"] 63 | namespace: metrics 64 | path: "/metrics/{{ metric }}" 65 | method: GET 66 | dedot.enabled: true 67 | fields: 68 | service: backend 69 | {% endfor %} 70 | 71 | - module: http 72 | metricsets: ["json"] 73 | period: 10s 74 | hosts: ["{{ frontend_server }}"] 75 | namespace: health 76 | path: /health 77 | method: GET 78 | fields: 79 | service: frontend 80 | 81 | {% for metric in ('http.server.requests', 'process.files.max', 'jvm.gc.memory.promoted', 'tomcat.cache.hit', 82 | 'jvm.memory.committed', 'system.load.average.1m', 'tomcat.cache.access', 'jvm.memory.used', 83 | 'jvm.gc.max.data.size', 'system.cpu.count', 'logback.events', 'tomcat.global.sent', 84 | 'jvm.buffer.memory.used', 'tomcat.sessions.created', 'jvm.memory.max', 'jvm.threads.daemon', 85 | 'system.cpu.usage', 'jvm.gc.memory.allocated', 'tomcat.global.request.max', 86 | 'tomcat.global.request', 'tomcat.sessions.expired', 'jvm.threads.live', 'jvm.threads.peak', 87 | 'tomcat.global.received', 'process.uptime', 'tomcat.sessions.rejected', 'process.cpu.usage', 88 | 'jvm.gc.pause', 'tomcat.threads.config.max', 'jvm.classes.loaded', 'jvm.classes.unloaded', 89 | 'tomcat.global.error', 'tomcat.sessions.active.current', 'tomcat.sessions.alive.max', 90 | 'jvm.gc.live.data.size', 'tomcat.servlet.request.max', 'tomcat.threads.current', 91 | 'tomcat.servlet.request', 'process.files.open', 'jvm.buffer.count', 'jvm.buffer.total.capacity', 92 | 'tomcat.sessions.active.max', 'tomcat.threads.busy', 'process.start.time', 'tomcat.servlet.error') %} 93 | - module: http 94 | metricsets: ["json"] 95 | period: 10s 96 | hosts: ["{{ frontend_server }}"] 97 | namespace: metrics 98 | path: "/metrics/{{ metric }}" 99 | method: GET 100 | dedot.enabled: true 101 | fields: 102 | service: backend 103 | {% endfor %} 104 | 105 | - module: jolokia 106 | metricsets: ["jmx"] 107 | hosts: ["{{ frontend_server }}"] 108 | namespace: metrics 109 | jmx.mappings: 110 | - mbean: "java.lang:type=Runtime" 111 | attributes: 112 | - attr: Uptime 113 | field: uptime 114 | - mbean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" 115 | attributes: 116 | - attr: CollectionTime 117 | field: gc.cms_collection_time 118 | event: gc 119 | - attr: CollectionCount 120 | field: gc.cms_collection_count 121 | event: gc 122 | - mbean: "java.lang:type=Memory" 123 | attributes: 124 | - attr: HeapMemoryUsage 125 | field: memory.heap_usage 126 | event: heap 127 | - attr: NonHeapMemoryUsage 128 | field: memory.non_heap_usage 129 | event: heap 130 | - mbean: "java.lang:type=Threading" 131 | attributes: 132 | - attr: ThreadCount 133 | field: thread.count 134 | event: thread 135 | - attr: DaemonThreadCount 136 | field: thread.daemon 137 | event: thread 138 | jmx.instance: "{{ inventory_hostname }}" 139 | fields: 140 | service: frontend 141 | 142 | - module: jolokia 143 | metricsets: ["jmx"] 144 | hosts: ["{{ backend_server }}"] 145 | namespace: metrics 146 | jmx.mappings: 147 | - mbean: "java.lang:type=Runtime" 148 | attributes: 149 | - attr: Uptime 150 | field: uptime 151 | - mbean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" 152 | attributes: 153 | - attr: CollectionTime 154 | field: gc.cms_collection_time 155 | event: gc 156 | - attr: CollectionCount 157 | field: gc.cms_collection_count 158 | event: gc 159 | - mbean: "java.lang:type=Memory" 160 | attributes: 161 | - attr: HeapMemoryUsage 162 | field: memory.heap_usage 163 | event: heap 164 | - attr: NonHeapMemoryUsage 165 | field: memory.non_heap_usage 166 | event: heap 167 | - mbean: "java.lang:type=Threading" 168 | attributes: 169 | - attr: ThreadCount 170 | field: thread.count 171 | event: thread 172 | - attr: DaemonThreadCount 173 | field: thread.daemon 174 | event: thread 175 | jmx.instance: "{{ inventory_hostname }}" 176 | fields: 177 | service: backend 178 | 179 | 180 | name: "{{ inventory_hostname }}" 181 | tags: ["{{ env }}", "lightsail"] 182 | 183 | 184 | processors: 185 | - add_cloud_metadata: ~ 186 | - add_host_metadata: ~ 187 | 188 | 189 | xpack.monitoring.enabled: true 190 | 191 | 192 | output.elasticsearch: 193 | hosts: ["{{ elasticsearch_host }}"] 194 | username: "{{ elasticsearch_user }}" 195 | password: "${ES_PWD}" 196 | 197 | 198 | setup: 199 | kibana: 200 | host: "{{ kibana_host }}" 201 | username: "{{ elasticsearch_user }}" 202 | password: "${ES_PWD}" 203 | dashboards.enabled: true 204 | -------------------------------------------------------------------------------- /workshop/templates/mysite.yml: -------------------------------------------------------------------------------- 1 | --- 2 | Name: {{ inventory_hostname }} 3 | --- 4 | SilverStripe\Core\Manifest\ModuleManifest: 5 | project: {{ inventory_hostname }} 6 | -------------------------------------------------------------------------------- /workshop/templates/nginx.conf: -------------------------------------------------------------------------------- 1 | # Don't send the nginx version number in error pages and server header 2 | server_tokens off; 3 | 4 | # Don't allow the page to render inside a frame of an iframe 5 | add_header X-Frame-Options DENY; 6 | 7 | # Disable sniffing for user supplied content 8 | add_header X-Content-Type-Options nosniff; 9 | 10 | 11 | # Java reverse proxy 12 | server { 13 | charset utf-8; 14 | listen 80; 15 | server_name _; 16 | 17 | # Proxy on port 8080 to the Java application 18 | location / { 19 | proxy_pass http://127.0.0.1:8080; 20 | proxy_redirect off; 21 | proxy_set_header Host $host; 22 | proxy_set_header X-Real-IP $remote_addr; 23 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 24 | proxy_set_header X-Forwarded-Host $server_name; 25 | } 26 | 27 | # Enable the stub status module for Metricbeat 28 | location /server-status { 29 | stub_status on; 30 | access_log off; 31 | } 32 | } 33 | 34 | 35 | # SilverStripe 36 | server { 37 | charset utf-8; 38 | listen 88; 39 | server_name _; 40 | 41 | root /var/www/html/silverstripe/public/; 42 | 43 | # SilverStripe rules 44 | location / { 45 | try_files $uri /index.php?$query_string; 46 | } 47 | error_page 404 /assets/error-404.html; 48 | error_page 500 /assets/error-500.html; 49 | location ^~ /assets/ { 50 | location ~ /\. { 51 | deny all; 52 | } 53 | sendfile on; 54 | try_files $uri /index.php?$query_string; 55 | } 56 | location ~ /\.. { 57 | deny all; 58 | } 59 | location ~ web\.config$ { 60 | deny all; 61 | } 62 | 63 | # Configure PHP 64 | location ~ \.php$ { 65 | fastcgi_keep_conn on; 66 | fastcgi_pass unix:/var/run/php/php{{ php_version }}-fpm.sock; 67 | fastcgi_index index.php; 68 | fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; 69 | fastcgi_buffer_size 32k; 70 | fastcgi_busy_buffers_size 64k; 71 | fastcgi_buffers 4 32k; 72 | fastcgi_read_timeout 600; 73 | include fastcgi_params; 74 | } 75 | 76 | # Enable the stub status module for Metricbeat 77 | location /server-status { 78 | stub_status on; 79 | access_log off; 80 | } 81 | 82 | # Enable PHP-FPM output for Metricbeat 83 | location /status { 84 | include fastcgi_params; 85 | fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; 86 | fastcgi_pass unix:/var/run/php/php{{ php_version }}-fpm.sock; 87 | access_log off; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /workshop/templates/osquery.conf: -------------------------------------------------------------------------------- 1 | { 2 | "options": { 3 | "enable_monitor": "true" 4 | }, 5 | "packs": { 6 | "it-compliance": "/etc/osquery/it-compliance.conf", 7 | "ossec-rootkit": "/etc/osquery/ossec-rootkit.conf" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /workshop/templates/packetbeat.yml: -------------------------------------------------------------------------------- 1 | packetbeat.interfaces.device: any 2 | 3 | 4 | packetbeat.flows: 5 | timeout: 30s 6 | period: 10s 7 | 8 | 9 | packetbeat.protocols: 10 | 11 | - type: icmp 12 | enabled: true 13 | 14 | - type: dns 15 | ports: [53] 16 | 17 | - type: http 18 | ports: [80, 5601, 8080, 8081] 19 | real_ip_header: "X-Forwarded-For" 20 | 21 | - type: mysql 22 | ports: [3306] 23 | 24 | - type: tls 25 | ports: [443, 8200] 26 | 27 | 28 | packetbeat.procs: 29 | enabled: true 30 | monitored: 31 | 32 | - process: apm-server 33 | cmdline_grep: apm-server 34 | 35 | - process: java 36 | cmdline_grep: java 37 | 38 | - process: mysql 39 | cmdline_grep: mysql 40 | 41 | - process: nginx 42 | cmdline_grep: nginx 43 | 44 | 45 | name: "{{ inventory_hostname }}" 46 | tags: ["{{ env }}", "lightsail"] 47 | 48 | 49 | processors: 50 | 51 | - add_cloud_metadata: ~ 52 | - add_host_metadata: ~ 53 | 54 | - drop_event: 55 | when: 56 | or: 57 | # Exclude pinging metrics via REST and JMX 58 | - contains.path: "/metrics/" 59 | - equals.path: "/jolokia" 60 | # Exclude pinging health 61 | - equals.path: "/health" 62 | # Exclude nginx status 63 | - equals.path: "/server-status" 64 | 65 | 66 | xpack.monitoring.enabled: true 67 | 68 | 69 | output.elasticsearch: 70 | hosts: ["{{ elasticsearch_host }}"] 71 | username: "{{ elasticsearch_user }}" 72 | password: "${ES_PWD}" 73 | 74 | 75 | setup: 76 | kibana: 77 | host: "{{ kibana_host }}" 78 | username: "{{ elasticsearch_user }}" 79 | password: "${ES_PWD}" 80 | dashboards.enabled: true 81 | -------------------------------------------------------------------------------- /workshop/templates/secret.txt: -------------------------------------------------------------------------------- 1 | My secret... 2 | -------------------------------------------------------------------------------- /workshop/templates/urls.txt: -------------------------------------------------------------------------------- 1 | http://127.0.0.1/ 2 | http://127.0.0.1/good 3 | http://127.0.0.1/generate 4 | http://127.0.0.1/good?name=Philipp 5 | http://127.0.0.1/add?name=Philipp 6 | http://127.0.0.1/search?q=Philipp 7 | http://127.0.0.1/bad 8 | http://127.0.0.1/null 9 | http://127.0.0.1/search 10 | -------------------------------------------------------------------------------- /workshop/terraform.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | # Credentials are defined in the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY 3 | region = "${var.region}" 4 | } 5 | 6 | 7 | # Create the SSH key pair 8 | resource "aws_lightsail_key_pair" "microservice_monitoring_key_pair" { 9 | name = "microservice_monitoring_key_pair" 10 | public_key = "${file("~/.ssh/id_rsa.pub")}" 11 | } 12 | 13 | 14 | # Create the given number of instances and asign a DNS entry 15 | resource "aws_lightsail_instance" "workshop" { 16 | count = "${var.count}" 17 | name = "workshop-${count.index}" 18 | availability_zone = "${var.region}a" 19 | blueprint_id = "${var.operating_system}" 20 | bundle_id = "${var.size}" 21 | key_pair_name = "microservice_monitoring_key_pair" 22 | depends_on = ["aws_lightsail_key_pair.microservice_monitoring_key_pair"] 23 | } 24 | resource "aws_route53_record" "workshop_dns" { 25 | count = "${var.count}" 26 | zone_id = "${var.zone_id}" 27 | name = "workshop-${count.index}.${var.domain}" 28 | type = "A" 29 | ttl = "60" 30 | records = ["${element(aws_lightsail_instance.workshop.*.public_ip_address, count.index)}"] 31 | } 32 | -------------------------------------------------------------------------------- /workshop/variables.tf: -------------------------------------------------------------------------------- 1 | # Default instance size 2 | # Options: nano_1_0, micro_1_0, small_1_0, medium_1_0, large_1_0 3 | # Override: -var 'size=your-size' 4 | variable "size" { 5 | default = "medium_1_0" 6 | } 7 | 8 | 9 | # Default AWS region 10 | # Options: eu-central-1, eu-west-1, eu-west-2, us-east-1, us-east-2, us-west-2 for Lightsail 11 | # Override: -var 'region=your-region' 12 | variable "region" { 13 | default = "eu-central-1" 14 | } 15 | 16 | 17 | # Default domain 18 | # Options: You need to use your own domain that you've registered in Route53. 19 | # Override: -var 'domain=your-domain.com' 20 | variable "domain" { 21 | default = "xeraa.wtf" 22 | } 23 | 24 | 25 | # Zone ID of the domain, no default 26 | # Options: You should provide the Zone ID of the domain in the environment variable TF_VAR_zone_id 27 | # Override: -var 'zone_id=XXXXXXXXXXXXX' 28 | variable "zone_id" {} 29 | 30 | 31 | # Default number of instances to create 32 | # Options: 1 to N 33 | # Override: -var 'count=X' 34 | variable "count" { 35 | default = 1 36 | } 37 | 38 | 39 | # Operating system on AWS Lightsail 40 | # Options: Only change this at your own risk; it will probably break things. 41 | # Override: -var 'operating_system=ubuntu_16_04' 42 | variable "operating_system" { 43 | default = "ubuntu_18_04" 44 | } 45 | -------------------------------------------------------------------------------- /workshop/variables.yml: -------------------------------------------------------------------------------- 1 | # Version to install 2 | elastic_version: 6.5.2 3 | elastic_download: https://artifacts.elastic.co 4 | apm_java_version: 1.1.0 5 | gradle_version: 4.10.2 6 | php_version: 7.2 7 | 8 | 9 | # Elasticsearch credentials 10 | elasticsearch_host: http://localhost:9200/ 11 | elasticsearch_user: admin 12 | elasticsearch_password: secret 13 | kibana_host: http://localhost:5601 14 | 15 | 16 | # Setup of the infrastructure 17 | env: workshop 18 | domain: xeraa.wtf 19 | ssh_password: $6$F4NWXRFtSdCi8$DsB5vvMJYusQhSbvGXrYDXL6Xj37MUuqFCd4dGXdKd6NyxT3lpdELN07/Kpo7EjjWnm9zusFg/LLFv6oc.ynu/ # "secret", generate with mkpasswd --method=sha-512 20 | frontend_server: localhost:8080 21 | backend_server: localhost:8081 22 | apm_server: localhost:8200 23 | apm_secret: secret 24 | 25 | 26 | # MySQL 27 | mysql_server: localhost:3306 28 | mysql_user: admin 29 | mysql_password: secret 30 | mysql_database: person 31 | 32 | 33 | # SilverStripe setup 34 | silverstripe_version: 4.2.2 35 | silverstripe_user: admin 36 | silverstripe_password: secret 37 | silverstripe_database: silverstripe 38 | silverstripe_directory: /var/www/html/silverstripe/ 39 | 40 | 41 | # Credentials for Kibana dashboard-only mode 42 | attendee_user: admin 43 | attendee_password: secret 44 | --------------------------------------------------------------------------------