├── samples ├── cockpit │ ├── .gitignore │ ├── package.json │ ├── public │ │ ├── html │ │ │ └── ember-app.html │ │ └── javascripts │ │ │ └── ember-app.js │ └── app.js ├── simple-webapp │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── web │ │ │ ├── notifier │ │ │ ├── SystemOutNotifier.java │ │ │ └── HttpNotifier.java │ │ │ ├── Server.java │ │ │ └── StreamMeetupComTask.java │ └── pom.xml ├── usa-gov-logfile-parser │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── logfile │ │ │ ├── HttpNotifier.java │ │ │ └── LogfileStreamer.java │ └── pom.xml └── README.md ├── .gitignore ├── .travis.yml ├── src ├── test │ ├── resources │ │ └── log4j.properties │ └── java │ │ └── org │ │ └── elasticsearch │ │ └── metrics │ │ └── ElasticsearchReporterTest.java └── main │ └── java │ └── org │ └── elasticsearch │ └── metrics │ ├── percolation │ └── Notifier.java │ ├── JsonMetrics.java │ ├── MetricsElasticsearchModule.java │ └── ElasticsearchReporter.java ├── pom.xml ├── README.md └── LICENSE /samples/cockpit/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | data/ 3 | .idea/ 4 | *.iml 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk7 4 | -------------------------------------------------------------------------------- /samples/cockpit/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "application-name", 3 | "version": "0.0.1", 4 | "private": true, 5 | "scripts": { 6 | "start": "node app.js" 7 | }, 8 | "dependencies": { 9 | "express": "3.3.1", 10 | "superagent": "0.15.1", 11 | "underscore": "1.4.4", 12 | "sockjs" : "0.3.7" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | es.logger.level=INFO 2 | log4j.rootLogger=${es.logger.level}, out 3 | 4 | log4j.logger.org.apache.http=INFO, out 5 | log4j.additivity.org.apache.http=false 6 | 7 | log4j.appender.out=org.apache.log4j.ConsoleAppender 8 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n -------------------------------------------------------------------------------- /samples/simple-webapp/src/main/java/web/notifier/SystemOutNotifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to ElasticSearch and Shay Banon under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package web.notifier; 20 | 21 | import org.elasticsearch.metrics.JsonMetrics; 22 | import org.elasticsearch.metrics.percolation.Notifier; 23 | 24 | /** 25 | * 26 | */ 27 | public class SystemOutNotifier implements Notifier { 28 | 29 | @Override 30 | public void notify(JsonMetrics.JsonMetric jsonMetric, String id) { 31 | System.out.println(String.format("Metric %s of type %s at date %s matched with percolation id %s", jsonMetric.name(), jsonMetric.type(), jsonMetric.timestampAsDate(), id)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/metrics/percolation/Notifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.metrics.percolation; 20 | 21 | import org.elasticsearch.metrics.JsonMetrics.JsonMetric; 22 | 23 | /** 24 | * A notifier interface, which is executed, in case a certain metric is matched on the percolation query 25 | */ 26 | public interface Notifier { 27 | 28 | /** 29 | * 30 | * @param jsonMetric The json metric, which matched the percolation 31 | * @param matchedId The name of the percolation id, that matched 32 | */ 33 | void notify(JsonMetric jsonMetric, String matchedId); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /samples/simple-webapp/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | metrics-elasticsearch-reporter-sample-app 8 | metrics-elasticsearch-reporter-sample-app 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | org.elasticsearch 14 | metrics-elasticsearch-reporter 15 | 2.1.1-SNAPSHOT 16 | 17 | 18 | com.sparkjava 19 | spark-core 20 | 1.1.1 21 | 22 | 23 | joda-time 24 | joda-time 25 | 2.9 26 | 27 | 28 | org.slf4j 29 | slf4j-simple 30 | 1.7.12 31 | 32 | 33 | 34 | 35 | 36 | 37 | org.apache.maven.plugins 38 | maven-compiler-plugin 39 | 3.3 40 | 41 | 1.7 42 | 1.7 43 | 44 | 45 | 46 | org.codehaus.mojo 47 | exec-maven-plugin 48 | 1.4.0 49 | 50 | 51 | 52 | java 53 | 54 | 55 | 56 | 57 | web.Server 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /samples/usa-gov-logfile-parser/src/main/java/logfile/HttpNotifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package logfile; 20 | 21 | import org.elasticsearch.metrics.JsonMetrics; 22 | import org.elasticsearch.metrics.percolation.Notifier; 23 | 24 | import java.io.IOException; 25 | import java.net.HttpURLConnection; 26 | import java.net.URL; 27 | 28 | public class HttpNotifier implements Notifier { 29 | 30 | @Override 31 | public void notify(JsonMetrics.JsonMetric jsonMetric, String percolateMatcher) { 32 | System.out.println("Sending notification for metric " + jsonMetric.name()); 33 | 34 | try { 35 | URL url = new URL("http://localhost:3000/notify"); 36 | HttpURLConnection connection = getUrlConnection(url); 37 | StringBuilder sb = new StringBuilder("metricName="); 38 | sb.append(jsonMetric.name()); 39 | sb.append("&percolatorId="); 40 | sb.append(percolateMatcher); 41 | connection.getOutputStream().write(sb.toString().getBytes()); 42 | 43 | if (connection.getResponseCode() != 200) { 44 | System.out.println("Got broken response code " + connection.getResponseCode() + 45 | " for metric " + jsonMetric.name()); 46 | } 47 | } catch (Exception e) { 48 | e.printStackTrace(); 49 | } 50 | } 51 | 52 | private HttpURLConnection getUrlConnection(URL url) throws IOException { 53 | HttpURLConnection connection = ( HttpURLConnection ) url.openConnection(); 54 | connection.setRequestMethod("POST"); 55 | connection.setUseCaches(false); 56 | connection.setDoOutput(true); 57 | connection.connect(); 58 | return connection; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /samples/simple-webapp/src/main/java/web/notifier/HttpNotifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to ElasticSearch and Shay Banon under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package web.notifier; 20 | 21 | import org.elasticsearch.metrics.JsonMetrics; 22 | import org.elasticsearch.metrics.percolation.Notifier; 23 | 24 | import java.io.IOException; 25 | import java.net.HttpURLConnection; 26 | import java.net.URL; 27 | 28 | /** 29 | * Send a post request to the cockpit sample application, so it gets shown as a notification in there 30 | */ 31 | public class HttpNotifier implements Notifier { 32 | 33 | @Override 34 | public void notify(JsonMetrics.JsonMetric jsonMetric, String percolateMatcher) { 35 | System.out.println("Sending notification for metric " + jsonMetric.name()); 36 | 37 | try { 38 | URL url = new URL("http://localhost:3000/notify"); 39 | HttpURLConnection connection = getUrlConnection(url); 40 | String data = String.format("metricName=%s&percolatorId=%s", jsonMetric.name(), percolateMatcher); 41 | connection.getOutputStream().write(data.getBytes()); 42 | 43 | if (connection.getResponseCode() != 200) { 44 | System.out.println("Got broken response code " + connection.getResponseCode() + " for metric " + jsonMetric.name()); 45 | } 46 | } catch (Exception e) { 47 | e.printStackTrace(); 48 | } 49 | } 50 | 51 | private HttpURLConnection getUrlConnection(URL url) throws IOException { 52 | HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 53 | connection.setRequestMethod("POST"); 54 | connection.setUseCaches(false); 55 | connection.setDoOutput(true); 56 | connection.connect(); 57 | return connection; 58 | }} 59 | -------------------------------------------------------------------------------- /samples/usa-gov-logfile-parser/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.elasticsearch.metrics 8 | usgov-logfile-parser 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | org.elasticsearch 14 | elasticsearch 15 | 2.0.0 16 | 17 | 18 | org.elasticsearch 19 | metrics-elasticsearch-reporter 20 | 2.1.1-SNAPSHOT 21 | 22 | 23 | com.fasterxml.jackson.core 24 | jackson-core 25 | 2.5.3 26 | 27 | 28 | com.fasterxml.jackson.core 29 | jackson-databind 30 | 2.5.3 31 | 32 | 33 | org.slf4j 34 | slf4j-simple 35 | 1.7.12 36 | 37 | 38 | 39 | 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-compiler-plugin 44 | 3.3 45 | 46 | 1.7 47 | 1.7 48 | 49 | 50 | 51 | org.codehaus.mojo 52 | exec-maven-plugin 53 | 1.4.0 54 | 55 | 56 | 57 | java 58 | 59 | 60 | 61 | 62 | logfile.LogfileStreamer 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /samples/README.md: -------------------------------------------------------------------------------- 1 | # Sample applications 2 | 3 | In order to check out these applications you need `maven` for the logfile parser and `node`/`npm` for the cockpit application. 4 | 5 | ## Logfile parser 6 | 7 | This is a stupidly simple project, to show how the metrics library can be used. 8 | The tool reads the stream of the currently visited US government sites and indexes the location as a geohash. 9 | 10 | The application features several metrics, like the metering the incoming requests, counting the heartbeats or checking how many indexing requests to elasticsearch are sent. 11 | 12 | ## Cockpit - a real time notification dashboard 13 | 14 | This little dashboard application can be used to draw graphs from the indexed metrics as well as getting realtime notifications. The application allows you to 15 | 16 | * Draw graphs from percolations 17 | * Add percolations for real-time notifications 18 | * Receive real-time notifications from the logfile parser application and immediately display them in the browser using websockets. 19 | 20 | ## Getting up and running 21 | 22 | Run this in the top level directory to put the metrics reporter into your local maven repo 23 | 24 | ``` 25 | mvn install 26 | ``` 27 | 28 | To get up and running you do not need to have elasticsearch installed, as it is started by default from the sample application. 29 | 30 | ``` 31 | cd samples/usa-gov-logfile-parser 32 | mvn compile exec:java 33 | ``` 34 | 35 | **Note**: The above command starts an own elasticsearch instance. 36 | If you already have an elasticsearch instance up and running, call `mvn exec:java -Dcreate.es.instance=no -Dcluster.name=YourClusterName`. The default cluster name is `metrics`, so it is likely you will have to change it, if you already have an elasticsearch instance running. 37 | 38 | Then fire up another terminal and prepare cockpit application 39 | 40 | ``` 41 | cd samples/cockpit 42 | npm install 43 | node app.js 44 | ``` 45 | 46 | **Note**: You need at least node v0.8 in order to get the application up and running. 47 | 48 | Open up your browser to `http://localhost:3000` 49 | 50 | The little application allows you to do one of the following 51 | 52 | * Draw a metric, like `usagov-indexing-requests` (just type it in the searchbar, it has autocomplete) 53 | * Add a percolation: `id` must be unique, i.e. `my-percolator-match`, name can be again `usagov-indexing-requests`, field could be the `m1_rate` and range selection could be `> 0.1` to make sure it matches everytime, so you see it works 54 | * Wait for a percolation to match, then delete it by clicking on the trashcan in the notification, if the notification is not important any more. 55 | 56 | # Sample application: simple web application 57 | 58 | The simple-webapp/ directory contains a very simple web application, which not only generates metrics for the incoming web requests, but also for a running background task, which reads the meetup.com reservations stream. You can start it by running `mvn clean compile exec:java` 59 | 60 | The application also includes a `HttpNotifier` which reports to the cockpit sample application, but just prints out percolated metrics on standard out by default. 61 | 62 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/metrics/JsonMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.metrics; 20 | 21 | import com.codahale.metrics.*; 22 | import java.util.Date; 23 | 24 | public class JsonMetrics { 25 | 26 | /** 27 | * A abstract json metric class, from which all other classes inherit 28 | * The other classes are simply concrete json implementations of the existing metrics classes 29 | */ 30 | public static abstract class JsonMetric { 31 | private final String name; 32 | private long timestamp; 33 | private final T value; 34 | 35 | public JsonMetric(String name, long timestamp, T value) { 36 | this.name = name; 37 | this.timestamp = timestamp; 38 | this.value = value; 39 | } 40 | 41 | public String name() { 42 | return name; 43 | } 44 | 45 | public long timestamp() { 46 | return timestamp; 47 | } 48 | 49 | public Date timestampAsDate() { 50 | return new Date(timestamp * 1000); 51 | } 52 | 53 | public T value() { 54 | return value; 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return String.format("%s %s %s", type(), name, timestamp); 60 | } 61 | 62 | public abstract String type(); 63 | } 64 | 65 | public static class JsonGauge extends JsonMetric { 66 | private static final String TYPE = "gauge"; 67 | 68 | public JsonGauge(String name, long timestamp, Gauge value) { 69 | super(name, timestamp, value); 70 | } 71 | 72 | @Override 73 | public String type() { 74 | return TYPE; 75 | } 76 | } 77 | 78 | public static class JsonCounter extends JsonMetric { 79 | private static final String TYPE = "counter"; 80 | 81 | public JsonCounter(String name, long timestamp, Counter value) { 82 | super(name, timestamp, value); 83 | } 84 | 85 | @Override 86 | public String type() { 87 | return TYPE; 88 | } 89 | } 90 | 91 | public static class JsonHistogram extends JsonMetric { 92 | private static final String TYPE = "histogram"; 93 | 94 | public JsonHistogram(String name, long timestamp, Histogram value) { 95 | super(name, timestamp, value); 96 | } 97 | 98 | @Override 99 | public String type() { 100 | return TYPE; 101 | } 102 | } 103 | 104 | public static class JsonMeter extends JsonMetric { 105 | private static final String TYPE = "meter"; 106 | public JsonMeter(String name, long timestamp, Meter value) { 107 | super(name, timestamp, value); 108 | } 109 | 110 | @Override 111 | public String type() { 112 | return TYPE; 113 | } 114 | } 115 | 116 | public static class JsonTimer extends JsonMetric { 117 | private static final String TYPE = "timer"; 118 | 119 | public JsonTimer(String name, long timestamp, Timer value) { 120 | super(name, timestamp, value); 121 | } 122 | 123 | @Override 124 | public String type() { 125 | return TYPE; 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /samples/simple-webapp/src/main/java/web/Server.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to ElasticSearch and Shay Banon under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package web; 20 | 21 | import com.codahale.metrics.MetricFilter; 22 | import com.codahale.metrics.MetricRegistry; 23 | import com.codahale.metrics.Timer; 24 | import org.elasticsearch.metrics.ElasticsearchReporter; 25 | import spark.Filter; 26 | import spark.Request; 27 | import spark.Response; 28 | import spark.Route; 29 | import web.notifier.SystemOutNotifier; 30 | 31 | import java.util.Random; 32 | import java.util.concurrent.ExecutorService; 33 | import java.util.concurrent.Executors; 34 | import java.util.concurrent.TimeUnit; 35 | 36 | import static spark.Spark.*; 37 | 38 | /** 39 | * 40 | */ 41 | public class Server { 42 | 43 | public static void main(String[] args) throws Exception { 44 | 45 | // configure reporter 46 | final MetricRegistry metrics = new MetricRegistry(); 47 | ElasticsearchReporter reporter = ElasticsearchReporter.forRegistry(metrics) 48 | // support for several es nodes 49 | .hosts("localhost:9200", "localhost:9201") 50 | // just create an index, no date format, means one index only 51 | .index("metrics") 52 | .indexDateFormat(null) 53 | // define a percolation check on all metrics 54 | .percolationFilter(MetricFilter.ALL) 55 | .percolationNotifier(new SystemOutNotifier()) 56 | //.percolationNotifier(new HttpNotifier()) 57 | .build(); 58 | // usually you set this to one minute 59 | reporter.start(10, TimeUnit.SECONDS); 60 | 61 | // start up background thread 62 | ExecutorService executorService = Executors.newSingleThreadExecutor(); 63 | executorService.submit(new StreamMeetupComTask(metrics)); 64 | 65 | // start up web app 66 | get(new Route("/") { 67 | @Override 68 | public Object handle(Request request, Response response) { 69 | return "this is /"; 70 | } 71 | }); 72 | 73 | post(new Route("/checkout") { 74 | @Override 75 | public Object handle(Request request, Response response) { 76 | // emulate calling external payment API 77 | Timer.Context timer = metrics.timer("payment.runtime").time(); 78 | 79 | try { 80 | Thread.sleep(new Random().nextInt(1000)); 81 | } catch (InterruptedException e) { 82 | // ignore 83 | } finally { 84 | timer.stop(); 85 | } 86 | 87 | response.redirect("/"); 88 | return null; 89 | } 90 | }); 91 | 92 | before(new Filter() { 93 | @Override 94 | public void handle(Request request, Response response) { 95 | metrics.counter("http.connections.concurrent").inc(); 96 | metrics.meter("http.connections.requests").mark(); 97 | metrics.counter("http.connections.stats." + request.raw().getMethod()).inc(); 98 | metrics.counter("http.connections.stats." + request.raw().getMethod() + "." + request.pathInfo()).inc(); 99 | } 100 | }); 101 | 102 | after(new Filter() { 103 | @Override 104 | public void handle(Request request, Response response) { 105 | metrics.counter("http.connections.concurrent").dec(); 106 | } 107 | }); 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /samples/simple-webapp/src/main/java/web/StreamMeetupComTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to ElasticSearch and Shay Banon under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package web; 20 | 21 | import com.codahale.metrics.MetricRegistry; 22 | import com.fasterxml.jackson.databind.MappingIterator; 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | import com.fasterxml.jackson.databind.ObjectReader; 25 | import org.joda.time.DateTime; 26 | import org.joda.time.DateTimeZone; 27 | import org.joda.time.Duration; 28 | 29 | import java.io.IOException; 30 | import java.io.InputStream; 31 | import java.net.HttpURLConnection; 32 | import java.net.URL; 33 | import java.util.Map; 34 | import java.util.TimeZone; 35 | import java.util.concurrent.Callable; 36 | 37 | /** 38 | * 39 | */ 40 | public class StreamMeetupComTask implements Callable { 41 | 42 | private MetricRegistry metrics; 43 | 44 | public StreamMeetupComTask(MetricRegistry metrics) { 45 | this.metrics = metrics; 46 | } 47 | 48 | @Override 49 | public Void call() { 50 | try { 51 | ObjectMapper objectMapper = new ObjectMapper(); 52 | ObjectReader reader = objectMapper.reader(Map.class); 53 | MappingIterator> iterator = reader.readValues(getInputStream()); 54 | 55 | while (iterator.hasNextValue()) { 56 | Map entry = iterator.nextValue(); 57 | 58 | // monitor the distribution of countries 59 | if (entry.containsKey("group") && entry.get("group") instanceof Map) { 60 | Map group = (Map) entry.get("group"); 61 | if (group.containsKey("group_country")) { 62 | metrics.meter("meetup.country." + group.get("group_country")).mark(); 63 | metrics.meter("meetup.country.total").mark(); 64 | } 65 | } 66 | 67 | // monitor the distribution of the number of guests 68 | if (entry.containsKey("guests") && entry.get("guests") instanceof Long) { 69 | metrics.histogram("meetup.guests").update((Long)entry.get("guests")); 70 | } 71 | 72 | // monitor reservation time upfront, 1d, 4d, 1w, 2w, 1m, 2m, - 73 | if (entry.containsKey("event") && entry.get("event") instanceof Map) { 74 | Map event = (Map) entry.get("event"); 75 | if (event.get("time") instanceof Long) { 76 | metrics.counter("meetup.reservation.time.total").inc(); 77 | metrics.counter("meetup.reservation.time." + getUpfrontReservationTime((Long) event.get("time"))).inc(); 78 | } 79 | } 80 | } 81 | } catch (Exception e) { 82 | e.printStackTrace(); 83 | } 84 | 85 | return null; 86 | } 87 | 88 | private InputStream getInputStream() throws IOException { 89 | URL url = new URL("http://stream.meetup.com/2/rsvps"); 90 | HttpURLConnection request = (HttpURLConnection) url.openConnection(); 91 | return request.getInputStream(); 92 | } 93 | 94 | // -1 (in past), 1d, 4d, 1w, 2w, 1m, 2m, - 95 | private String getUpfrontReservationTime(Long dateInMillis) { 96 | DateTime now = new DateTime(DateTimeZone.forTimeZone(TimeZone.getTimeZone("UTC"))); 97 | DateTime event = new DateTime(dateInMillis, DateTimeZone.forTimeZone(TimeZone.getTimeZone("UTC"))); 98 | Duration duration = new Duration(now, event); 99 | 100 | if (duration.getMillis() < 0) { 101 | return "-1"; 102 | } else if (duration.getStandardSeconds() < 86400) { 103 | return "1d"; 104 | } else if (duration.getStandardDays() < 4) { 105 | return "4d"; 106 | } else if (duration.getStandardDays() < 7) { 107 | return "1w"; 108 | } else if (duration.getStandardDays() < 14) { 109 | return "2w"; 110 | } else if (duration.getStandardDays() < 28) { 111 | return "4w"; 112 | } else if (duration.getStandardDays() < 56) { 113 | return "8w"; 114 | } else { 115 | return "-"; 116 | } 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /samples/cockpit/public/html/ember-app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Monitoring cockpit 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 109 | 110 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.elasticsearch 5 | metrics-elasticsearch-reporter 6 | 2.3.0-SNAPSHOT 7 | 8 | 9 | 5.5.0 10 | 2.3.1 11 | 2.7.3 12 | 2.3.3 13 | 14 | 15 | 4.0.0 16 | jar 17 | metrics-elasticsearch-reporter 18 | Reporter for the metrics library which reports into Elasticsearch 19 | 2013 20 | 21 | 22 | scm:git:git@github.com:elasticsearch/elasticsearch-metrics-reporter-java.git 23 | scm:git:git@github.com:elasticsearch/elasticsearch-metrics-reporter-java.git 24 | http://github.com/elasticsearch/elasticsearch-metrics-reporter 25 | 26 | 27 | 28 | 29 | The Apache Software License, Version 2.0 30 | http://www.apache.org/licenses/LICENSE-2.0.txt 31 | repo 32 | 33 | 34 | 35 | 36 | org.sonatype.oss 37 | oss-parent 38 | 9 39 | 40 | 41 | 42 | GitHub 43 | https://github.com/elasticsearch/elasticsearch-metrics-reporter-java/issues/ 44 | 45 | 46 | 47 | 48 | 49 | org.apache.maven.plugins 50 | maven-surefire-plugin 51 | 2.19.1 52 | 53 | 54 | false 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-compiler-plugin 61 | 3.5 62 | 63 | 1.7 64 | 1.7 65 | 66 | 67 | 68 | org.apache.maven.plugins 69 | maven-source-plugin 70 | 2.4 71 | 72 | 73 | attach-sources 74 | 75 | jar 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | io.dropwizard.metrics 86 | metrics-core 87 | 3.1.2 88 | 89 | 90 | com.fasterxml.jackson.core 91 | jackson-databind 92 | ${jackson.version} 93 | 94 | 95 | com.fasterxml.jackson.module 96 | jackson-module-afterburner 97 | ${jackson.version} 98 | 99 | 100 | org.apache.lucene 101 | lucene-test-framework 102 | ${lucene.version} 103 | test 104 | 105 | 106 | org.elasticsearch 107 | elasticsearch 108 | ${elasticsearch.version} 109 | test 110 | test-jar 111 | 112 | 113 | org.elasticsearch 114 | elasticsearch 115 | ${elasticsearch.version} 116 | test 117 | 118 | 119 | com.carrotsearch.randomizedtesting 120 | randomizedtesting-runner 121 | ${randomized.testrunner.version} 122 | test 123 | 124 | 125 | org.slf4j 126 | slf4j-simple 127 | 1.7.19 128 | test 129 | 130 | 131 | junit 132 | junit 133 | 4.12 134 | test 135 | 136 | 137 | org.hamcrest 138 | hamcrest-core 139 | 140 | 141 | 142 | 143 | org.hamcrest 144 | hamcrest-all 145 | 1.3 146 | test 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /samples/cockpit/app.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | var express = require('express') 21 | , http = require('http') 22 | , sockjs = require('sockjs') 23 | , request = require('superagent') 24 | , _ = require('underscore') 25 | , path = require('path'); 26 | 27 | var configuration = { es : { url: "http://localhost:9200/metrics/_search", percolateUrl: "http://localhost:9200/_percolator/" } } 28 | var app = express(); 29 | 30 | // all environments 31 | app.set('port', process.env.PORT || 3000); 32 | app.use(express.favicon()); 33 | app.use(express.logger('dev')); 34 | app.use(express.bodyParser()); 35 | app.use(express.methodOverride()); 36 | app.use(app.router); 37 | app.use(express.static(path.join(__dirname, 'public'))); 38 | 39 | // development only 40 | if ('development' == app.get('env')) { 41 | app.use(express.errorHandler()); 42 | } 43 | 44 | // sockjs init 45 | var sockjs_opts = {sockjs_url: "http://cdn.sockjs.org/sockjs-0.3.min.js"}; 46 | 47 | var wsConnections = []; 48 | var sockjs_echo = sockjs.createServer(sockjs_opts); 49 | sockjs_echo.on('connection', function(conn) { 50 | wsConnections.push(conn); 51 | conn.on('close', function() { 52 | for (var i = 0; i < wsConnections.length; i++) { 53 | if (wsConnections[i] === conn) wsConnections.splice(i, 1); 54 | } 55 | }); 56 | }); 57 | 58 | // one page app FTW 59 | app.get('/', function(req, res) { res.sendfile('public/html/ember-app.html') /*res.render('index', { title: 'Express' })*/ }); 60 | 61 | // get all available names from es 62 | // OUTPUT: list of [{name:'', type:''}] 63 | app.get('/graphNames', function(req, res) { 64 | request.post(configuration.es.url) 65 | .send({ size: 0, facets: { names : { terms : { script_field: 'doc["_type"].value + ":" + doc["name"].value', size: 500 } } } }) 66 | .type('json') 67 | .on('error', function(err) { console.log("Error connecting to " + configuration.es.url + ": " + err) }) 68 | .end(function(response) { 69 | var data = _.map(response.body.facets.names.terms, function(entry) { d=entry.term.split(':') ; return { type: d[0], name: d[1] } } ); 70 | res.end(JSON.stringify(data)) 71 | }); 72 | }); 73 | 74 | // get all available data points for a graph name 75 | // INPUT: graph name, metric name, from timestamp, to timestamp 76 | // OUTPUT: array of data points 77 | app.get('/graphData', function(req, res) { 78 | var name = req.query.name 79 | var type = req.query.type 80 | var time = req.query.time 81 | var fields = [ "name", "@timestamp", req.query.values ] 82 | 83 | request.post(configuration.es.url) 84 | // max size is one month, so it can be 24 * 60 * 31, usually we should calculate this dependent on input size 85 | .send({ size: 44640, filter: { and: [ { term: { name: name } }, { type : { value: type } }, { range: { '@timestamp': { from: 'now-' + time, to: 'now' } }} ] }, fields: fields, sort: [ { '@timestamp': { order: "asc" }} ] }) 86 | .type('json') 87 | .on('error', function(err) { console.log("Error connecting to " + configuration.es.url + ": " + err) }) 88 | .end(function(response) { 89 | var hits = response.body.hits.hits 90 | //console.log(JSON.stringify(hits)); 91 | var valuesToGraph = req.query.values; 92 | var dataToRenderOut = [] 93 | for (var i = 0 ; i < valuesToGraph.length ; i++) { 94 | var currentName = req.query.values[i] 95 | var values = _.map(hits, function(hit) { return { x: hit.sort[0], y: hit.fields[currentName] } }) 96 | dataToRenderOut.push({key:currentName, values: values}) 97 | } 98 | res.end(JSON.stringify(dataToRenderOut)) 99 | }); 100 | }); 101 | 102 | app.post('/notify', function(req, res) { 103 | for (var i = 0; i < wsConnections.length; i++) { 104 | wsConnections[i].write(JSON.stringify({ metricName: req.body.metricName, percolatorId: req.body.percolatorId})) 105 | } 106 | res.end(); 107 | }); 108 | 109 | // TODO REGISTER NEW NOTIFICATION/PERCOLATION 110 | app.post('/percolator', function(req, res) { 111 | var id = req.body.id 112 | var field = req.body.field 113 | var name = req.body.name 114 | var range = req.body.range 115 | var rangeValue = req.body.rangeValue 116 | 117 | var url = configuration.es.percolateUrl + "/metrics/" + id 118 | var query = { query : { bool : { must : [ { term : { name: name } } ] } } } 119 | var rangeQuery = { range : {}} 120 | if (range === '<') rangeQuery.range[field] = { to: rangeValue, include_upper:false } 121 | if (range === '<=') rangeQuery.range[field] = { to: rangeValue, include_upper:true } 122 | if (range === '>') rangeQuery.range[field] = { from: rangeValue, include_lower:false } 123 | if (range === '=>') rangeQuery.range[field] = { from: rangeValue, include_lower:true } 124 | 125 | query.query.bool.must.push(rangeQuery) 126 | request.post(url).type("json") 127 | .send(query) 128 | .on('error', function(err) { console.log("Error connecting to " + url + ": " + err) }) 129 | .end(function(response) { 130 | //console.log(JSON.stringify(response.body)) 131 | res.end(); 132 | }) 133 | }); 134 | 135 | // delete percolator 136 | app.delete('/percolator', function(req, res) { 137 | var id = req.body.id 138 | var url = configuration.es.percolateUrl + "/metrics/" + id 139 | console.log("URL " + url) 140 | request.del(url).type('json') 141 | .send() 142 | .on('error', function(err) { console.log("Error connecting to " + url + ": " + err) }) 143 | .end(function() { res.end(); }) 144 | }) 145 | 146 | var server = http.createServer(app) 147 | sockjs_echo.installHandlers(server, {prefix:'/ws'}); 148 | 149 | server.listen(app.get('port'), function(){ 150 | console.log('Express server listening on port ' + app.get('port')); 151 | }); 152 | -------------------------------------------------------------------------------- /samples/cockpit/public/javascripts/ember-app.js: -------------------------------------------------------------------------------- 1 | // Ember App 2 | App = Ember.Application.create(); 3 | 4 | // static metric type configuration 5 | // first one is the default 6 | App.metricTypes = { 7 | meter: [ 'm1_rate', 'count', 'mean_rate', 'm5_rate', 'm15_rate' ], 8 | timer: [ 'm1_rate', 'count', 'mean_rate', 'm5_rate', 'm15_rate', 'min', 'max', 'mean', 'p50', 'p75', 'p95', 'p98', 'p99', 'p999', 'stddev' ], 9 | counter: [ 'count' ], 10 | histogram: [ 'mean', 'count', 'min', 'max', 'p50', 'p75', 'p95', 'p98', 'p99', 'p999', 'stddev' ], 11 | gauge: [ 'value' ] 12 | } 13 | 14 | App.metricTimes = [ '1d', '1h', '12h', '1w', '1m' ] 15 | 16 | App.percolatorOperators = [ '>', '>=', '<', '<=', '=',] 17 | 18 | // models 19 | App.Notification = Ember.Object.extend({metricName:null, percolatorId: null}) 20 | App.Percolation = Ember.Object.extend({ id: null, name: null, field: null, operator: null, value: null}) 21 | App.Graph = Ember.Object.extend({ 22 | id: null, 23 | 24 | draw : function(metricName, metricValue, metricTime) { 25 | var anyParameterEmpty = _.some([metricName, metricValue, metricTime], _.isEmpty) 26 | if (anyParameterEmpty) return; 27 | 28 | var graph = _.find(App.metricNames, function(entry) { return entry.name === metricName }) 29 | var id = this.get("id") 30 | 31 | var values = [ metricValue ] 32 | // Including min/max vastly destroys the graphs scale in my tests, so I left it out for now 33 | // if (_.contains(App.metricTypes[graph.type], 'max')) values.push('max') 34 | // if (_.contains(App.metricTypes[graph.type], 'min')) values.push('min') 35 | 36 | $.getJSON('/graphData', {name: graph.name, type: graph.type, values: values, time: metricTime}, function(data) { 37 | 38 | nv.addGraph(function() { 39 | var chart = nv.models.lineWithFocusChart(); 40 | chart.yAxis.tickFormat(d3.format(',.2f')); 41 | chart.y2Axis.tickFormat(d3.format(',.2f')); 42 | 43 | chart.xAxis.tickFormat(function (d) { return moment(d).format("HH:mm") }); 44 | chart.x2Axis.tickFormat(function (d) { return moment(d).format("HH:mm") }); 45 | 46 | d3.select("#" + id + ' svg') 47 | .datum(data) 48 | .transition().duration(500) 49 | .call(chart); 50 | 51 | nv.utils.windowResize(chart.update); 52 | 53 | return chart; 54 | }); 55 | }); 56 | } 57 | }) 58 | 59 | // list of all active notifications, graph init (would allow us to create arbitrary/many graphs!) 60 | App.notifications = Ember.A(); 61 | App.graph = App.Graph.create({id: "chart"}) 62 | 63 | // start route 64 | App.IndexRoute = Ember.Route.extend({ 65 | model: function () { 66 | return App.notifications; 67 | } 68 | }); 69 | 70 | App.NotificationController = Ember.ObjectController.extend({ 71 | removeNotification: function () { 72 | var notification = this.get('model'); 73 | App.notifications.removeObject(notification) 74 | }, 75 | deletePercolation : function() { 76 | var notification = this.get('model'); 77 | $.ajax({ url: '/percolator', data: { id: this.get('percolatorId') }, type: 'DELETE' }) 78 | this.removeNotification() 79 | }, 80 | drawMetric: function() { 81 | var notification = this.get('model'); 82 | var metric = _.find(App.metricNames, function(entry) { return entry.name === notification.metricName }) 83 | if (metric != undefined) { 84 | App.graph.draw(metric.name, App.metricTypes[metric.type][0]) 85 | } 86 | } 87 | }); 88 | 89 | App.IndexController = Ember.ArrayController.extend({ 90 | drawGraph: function() { 91 | App.graph.draw(this.get('metricName'), this.get('metricValue'), this.get('metricTime')) 92 | }, 93 | 94 | metricNameDidChange : Ember.observer(function() { 95 | this.setEmberSelectBox('metricName', 'metricValues', 'metricValue'); 96 | },'metricName'), 97 | 98 | metricTimeDidChange : Ember.observer(function() { 99 | this.drawGraph(); 100 | }, 'metricTime'), 101 | 102 | percolatorMetricNameDidChange : Ember.observer(function() { 103 | this.setEmberSelectBox('percolatorMetricName', 'percolatorFieldValues', 'percolatorFieldValue') 104 | },'percolatorMetricName'), 105 | 106 | setEmberSelectBox: function(inputFieldProperty, allowedValuesProperties, selectedAllowedValue) { 107 | var metricName = this.get(inputFieldProperty) 108 | if (metricName.length < 1) return; 109 | 110 | var metric = _.find(App.metricNames, function(entry) { return entry.name === metricName }) 111 | if (metric !== undefined) { 112 | var allowedMetricValues = App.metricTypes[metric.type]; 113 | this.set(allowedValuesProperties, allowedMetricValues) 114 | this.set(selectedAllowedValue, allowedMetricValues[0]) 115 | } 116 | }, 117 | 118 | savePercolation : function() { 119 | var that = this 120 | $.post('/percolator', { id: this.get('percolatorId'), 121 | name: this.get('percolatorMetricName'), 122 | field: this.get('percolatorFieldValue'), 123 | range: this.get('percolatorOperator'), 124 | rangeValue: this.get('percolatorValue') 125 | }).done(function() { 126 | $('#myModal').modal('hide') 127 | that.set('percolatorId', '') 128 | that.set('percolatorMetricName', '') 129 | that.set('percolatorFieldValue', '') 130 | that.set('percolatorOperator', '>') 131 | that.set('percolatorValue', '') 132 | }) 133 | } 134 | }); 135 | 136 | // On app load, load graph names 137 | App.ready = function() { 138 | $.getJSON('/graphNames', function(data) { 139 | App.metricNames = data 140 | // not sure if this can be emberized.. works now 141 | $('#search').typeahead({source: _.map(data, function(val) { return val.name })}) 142 | $('#metricName').typeahead({source: _.map(data, function(val) { return val.name })}) 143 | }) 144 | 145 | // SockJS init 146 | var sockjs = new SockJS('/ws'); 147 | sockjs.onclose = function() { console.log("Web socket close, maybe reload?") } 148 | sockjs.onmessage = function(message) { 149 | var data = $.parseJSON(message.data) 150 | // add only if it is not yet in list 151 | var isNotificationAlreadyInList = App.notifications.some(function(item, index, enumerable) { return item.metricName == data.metricName }) 152 | if (!isNotificationAlreadyInList) { 153 | App.notifications.pushObject(App.Notification.create({metricName: data.metricName, percolatorId: data.percolatorId })) 154 | } 155 | } 156 | } 157 | 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Metrics Elasticsearch Reporter 2 | 3 | ![Project unmaintained](https://img.shields.io/badge/project-unmaintained-red.svg) 4 | 5 | **This project is no longer maintained. If you want to maintain it, please fork and we will link to your new repository.** 6 | 7 | ## Introduction 8 | 9 | This is a reporter for the excellent [Metrics library](http://metrics.dropwizard.io/), similar to the [Graphite](http://metrics.dropwizard.io/3.1.0/manual/graphite/) or [Ganglia](http://metrics.dropwizard.io/3.1.0/manual/ganglia/) reporters, except that it reports to an Elasticsearch server. 10 | 11 | In case, you are worried, that you need to include the 20MB elasticsearch dependency in your project, you do not need to be. As this reporter is using HTTP for putting data into elasticsearch, the only library needed is the awesome [Jackson JSON library](http://wiki.fasterxml.com/JacksonHome), more exactly the Jackson Databind library to easily serialize the metrics objects. 12 | 13 | If you want to see this in action, go to the `samples/` directory and read the readme over there, to get up and running with a sample application using the Metrics library as well as a dashboard application to graph. 14 | 15 | ## Compatibility 16 | 17 | | Metrics-elasticsearch-reporter | elasticsearch | Release date | 18 | |-----------------------------------|---------------------|:------------:| 19 | | 2.3.0-SNAPSHOT | 2.3.0 -> master | NONE | 20 | | 2.2.0 | 2.2.0 -> 2.2.x | 2016-02-10 | 21 | | 2.0 | 1.0.0 -> 1.7.x | 2014-02-16 | 22 | | 1.0 | 0.90.7 -> 0.90.x | 2014-02-05 | 23 | 24 | ## Travis CI build status 25 | 26 | [![Build status](https://api.travis-ci.org/elastic/elasticsearch-metrics-reporter-java.svg?branch=master)](https://travis-ci.org/elastic/elasticsearch-metrics-reporter-java) 27 | 28 | ## Installation 29 | 30 | You can simply add a dependency in your `pom.xml` (or whatever dependency resolution system you might have) 31 | 32 | ``` 33 | 34 | org.elasticsearch 35 | metrics-elasticsearch-reporter 36 | 2.2.0 37 | 38 | ``` 39 | 40 | ## Configuration 41 | 42 | ``` 43 | final MetricRegistry registry = new MetricRegistry(); 44 | ElasticsearchReporter reporter = ElasticsearchReporter.forRegistry(registry) 45 | .hosts("localhost:9200", "localhost:9201") 46 | .build(); 47 | reporter.start(60, TimeUnit.SECONDS); 48 | ``` 49 | 50 | Define your metrics and registries as usual 51 | 52 | ``` 53 | private final Meter incomingRequestsMeter = registry.meter("incoming-http-requests"); 54 | 55 | // in your app code 56 | incomingRequestsMeter.mark(1); 57 | ``` 58 | 59 | 60 | ### Options 61 | 62 | * `hosts()`: A list of hosts used to connect to, must be in the format `hostname:port`, default is `localhost:9200` 63 | * `timeout()`: Milliseconds to wait for an established connections, before the next host in the list is tried. Defaults to `1000` 64 | * `bulkSize()`: Defines how many metrics are sent per bulk requests, defaults to `2500` 65 | * `filter()`: A `MetricFilter` to define which metrics written to the elasticsearch 66 | * `percolationFilter()`: A `MetricFilter` to define which metrics should be percolated against. See below for an example 67 | * `percolationNotifier()`: An implementation of the `Notifier` interface, which is executed upon a matching percolator. See below for an example. 68 | * `index()`: The name of the index to write to, defaults to `metrics` 69 | * `indexDateFormat()`: The date format to make sure to rotate to a new index, defaults to `yyyy-MM` 70 | * `timestampFieldname()`: The field name of the timestamp, defaults to `@timestamp`, which makes it easy to use with kibana 71 | 72 | ### Mapping 73 | 74 | **Note**: The reporter automatically checks for the existence of an index template called `metrics_template`. If this template does not exist, it is created. This template ensures that all strings used in metrics are set to `not_analyzed` and disables the `_all` field. 75 | 76 | 77 | ## Notifications with percolations 78 | 79 | ``` 80 | ElasticsearchReporter reporter = ElasticsearchReporter.forRegistry(registry) 81 | .percolationNotifier(new PagerNotifier()) 82 | .percolationFilter(MetricFilter.ALL) 83 | .build(); 84 | reporter.start(60, TimeUnit.SECONDS); 85 | ``` 86 | 87 | Write a custom notifier 88 | 89 | ``` 90 | public class PagerNotifier implements Notifier { 91 | 92 | @Override 93 | public void notify(JsonMetrics.JsonMetric jsonMetric, String percolateMatcher) { 94 | // send pager duty here 95 | } 96 | } 97 | ``` 98 | 99 | Add a percolation 100 | 101 | ``` 102 | curl http://localhost:9200/metrics/.percolator/http-monitor -X PUT -d '{ 103 | "query" : { 104 | "bool" : { 105 | "must": [ 106 | { "term": { "name" : "incoming-http-requests" } }, 107 | { "range": { "m1_rate": { "to" : "10" } } } 108 | ] 109 | } 110 | } 111 | }' 112 | ``` 113 | 114 | ## JSON Format of metrics 115 | 116 | This is how the serialized metrics looks like in elasticsearch 117 | 118 | ### Counter 119 | 120 | ``` 121 | { 122 | "name": "usa-gov-heartbearts", 123 | "@timestamp": "2013-07-20T09:29:58.000+0000", 124 | "count": 18 125 | } 126 | ``` 127 | 128 | ### Timer 129 | 130 | ``` 131 | { 132 | "name" : "bulk-request-timer", 133 | "@timestamp" : "2013-07-20T09:43:58.000+0000", 134 | "count" : 114, 135 | "max" : 109.681, 136 | "mean" : 5.439666666666667, 137 | "min" : 2.457, 138 | "p50" : 4.3389999999999995, 139 | "p75" : 5.0169999999999995, 140 | "p95" : 8.37175, 141 | "p98" : 9.6832, 142 | "p99" : 94.68429999999942, 143 | "p999" : 109.681, 144 | "stddev" : 9.956913151098842, 145 | "m15_rate" : 0.10779994503690074, 146 | "m1_rate" : 0.07283351433589833, 147 | "m5_rate" : 0.10101298115113727, 148 | "mean_rate" : 0.08251056571678642, 149 | "duration_units" : "milliseconds", 150 | "rate_units" : "calls/second" 151 | } 152 | ``` 153 | 154 | ### Meter 155 | 156 | ``` 157 | { 158 | "name" : "usagov-incoming-requests", 159 | "@timestamp" : "2013-07-20T09:29:58.000+0000", 160 | "count" : 224, 161 | "m1_rate" : 0.3236309568191993, 162 | "m5_rate" : 0.45207208204948995, 163 | "m15_rate" : 0.5014348927301423, 164 | "mean_rate" : 0.4135529888278531, 165 | "units" : "events/second" 166 | } 167 | ``` 168 | 169 | ### Histogram 170 | 171 | ``` 172 | { 173 | "name" : "my-histgram", 174 | "@timestamp" : "2013-07-20T09:29:58.000+0000", 175 | "count" : 114, 176 | "max" : 109.681, 177 | "mean" : 5.439666666666667, 178 | "min" : 2.457, 179 | "p50" : 4.3389999999999995, 180 | "p75" : 5.0169999999999995, 181 | "p95" : 8.37175, 182 | "p98" : 9.6832, 183 | "p99" : 94.68429999999942, 184 | "p999" : 109.681, 185 | "stddev" : 9.956913151098842,} 186 | } 187 | ``` 188 | 189 | ### Gauge 190 | 191 | ``` 192 | { 193 | "name" : "usagov-incoming-requests", 194 | "@timestamp" : "2013-07-20T09:29:58.000+0000", 195 | "value" : 123 196 | } 197 | ``` 198 | 199 | 200 | ## Next steps 201 | 202 | * Integration with Kibana would be awesome 203 | 204 | -------------------------------------------------------------------------------- /samples/usa-gov-logfile-parser/src/main/java/logfile/LogfileStreamer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package logfile; 20 | 21 | import com.codahale.metrics.Counter; 22 | import com.codahale.metrics.Meter; 23 | import com.codahale.metrics.MetricFilter; 24 | import com.codahale.metrics.MetricRegistry; 25 | import com.codahale.metrics.Timer; 26 | import com.fasterxml.jackson.databind.MappingIterator; 27 | import com.fasterxml.jackson.databind.ObjectMapper; 28 | import com.fasterxml.jackson.databind.ObjectReader; 29 | import com.fasterxml.jackson.databind.SerializationFeature; 30 | import org.elasticsearch.action.bulk.BulkRequestBuilder; 31 | import org.elasticsearch.action.bulk.BulkResponse; 32 | import org.elasticsearch.client.transport.TransportClient; 33 | import org.elasticsearch.common.StopWatch; 34 | import org.elasticsearch.common.geo.GeoHashUtils; 35 | import org.elasticsearch.common.settings.Settings; 36 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 37 | import org.elasticsearch.common.xcontent.XContentBuilder; 38 | import org.elasticsearch.common.xcontent.XContentFactory; 39 | import org.elasticsearch.metrics.ElasticsearchReporter; 40 | import org.elasticsearch.node.Node; 41 | import org.elasticsearch.node.NodeBuilder; 42 | 43 | import java.io.IOException; 44 | import java.io.InputStream; 45 | import java.net.HttpURLConnection; 46 | import java.net.InetSocketAddress; 47 | import java.net.URL; 48 | import java.nio.file.Files; 49 | import java.nio.file.Path; 50 | import java.text.DateFormat; 51 | import java.text.SimpleDateFormat; 52 | import java.util.Date; 53 | import java.util.List; 54 | import java.util.Locale; 55 | import java.util.Map; 56 | import java.util.concurrent.TimeUnit; 57 | 58 | import static org.elasticsearch.client.Requests.indexRequest; 59 | 60 | /* 61 | curl http://localhost:9200/_percolator/metrics/indexing-request-monitor -X PUT -d '{"query":{"bool":{"must":[{"term":{"name":"usagov-indexing-requests"}},{"range":{"mean_rate":{"from":"0.10","include_lower":false}}}]}}}' 62 | */ 63 | public class LogfileStreamer { 64 | 65 | private static final String INDEX = "logfile"; 66 | private static final String TYPE = "log"; 67 | private static final int MAX_BULK_SIZE = 100; 68 | private static final String ISO_8601_DATE_FORMAT_STRING = "yyyy-MM-dd'T'HH:mm:ssZ"; 69 | private static final DateFormat ISO_8601_DATE_FORMAT = new SimpleDateFormat(ISO_8601_DATE_FORMAT_STRING, Locale.US); 70 | 71 | private final TransportClient client; 72 | private BulkRequestBuilder bulk; 73 | private StopWatch sw; 74 | private long startTimestamp; 75 | 76 | // statistics 77 | private final MetricRegistry registry = new MetricRegistry(); 78 | private final Meter entryMeter = registry.meter("usagov-incoming-requests"); 79 | private final Meter indexingMeter = registry.meter("usagov-indexing-requests"); 80 | private final Counter heartbeatCounter = registry.counter("usa-gov-heartbearts-count"); 81 | private final Timer bulkRequestTimer = registry.timer("bulk-request-timer"); 82 | 83 | private static final String clusterName = System.getProperty("cluster.name", "metrics"); 84 | 85 | public static void main(String[] args) throws Exception { 86 | new LogfileStreamer().run(); 87 | } 88 | 89 | public LogfileStreamer() { 90 | final Settings settings = Settings.builder().put("cluster.name", clusterName).build(); 91 | client = TransportClient.builder() 92 | .settings(settings) 93 | .build() 94 | .addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress("localhost", 9300))); 95 | reset(); 96 | } 97 | 98 | public void run() throws Exception { 99 | startElasticsearchIfNecessary(); 100 | createIndexAndMappingIfNecessary(); 101 | 102 | // index into the metrics index without date formatting 103 | ElasticsearchReporter reporter = ElasticsearchReporter.forRegistry(registry) 104 | .hosts("localhost:9200") 105 | .indexDateFormat("") 106 | .percolationNotifier(new HttpNotifier()) 107 | .percolationFilter(MetricFilter.ALL) 108 | .build(); 109 | reporter.start(60, TimeUnit.SECONDS); 110 | 111 | ObjectMapper objectMapper = new ObjectMapper(); 112 | objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); 113 | ObjectReader reader = objectMapper.reader(Map.class); 114 | MappingIterator> iterator = reader.readValues(getInputStream()); 115 | 116 | try { 117 | while (iterator.hasNextValue()) { 118 | Map entry = iterator.nextValue(); 119 | if (entry.containsKey("_heartbeat_")) { 120 | heartbeatCounter.inc(); 121 | continue; 122 | } 123 | 124 | if (entry.containsKey("ll") && entry.containsKey("t")) { 125 | long timestamp = ((Integer) entry.get("t")).longValue(); 126 | List location = (List) entry.get("ll"); 127 | double latitude = location.get(0).doubleValue(); 128 | double longitude = location.get(1).doubleValue(); 129 | 130 | addToBulkRequest(timestamp, latitude, longitude); 131 | entryMeter.mark(1); 132 | } 133 | } 134 | } finally { 135 | executeBulkRequest(); 136 | } 137 | } 138 | 139 | private void addToBulkRequest(long timestamp, double latitude, double longitude) { 140 | String geohash = GeoHashUtils.encode(latitude, longitude); 141 | String isoDate = ISO_8601_DATE_FORMAT.format(new Date(timestamp * 1000)); 142 | String json = String.format("{\"date\":\"%s\", \"geohash\":\"%s\" }", isoDate, geohash); 143 | bulk.add(indexRequest().index(INDEX).type(TYPE).source(json)); 144 | System.out.print("."); 145 | 146 | executeBulkRequest(); 147 | } 148 | 149 | private void executeBulkRequest() { 150 | if (bulk.numberOfActions() == 0) return; 151 | long secondsSinceLastUpdate = System.currentTimeMillis() / 1000 - startTimestamp; 152 | if (bulk.numberOfActions() < MAX_BULK_SIZE && secondsSinceLastUpdate < 10) return; 153 | 154 | BulkResponse bulkResponse = null; 155 | final Timer.Context context = bulkRequestTimer.time(); 156 | try { 157 | bulkResponse = bulk.execute().actionGet(); 158 | } finally { 159 | context.stop(); 160 | } 161 | logStatistics(bulkResponse.getItems().length); 162 | reset(); 163 | } 164 | 165 | private void logStatistics(long itemsIndexed) { 166 | long totalTimeInSeconds = sw.stop().totalTime().seconds(); 167 | double totalDocumentsPerSecond = (totalTimeInSeconds == 0) ? itemsIndexed : (double) itemsIndexed / totalTimeInSeconds; 168 | System.out.println(String.format("\nIndexed %s documents, %.2f per second in %s seconds", itemsIndexed, totalDocumentsPerSecond, totalTimeInSeconds)); 169 | indexingMeter.mark(1); 170 | } 171 | 172 | private void reset() { 173 | sw = new StopWatch().start(); 174 | startTimestamp = System.currentTimeMillis() / 1000; 175 | bulk = client.prepareBulk(); 176 | } 177 | 178 | private InputStream getInputStream() throws Exception { 179 | URL url = new URL("http://developer.usa.gov/1usagov"); 180 | HttpURLConnection request = (HttpURLConnection) url.openConnection(); 181 | return request.getInputStream(); 182 | } 183 | 184 | private void createIndexAndMappingIfNecessary() { 185 | try { 186 | client.admin().indices().prepareCreate("logfile").execute().actionGet(); 187 | } catch (Exception e) { 188 | e.printStackTrace(); 189 | } 190 | 191 | try { 192 | XContentBuilder mappingContent = XContentFactory.jsonBuilder().startObject().startObject("log") 193 | .startObject("properties") 194 | .startObject("geohash").field("type", "geo_point").field("geohash", true).endObject() 195 | .endObject() 196 | .endObject().endObject(); 197 | 198 | client.admin().indices().preparePutMapping("logfile").setType("log").setSource(mappingContent).execute().actionGet(); 199 | } catch (Exception e) { 200 | e.printStackTrace(); 201 | } 202 | } 203 | 204 | private void startElasticsearchIfNecessary() throws IOException { 205 | if (!"no".equals(System.getProperty("create.es.instance"))) { 206 | final Path tempDirectory = Files.createTempDirectory("usa-gov-logfile-parser"); 207 | System.out.println("Starting elasticsearch instance (data: " + tempDirectory + ")"); 208 | final Settings settings = Settings.settingsBuilder() 209 | .put("cluster.name", clusterName) 210 | .put("path.home", tempDirectory.toString()) 211 | .put("path.data", tempDirectory.toString()) 212 | .put(Node.HTTP_ENABLED, true) 213 | .build(); 214 | NodeBuilder.nodeBuilder().settings(settings).node().start(); 215 | } else { 216 | System.out.println("Not starting elasticsearch instance, please check if available at localhost:9200"); 217 | } 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "[]" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright [yyyy] [name of copyright owner] 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/metrics/MetricsElasticsearchModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.metrics; 20 | 21 | import com.codahale.metrics.Histogram; 22 | import com.codahale.metrics.Meter; 23 | import com.codahale.metrics.Snapshot; 24 | import com.codahale.metrics.Timer; 25 | import com.fasterxml.jackson.core.JsonGenerator; 26 | import com.fasterxml.jackson.core.Version; 27 | import com.fasterxml.jackson.databind.JsonSerializer; 28 | import com.fasterxml.jackson.databind.Module; 29 | import com.fasterxml.jackson.databind.SerializerProvider; 30 | import com.fasterxml.jackson.databind.module.SimpleSerializers; 31 | import com.fasterxml.jackson.databind.ser.std.StdSerializer; 32 | 33 | import java.io.IOException; 34 | import java.util.Arrays; 35 | import java.util.Locale; 36 | import java.util.Map; 37 | import java.util.concurrent.TimeUnit; 38 | 39 | import static org.elasticsearch.metrics.JsonMetrics.JsonCounter; 40 | import static org.elasticsearch.metrics.JsonMetrics.JsonGauge; 41 | import static org.elasticsearch.metrics.JsonMetrics.JsonHistogram; 42 | import static org.elasticsearch.metrics.JsonMetrics.JsonMeter; 43 | import static org.elasticsearch.metrics.JsonMetrics.JsonTimer; 44 | 45 | public class MetricsElasticsearchModule extends Module { 46 | 47 | public static final Version VERSION = new Version(3, 0, 0, "", "metrics-elasticsearch-reporter", "metrics-elasticsearch-reporter"); 48 | 49 | private static void writeAdditionalFields(final Map additionalFields, final JsonGenerator json) throws IOException { 50 | if (additionalFields != null) { 51 | for (final Map.Entry field : additionalFields.entrySet()) { 52 | json.writeObjectField(field.getKey(), field.getValue()); 53 | } 54 | } 55 | } 56 | 57 | private static class GaugeSerializer extends StdSerializer { 58 | private final String timestampFieldname; 59 | private final Map additionalFields; 60 | 61 | private GaugeSerializer(String timestampFieldname, Map additionalFields) { 62 | super(JsonGauge.class); 63 | this.timestampFieldname = timestampFieldname; 64 | this.additionalFields = additionalFields; 65 | } 66 | 67 | @Override 68 | public void serialize(JsonGauge gauge, 69 | JsonGenerator json, 70 | SerializerProvider provider) throws IOException { 71 | json.writeStartObject(); 72 | json.writeStringField("name", gauge.name()); 73 | json.writeObjectField(timestampFieldname, gauge.timestampAsDate()); 74 | final Object value; 75 | try { 76 | value = gauge.value().getValue(); 77 | json.writeObjectField("value", value); 78 | } catch (RuntimeException e) { 79 | json.writeObjectField("error", e.toString()); 80 | } 81 | writeAdditionalFields(additionalFields, json); 82 | json.writeEndObject(); 83 | } 84 | } 85 | 86 | private static class CounterSerializer extends StdSerializer { 87 | private final String timestampFieldname; 88 | private final Map additionalFields; 89 | 90 | private CounterSerializer(String timestampFieldname, Map additionalFields) { 91 | super(JsonCounter.class); 92 | this.timestampFieldname = timestampFieldname; 93 | this.additionalFields = additionalFields; 94 | } 95 | 96 | @Override 97 | public void serialize(JsonCounter counter, 98 | JsonGenerator json, 99 | SerializerProvider provider) throws IOException { 100 | json.writeStartObject(); 101 | json.writeStringField("name", counter.name()); 102 | json.writeObjectField(timestampFieldname, counter.timestampAsDate()); 103 | json.writeNumberField("count", counter.value().getCount()); 104 | writeAdditionalFields(additionalFields, json); 105 | json.writeEndObject(); 106 | } 107 | } 108 | 109 | private static class HistogramSerializer extends StdSerializer { 110 | 111 | private final String timestampFieldname; 112 | private final Map additionalFields; 113 | 114 | private HistogramSerializer(String timestampFieldname, Map additionalFields) { 115 | super(JsonHistogram.class); 116 | this.timestampFieldname = timestampFieldname; 117 | this.additionalFields = additionalFields; 118 | } 119 | 120 | @Override 121 | public void serialize(JsonHistogram jsonHistogram, 122 | JsonGenerator json, 123 | SerializerProvider provider) throws IOException { 124 | json.writeStartObject(); 125 | json.writeStringField("name", jsonHistogram.name()); 126 | json.writeObjectField(timestampFieldname, jsonHistogram.timestampAsDate()); 127 | Histogram histogram = jsonHistogram.value(); 128 | 129 | final Snapshot snapshot = histogram.getSnapshot(); 130 | json.writeNumberField("count", histogram.getCount()); 131 | json.writeNumberField("max", snapshot.getMax()); 132 | json.writeNumberField("mean", snapshot.getMean()); 133 | json.writeNumberField("min", snapshot.getMin()); 134 | json.writeNumberField("p50", snapshot.getMedian()); 135 | json.writeNumberField("p75", snapshot.get75thPercentile()); 136 | json.writeNumberField("p95", snapshot.get95thPercentile()); 137 | json.writeNumberField("p98", snapshot.get98thPercentile()); 138 | json.writeNumberField("p99", snapshot.get99thPercentile()); 139 | json.writeNumberField("p999", snapshot.get999thPercentile()); 140 | 141 | json.writeNumberField("stddev", snapshot.getStdDev()); 142 | writeAdditionalFields(additionalFields, json); 143 | json.writeEndObject(); 144 | } 145 | } 146 | 147 | private static class MeterSerializer extends StdSerializer { 148 | private final String rateUnit; 149 | private final double rateFactor; 150 | private final String timestampFieldname; 151 | private final Map additionalFields; 152 | 153 | public MeterSerializer(TimeUnit rateUnit, String timestampFieldname, Map additionalFields) { 154 | super(JsonMeter.class); 155 | this.timestampFieldname = timestampFieldname; 156 | this.rateFactor = rateUnit.toSeconds(1); 157 | this.rateUnit = calculateRateUnit(rateUnit, "events"); 158 | this.additionalFields = additionalFields; 159 | } 160 | 161 | @Override 162 | public void serialize(JsonMeter jsonMeter, 163 | JsonGenerator json, 164 | SerializerProvider provider) throws IOException { 165 | json.writeStartObject(); 166 | json.writeStringField("name", jsonMeter.name()); 167 | json.writeObjectField(timestampFieldname, jsonMeter.timestampAsDate()); 168 | Meter meter = jsonMeter.value(); 169 | json.writeNumberField("count", meter.getCount()); 170 | json.writeNumberField("m1_rate", meter.getOneMinuteRate() * rateFactor); 171 | json.writeNumberField("m5_rate", meter.getFiveMinuteRate() * rateFactor); 172 | json.writeNumberField("m15_rate", meter.getFifteenMinuteRate() * rateFactor); 173 | json.writeNumberField("mean_rate", meter.getMeanRate() * rateFactor); 174 | json.writeStringField("units", rateUnit); 175 | writeAdditionalFields(additionalFields, json); 176 | json.writeEndObject(); 177 | } 178 | } 179 | 180 | private static class TimerSerializer extends StdSerializer { 181 | private final String rateUnit; 182 | private final double rateFactor; 183 | private final String durationUnit; 184 | private final double durationFactor; 185 | private final String timestampFieldname; 186 | private final Map additionalFields; 187 | 188 | private TimerSerializer(TimeUnit rateUnit, TimeUnit durationUnit, String timestampFieldname, Map additionalFields) { 189 | super(JsonTimer.class); 190 | this.timestampFieldname = timestampFieldname; 191 | this.rateUnit = calculateRateUnit(rateUnit, "calls"); 192 | this.rateFactor = rateUnit.toSeconds(1); 193 | this.durationUnit = durationUnit.toString().toLowerCase(Locale.US); 194 | this.durationFactor = 1.0 / durationUnit.toNanos(1); 195 | this.additionalFields = additionalFields; 196 | } 197 | 198 | @Override 199 | public void serialize(JsonTimer jsonTimer, 200 | JsonGenerator json, 201 | SerializerProvider provider) throws IOException { 202 | json.writeStartObject(); 203 | json.writeStringField("name", jsonTimer.name()); 204 | json.writeObjectField(timestampFieldname, jsonTimer.timestampAsDate()); 205 | Timer timer = jsonTimer.value(); 206 | final Snapshot snapshot = timer.getSnapshot(); 207 | json.writeNumberField("count", timer.getCount()); 208 | json.writeNumberField("max", snapshot.getMax() * durationFactor); 209 | json.writeNumberField("mean", snapshot.getMean() * durationFactor); 210 | json.writeNumberField("min", snapshot.getMin() * durationFactor); 211 | 212 | json.writeNumberField("p50", snapshot.getMedian() * durationFactor); 213 | json.writeNumberField("p75", snapshot.get75thPercentile() * durationFactor); 214 | json.writeNumberField("p95", snapshot.get95thPercentile() * durationFactor); 215 | json.writeNumberField("p98", snapshot.get98thPercentile() * durationFactor); 216 | json.writeNumberField("p99", snapshot.get99thPercentile() * durationFactor); 217 | json.writeNumberField("p999", snapshot.get999thPercentile() * durationFactor); 218 | 219 | /* 220 | if (showSamples) { 221 | final long[] values = snapshot.getValues(); 222 | final double[] scaledValues = new double[values.length]; 223 | for (int i = 0; i < values.length; i++) { 224 | scaledValues[i] = values[i] * durationFactor; 225 | } 226 | json.writeObjectField("values", scaledValues); 227 | } 228 | */ 229 | 230 | json.writeNumberField("stddev", snapshot.getStdDev() * durationFactor); 231 | json.writeNumberField("m1_rate", timer.getOneMinuteRate() * rateFactor); 232 | json.writeNumberField("m5_rate", timer.getFiveMinuteRate() * rateFactor); 233 | json.writeNumberField("m15_rate", timer.getFifteenMinuteRate() * rateFactor); 234 | json.writeNumberField("mean_rate", timer.getMeanRate() * rateFactor); 235 | json.writeStringField("duration_units", durationUnit); 236 | json.writeStringField("rate_units", rateUnit); 237 | writeAdditionalFields(additionalFields, json); 238 | json.writeEndObject(); 239 | } 240 | } 241 | 242 | 243 | /** 244 | * Serializer for the first line of the bulk index operation before the json metric is written 245 | */ 246 | private static class BulkIndexOperationHeaderSerializer extends StdSerializer { 247 | 248 | public BulkIndexOperationHeaderSerializer() { 249 | super(BulkIndexOperationHeader.class); 250 | } 251 | 252 | @Override 253 | public void serialize(BulkIndexOperationHeader bulkIndexOperationHeader, JsonGenerator json, SerializerProvider provider) throws IOException { 254 | json.writeStartObject(); 255 | json.writeObjectFieldStart("index"); 256 | if (bulkIndexOperationHeader.index != null) { 257 | json.writeStringField("_index", bulkIndexOperationHeader.index); 258 | } 259 | if (bulkIndexOperationHeader.type != null) { 260 | json.writeStringField("_type", bulkIndexOperationHeader.type); 261 | } 262 | json.writeEndObject(); 263 | json.writeEndObject(); 264 | } 265 | } 266 | 267 | public static class BulkIndexOperationHeader { 268 | public String index; 269 | public String type; 270 | 271 | public BulkIndexOperationHeader(String index, String type) { 272 | this.index = index; 273 | this.type = type; 274 | } 275 | } 276 | 277 | private final TimeUnit rateUnit; 278 | private final TimeUnit durationUnit; 279 | private final String timestampFieldname; 280 | private final Map additionalFields; 281 | 282 | public MetricsElasticsearchModule(TimeUnit rateUnit, TimeUnit durationUnit, String timestampFieldname, Map additionalFields) { 283 | this.rateUnit = rateUnit; 284 | this.durationUnit = durationUnit; 285 | this.timestampFieldname = timestampFieldname; 286 | this.additionalFields = additionalFields; 287 | } 288 | 289 | @Override 290 | public String getModuleName() { 291 | return "metrics-elasticsearch-serialization"; 292 | } 293 | 294 | @Override 295 | public Version version() { 296 | return VERSION; 297 | } 298 | 299 | @Override 300 | public void setupModule(SetupContext context) { 301 | context.addSerializers(new SimpleSerializers(Arrays.>asList( 302 | new GaugeSerializer(timestampFieldname, additionalFields), 303 | new CounterSerializer(timestampFieldname, additionalFields), 304 | new HistogramSerializer(timestampFieldname, additionalFields), 305 | new MeterSerializer(rateUnit, timestampFieldname, additionalFields), 306 | new TimerSerializer(rateUnit, durationUnit, timestampFieldname, additionalFields), 307 | new BulkIndexOperationHeaderSerializer() 308 | ))); 309 | } 310 | 311 | private static String calculateRateUnit(TimeUnit unit, String name) { 312 | final String s = unit.toString().toLowerCase(Locale.US); 313 | return name + '/' + s.substring(0, s.length() - 1); 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/metrics/ElasticsearchReporterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.metrics; 20 | 21 | import com.codahale.metrics.Counter; 22 | import com.codahale.metrics.Gauge; 23 | import com.codahale.metrics.Histogram; 24 | import com.codahale.metrics.Meter; 25 | import com.codahale.metrics.Metric; 26 | import com.codahale.metrics.MetricFilter; 27 | import com.codahale.metrics.MetricRegistry; 28 | import com.codahale.metrics.Timer; 29 | import org.elasticsearch.ElasticsearchException; 30 | import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; 31 | import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; 32 | import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; 33 | import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; 34 | import org.elasticsearch.action.search.SearchResponse; 35 | import org.elasticsearch.cluster.metadata.IndexMetaData; 36 | import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; 37 | import org.elasticsearch.common.settings.Settings; 38 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 39 | import org.elasticsearch.common.transport.TransportAddress; 40 | import org.elasticsearch.http.HttpServerTransport; 41 | import org.elasticsearch.index.query.QueryBuilder; 42 | import org.elasticsearch.index.query.QueryBuilders; 43 | import org.elasticsearch.metrics.percolation.Notifier; 44 | import org.elasticsearch.node.Node; 45 | import org.elasticsearch.test.ESIntegTestCase; 46 | import org.joda.time.format.ISODateTimeFormat; 47 | import org.junit.Before; 48 | import org.junit.Test; 49 | 50 | import java.io.IOException; 51 | import java.util.Calendar; 52 | import java.util.HashMap; 53 | import java.util.Map; 54 | import java.util.concurrent.TimeUnit; 55 | 56 | import static com.codahale.metrics.MetricRegistry.name; 57 | import static org.elasticsearch.common.settings.Settings.settingsBuilder; 58 | import static org.hamcrest.CoreMatchers.instanceOf; 59 | import static org.hamcrest.CoreMatchers.is; 60 | import static org.hamcrest.CoreMatchers.notNullValue; 61 | import static org.hamcrest.Matchers.hasKey; 62 | import static org.hamcrest.Matchers.hasSize; 63 | 64 | public class ElasticsearchReporterTest extends ESIntegTestCase { 65 | 66 | private ElasticsearchReporter elasticsearchReporter; 67 | private MetricRegistry registry = new MetricRegistry(); 68 | private String index = randomAsciiOfLength(12).toLowerCase(); 69 | private String indexWithDate = String.format("%s-%s-%02d", index, Calendar.getInstance().get(Calendar.YEAR), Calendar.getInstance().get(Calendar.MONTH) + 1); 70 | private String prefix = randomAsciiOfLength(12).toLowerCase(); 71 | 72 | @Override 73 | protected Settings nodeSettings(int nodeOrdinal) { 74 | return settingsBuilder() 75 | .put(super.nodeSettings(nodeOrdinal)) 76 | .put(Node.HTTP_ENABLED, true) 77 | .build(); 78 | } 79 | 80 | @Before 81 | public void setup() throws IOException { 82 | elasticsearchReporter = createElasticsearchReporterBuilder().build(); 83 | } 84 | 85 | @Test 86 | public void testThatTemplateIsAdded() throws Exception { 87 | GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("metrics_template").get(); 88 | 89 | assertThat(response.getIndexTemplates(), hasSize(1)); 90 | IndexTemplateMetaData templateData = response.getIndexTemplates().get(0); 91 | assertThat(templateData.order(), is(0)); 92 | assertThat(templateData.getMappings().get("_default_"), is(notNullValue())); 93 | } 94 | 95 | @Test 96 | public void testThatMappingFromTemplateIsApplied() throws Exception { 97 | registry.counter(name("test", "cache-evictions")).inc(); 98 | reportAndRefresh(); 99 | 100 | // somehow the cluster state is not immediately updated... need to check 101 | Thread.sleep(200); 102 | ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().setRoutingTable(false) 103 | .setLocal(false) 104 | .setNodes(true) 105 | .setIndices(indexWithDate) 106 | .execute().actionGet(); 107 | 108 | assertThat(clusterStateResponse.getState().getMetaData().getIndices().containsKey(indexWithDate), is(true)); 109 | IndexMetaData indexMetaData = clusterStateResponse.getState().getMetaData().getIndices().get(indexWithDate); 110 | assertThat(indexMetaData.getMappings().containsKey("counter"), is(true)); 111 | Map properties = getAsMap(indexMetaData.mapping("counter").sourceAsMap(), "properties"); 112 | Map mapping = getAsMap(properties, "name"); 113 | assertThat(mapping, hasKey("index")); 114 | assertThat(mapping.get("index").toString(), is("not_analyzed")); 115 | } 116 | 117 | @SuppressWarnings("unchecked") 118 | private Map getAsMap(Map map, String key) { 119 | assertThat(map, hasKey(key)); 120 | assertThat(map.get(key), instanceOf(Map.class)); 121 | return (Map) map.get(key); 122 | } 123 | 124 | @Test 125 | public void testThatTemplateIsNotOverWritten() throws Exception { 126 | client().admin().indices().preparePutTemplate("metrics_template").setTemplate("foo*").setSettings("{ \"index.number_of_shards\" : \"1\"}").execute().actionGet(); 127 | 128 | elasticsearchReporter = createElasticsearchReporterBuilder().build(); 129 | 130 | GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("metrics_template").get(); 131 | 132 | assertThat(response.getIndexTemplates(), hasSize(1)); 133 | IndexTemplateMetaData templateData = response.getIndexTemplates().get(0); 134 | assertThat(templateData.template(), is("foo*")); 135 | } 136 | 137 | @Test 138 | public void testThatTimeBasedIndicesCanBeDisabled() throws Exception { 139 | elasticsearchReporter = createElasticsearchReporterBuilder().indexDateFormat("").build(); 140 | indexWithDate = index; 141 | 142 | registry.counter(name("test", "cache-evictions")).inc(); 143 | reportAndRefresh(); 144 | 145 | SearchResponse searchResponse = client().prepareSearch(index).setTypes("counter").execute().actionGet(); 146 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 147 | } 148 | 149 | @Test 150 | public void testCounter() throws Exception { 151 | final Counter evictions = registry.counter(name("test", "cache-evictions")); 152 | evictions.inc(25); 153 | reportAndRefresh(); 154 | 155 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet(); 156 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 157 | 158 | Map hit = searchResponse.getHits().getAt(0).sourceAsMap(); 159 | assertTimestamp(hit); 160 | assertKey(hit, "count", 25); 161 | assertKey(hit, "name", prefix + ".test.cache-evictions"); 162 | assertKey(hit, "host", "localhost"); 163 | } 164 | 165 | @Test 166 | public void testHistogram() { 167 | final Histogram histogram = registry.histogram(name("foo", "bar")); 168 | histogram.update(20); 169 | histogram.update(40); 170 | reportAndRefresh(); 171 | 172 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("histogram").execute().actionGet(); 173 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 174 | 175 | Map hit = searchResponse.getHits().getAt(0).sourceAsMap(); 176 | assertTimestamp(hit); 177 | assertKey(hit, "name", prefix + ".foo.bar"); 178 | assertKey(hit, "count", 2); 179 | assertKey(hit, "max", 40); 180 | assertKey(hit, "min", 20); 181 | assertKey(hit, "mean", 30.0); 182 | assertKey(hit, "host", "localhost"); 183 | } 184 | 185 | @Test 186 | public void testMeter() { 187 | final Meter meter = registry.meter(name("foo", "bar")); 188 | meter.mark(10); 189 | meter.mark(20); 190 | reportAndRefresh(); 191 | 192 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("meter").execute().actionGet(); 193 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 194 | 195 | Map hit = searchResponse.getHits().getAt(0).sourceAsMap(); 196 | assertTimestamp(hit); 197 | assertKey(hit, "name", prefix + ".foo.bar"); 198 | assertKey(hit, "count", 30); 199 | assertKey(hit, "host", "localhost"); 200 | } 201 | 202 | @Test 203 | public void testTimer() throws Exception { 204 | final Timer timer = registry.timer(name("foo", "bar")); 205 | final Timer.Context timerContext = timer.time(); 206 | Thread.sleep(200); 207 | timerContext.stop(); 208 | reportAndRefresh(); 209 | 210 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("timer").execute().actionGet(); 211 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 212 | 213 | Map hit = searchResponse.getHits().getAt(0).sourceAsMap(); 214 | assertTimestamp(hit); 215 | assertKey(hit, "name", prefix + ".foo.bar"); 216 | assertKey(hit, "count", 1); 217 | assertKey(hit, "host", "localhost"); 218 | } 219 | 220 | @Test 221 | public void testGauge() throws Exception { 222 | registry.register(name("foo", "bar"), new Gauge() { 223 | @Override 224 | public Integer getValue() { 225 | return 1234; 226 | } 227 | }); 228 | reportAndRefresh(); 229 | 230 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("gauge").execute().actionGet(); 231 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 232 | 233 | Map hit = searchResponse.getHits().getAt(0).sourceAsMap(); 234 | assertTimestamp(hit); 235 | assertKey(hit, "name", prefix + ".foo.bar"); 236 | assertKey(hit, "value", 1234); 237 | assertKey(hit, "host", "localhost"); 238 | } 239 | 240 | @Test 241 | public void testThatSpecifyingSeveralHostsWork() throws Exception { 242 | elasticsearchReporter = createElasticsearchReporterBuilder().hosts("localhost:10000", "localhost:" + getPortOfRunningNode()).build(); 243 | 244 | registry.counter(name("test", "cache-evictions")).inc(); 245 | reportAndRefresh(); 246 | 247 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet(); 248 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 249 | } 250 | 251 | @Test 252 | public void testGracefulFailureIfNoHostIsReachable() throws IOException { 253 | // if no exception is thrown during the test, we consider it all graceful, as we connected to a dead host 254 | elasticsearchReporter = createElasticsearchReporterBuilder().hosts("localhost:10000").build(); 255 | registry.counter(name("test", "cache-evictions")).inc(); 256 | elasticsearchReporter.report(); 257 | } 258 | 259 | @Test 260 | public void testThatBulkIndexingWorks() { 261 | for (int i = 0; i < 2020; i++) { 262 | final Counter evictions = registry.counter(name("foo", "bar", String.valueOf(i))); 263 | evictions.inc(i); 264 | } 265 | reportAndRefresh(); 266 | 267 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet(); 268 | assertThat(searchResponse.getHits().totalHits(), is(2020L)); 269 | } 270 | 271 | @Test 272 | public void testThatPercolationNotificationWorks() throws IOException, InterruptedException { 273 | SimpleNotifier notifier = new SimpleNotifier(); 274 | 275 | MetricFilter percolationFilter = new MetricFilter() { 276 | @Override 277 | public boolean matches(String name, Metric metric) { 278 | return name.startsWith(prefix + ".foo"); 279 | } 280 | }; 281 | elasticsearchReporter = createElasticsearchReporterBuilder() 282 | .percolationFilter(percolationFilter) 283 | .percolationNotifier(notifier) 284 | .build(); 285 | 286 | final Counter evictions = registry.counter("foo"); 287 | evictions.inc(18); 288 | reportAndRefresh(); 289 | 290 | QueryBuilder queryBuilder = QueryBuilders.boolQuery() 291 | .must(QueryBuilders.matchAllQuery()) 292 | .filter( 293 | QueryBuilders.boolQuery() 294 | .must(QueryBuilders.rangeQuery("count").gte(20)) 295 | .must(QueryBuilders.termQuery("name", prefix + ".foo")) 296 | ); 297 | String json = String.format("{ \"query\" : %s }", queryBuilder.buildAsBytes().toUtf8()); 298 | client().prepareIndex(indexWithDate, ".percolator", "myName").setRefresh(true).setSource(json).execute().actionGet(); 299 | 300 | evictions.inc(1); 301 | reportAndRefresh(); 302 | assertThat(notifier.metrics.size(), is(0)); 303 | 304 | evictions.inc(2); 305 | reportAndRefresh(); 306 | assertThat(notifier.metrics.size(), is(1)); 307 | assertThat(notifier.metrics, hasKey("myName")); 308 | assertThat(notifier.metrics.get("myName").name(), is(prefix + ".foo")); 309 | 310 | notifier.metrics.clear(); 311 | evictions.dec(2); 312 | reportAndRefresh(); 313 | assertThat(notifier.metrics.size(), is(0)); 314 | } 315 | 316 | @Test 317 | public void testThatWronglyConfiguredHostDoesNotLeadToApplicationStop() throws IOException { 318 | createElasticsearchReporterBuilder().hosts("dafuq/1234").build(); 319 | elasticsearchReporter.report(); 320 | } 321 | 322 | @Test 323 | public void testThatTimestampFieldnameCanBeConfigured() throws Exception { 324 | elasticsearchReporter = createElasticsearchReporterBuilder().timestampFieldname("myTimeStampField").build(); 325 | registry.counter(name("myMetrics", "cache-evictions")).inc(); 326 | reportAndRefresh(); 327 | 328 | SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet(); 329 | assertThat(searchResponse.getHits().totalHits(), is(1L)); 330 | 331 | Map hit = searchResponse.getHits().getAt(0).sourceAsMap(); 332 | assertThat(hit, hasKey("myTimeStampField")); 333 | } 334 | 335 | @Test // issue #6 336 | public void testThatEmptyMetricsDoNotResultInBrokenBulkRequest() throws Exception { 337 | long connectionsBeforeReporting = getTotalHttpConnections(); 338 | elasticsearchReporter.report(); 339 | long connectionsAfterReporting = getTotalHttpConnections(); 340 | 341 | assertThat(connectionsAfterReporting, is(connectionsBeforeReporting)); 342 | } 343 | 344 | private long getTotalHttpConnections() { 345 | NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().setHttp(true).get(); 346 | int totalOpenConnections = 0; 347 | for (NodeStats stats : nodeStats.getNodes()) { 348 | totalOpenConnections += stats.getHttp().getTotalOpen(); 349 | } 350 | return totalOpenConnections; 351 | } 352 | 353 | private class SimpleNotifier implements Notifier { 354 | 355 | public Map metrics = new HashMap<>(); 356 | 357 | @Override 358 | public void notify(JsonMetrics.JsonMetric jsonMetric, String match) { 359 | metrics.put(match, jsonMetric); 360 | } 361 | } 362 | 363 | private void reportAndRefresh() { 364 | elasticsearchReporter.report(); 365 | client().admin().indices().prepareRefresh(indexWithDate).execute().actionGet(); 366 | } 367 | 368 | private void assertKey(Map hit, String key, double value) { 369 | assertKey(hit, key, Double.toString(value)); 370 | } 371 | 372 | private void assertKey(Map hit, String key, int value) { 373 | assertKey(hit, key, Integer.toString(value)); 374 | } 375 | 376 | private void assertKey(Map hit, String key, String value) { 377 | assertThat(hit, hasKey(key)); 378 | assertThat(hit.get(key).toString(), is(value)); 379 | } 380 | 381 | private void assertTimestamp(Map hit) { 382 | assertThat(hit, hasKey("@timestamp")); 383 | // no exception means everything is cool 384 | ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(hit.get("@timestamp").toString()); 385 | } 386 | 387 | private int getPortOfRunningNode() { 388 | TransportAddress transportAddress = internalCluster().getInstance(HttpServerTransport.class).boundAddress().publishAddress(); 389 | if (transportAddress instanceof InetSocketTransportAddress) { 390 | return ((InetSocketTransportAddress) transportAddress).address().getPort(); 391 | } 392 | throw new ElasticsearchException("Could not find running tcp port"); 393 | } 394 | 395 | private ElasticsearchReporter.Builder createElasticsearchReporterBuilder() { 396 | Map additionalFields = new HashMap<>(); 397 | additionalFields.put("host", "localhost"); 398 | return ElasticsearchReporter.forRegistry(registry) 399 | .hosts("localhost:" + getPortOfRunningNode()) 400 | .prefixedWith(prefix) 401 | .convertRatesTo(TimeUnit.SECONDS) 402 | .convertDurationsTo(TimeUnit.MILLISECONDS) 403 | .filter(MetricFilter.ALL) 404 | .index(index) 405 | .additionalFields(additionalFields); 406 | } 407 | } 408 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/metrics/ElasticsearchReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. ElasticSearch licenses this 6 | * file to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.metrics; 20 | 21 | import com.codahale.metrics.*; 22 | import com.codahale.metrics.Timer; 23 | import com.fasterxml.jackson.core.JsonFactory; 24 | import com.fasterxml.jackson.core.JsonGenerator; 25 | import com.fasterxml.jackson.core.type.TypeReference; 26 | import com.fasterxml.jackson.databind.ObjectMapper; 27 | import com.fasterxml.jackson.databind.ObjectWriter; 28 | import com.fasterxml.jackson.databind.SerializationFeature; 29 | import com.fasterxml.jackson.module.afterburner.AfterburnerModule; 30 | import org.elasticsearch.metrics.percolation.Notifier; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import java.io.IOException; 35 | import java.io.OutputStream; 36 | import java.net.HttpURLConnection; 37 | import java.net.MalformedURLException; 38 | import java.net.URL; 39 | import java.text.SimpleDateFormat; 40 | import java.util.*; 41 | import java.util.concurrent.TimeUnit; 42 | import java.util.concurrent.atomic.AtomicInteger; 43 | 44 | import static com.codahale.metrics.MetricRegistry.name; 45 | import static org.elasticsearch.metrics.JsonMetrics.*; 46 | import static org.elasticsearch.metrics.MetricsElasticsearchModule.BulkIndexOperationHeader; 47 | 48 | public class ElasticsearchReporter extends ScheduledReporter { 49 | 50 | public static Builder forRegistry(MetricRegistry registry) { 51 | return new Builder(registry); 52 | } 53 | 54 | public static class Builder { 55 | private final MetricRegistry registry; 56 | private Clock clock; 57 | private String prefix; 58 | private TimeUnit rateUnit; 59 | private TimeUnit durationUnit; 60 | private MetricFilter filter; 61 | private String[] hosts = new String[]{ "localhost:9200" }; 62 | private String index = "metrics"; 63 | private String indexDateFormat = "yyyy-MM"; 64 | private int bulkSize = 2500; 65 | private Notifier percolationNotifier; 66 | private MetricFilter percolationFilter; 67 | private int timeout = 1000; 68 | private String timestampFieldname = "@timestamp"; 69 | private Map additionalFields; 70 | 71 | private Builder(MetricRegistry registry) { 72 | this.registry = registry; 73 | this.clock = Clock.defaultClock(); 74 | this.prefix = null; 75 | this.rateUnit = TimeUnit.SECONDS; 76 | this.durationUnit = TimeUnit.MILLISECONDS; 77 | this.filter = MetricFilter.ALL; 78 | } 79 | 80 | /** 81 | * Inject your custom definition of how time passes. Usually the default clock is sufficient 82 | */ 83 | public Builder withClock(Clock clock) { 84 | this.clock = clock; 85 | return this; 86 | } 87 | 88 | /** 89 | * Configure a prefix for each metric name. Optional, but useful to identify single hosts 90 | */ 91 | public Builder prefixedWith(String prefix) { 92 | this.prefix = prefix; 93 | return this; 94 | } 95 | 96 | /** 97 | * Convert all the rates to a certain timeunit, defaults to seconds 98 | */ 99 | public Builder convertRatesTo(TimeUnit rateUnit) { 100 | this.rateUnit = rateUnit; 101 | return this; 102 | } 103 | 104 | /** 105 | * Convert all the durations to a certain timeunit, defaults to milliseconds 106 | */ 107 | public Builder convertDurationsTo(TimeUnit durationUnit) { 108 | this.durationUnit = durationUnit; 109 | return this; 110 | } 111 | 112 | /** 113 | * Allows to configure a special MetricFilter, which defines what metrics are reported 114 | */ 115 | public Builder filter(MetricFilter filter) { 116 | this.filter = filter; 117 | return this; 118 | } 119 | 120 | /** 121 | * Configure an array of hosts to send data to. 122 | * Note: Data is always sent to only one host, but this makes sure, that even if a part of your elasticsearch cluster 123 | * is not running, reporting still happens 124 | * A host must be in the format hostname:port 125 | * The port must be the HTTP port of your elasticsearch instance 126 | */ 127 | public Builder hosts(String ... hosts) { 128 | this.hosts = hosts; 129 | return this; 130 | } 131 | 132 | /** 133 | * The timeout to wait for until a connection attempt is and the next host is tried 134 | */ 135 | public Builder timeout(int timeout) { 136 | this.timeout = timeout; 137 | return this; 138 | } 139 | 140 | /** 141 | * The index name to index in 142 | */ 143 | public Builder index(String index) { 144 | this.index = index; 145 | return this; 146 | } 147 | 148 | /** 149 | * The index date format used for rolling indices 150 | * This is appended to the index name, split by a '-' 151 | */ 152 | public Builder indexDateFormat(String indexDateFormat) { 153 | this.indexDateFormat = indexDateFormat; 154 | return this; 155 | } 156 | 157 | /** 158 | * The bulk size per request, defaults to 2500 (as metrics are quite small) 159 | */ 160 | public Builder bulkSize(int bulkSize) { 161 | this.bulkSize = bulkSize; 162 | return this; 163 | } 164 | 165 | /** 166 | * A metrics filter to define the metrics which should be used for percolation/notification 167 | */ 168 | public Builder percolationFilter(MetricFilter percolationFilter) { 169 | this.percolationFilter = percolationFilter; 170 | return this; 171 | } 172 | 173 | /** 174 | * An instance of the notifier implemention which should be executed in case of a matching percolation 175 | */ 176 | public Builder percolationNotifier(Notifier notifier) { 177 | this.percolationNotifier = notifier; 178 | return this; 179 | } 180 | 181 | /** 182 | * Configure the name of the timestamp field, defaults to '@timestamp' 183 | */ 184 | public Builder timestampFieldname(String fieldName) { 185 | this.timestampFieldname = fieldName; 186 | return this; 187 | } 188 | 189 | /** 190 | * Additional fields to be included for each metric 191 | * @param additionalFields 192 | * @return 193 | */ 194 | public Builder additionalFields(Map additionalFields) { 195 | this.additionalFields = additionalFields; 196 | return this; 197 | } 198 | 199 | public ElasticsearchReporter build() throws IOException { 200 | return new ElasticsearchReporter(registry, 201 | hosts, 202 | timeout, 203 | index, 204 | indexDateFormat, 205 | bulkSize, 206 | clock, 207 | prefix, 208 | rateUnit, 209 | durationUnit, 210 | filter, 211 | percolationFilter, 212 | percolationNotifier, 213 | timestampFieldname, 214 | additionalFields); 215 | } 216 | } 217 | 218 | private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchReporter.class); 219 | 220 | private final String[] hosts; 221 | private final Clock clock; 222 | private final String prefix; 223 | private final String index; 224 | private final int bulkSize; 225 | private final int timeout; 226 | private final ObjectMapper objectMapper = new ObjectMapper(); 227 | private final ObjectWriter writer; 228 | private MetricFilter percolationFilter; 229 | private Notifier notifier; 230 | private String currentIndexName; 231 | private SimpleDateFormat indexDateFormat = null; 232 | private boolean checkedForIndexTemplate = false; 233 | 234 | public ElasticsearchReporter(MetricRegistry registry, String[] hosts, int timeout, 235 | String index, String indexDateFormat, int bulkSize, Clock clock, String prefix, TimeUnit rateUnit, TimeUnit durationUnit, 236 | MetricFilter filter, MetricFilter percolationFilter, Notifier percolationNotifier, String timestampFieldname, Map additionalFields) throws MalformedURLException { 237 | super(registry, "elasticsearch-reporter", filter, rateUnit, durationUnit); 238 | this.hosts = hosts; 239 | this.index = index; 240 | this.bulkSize = bulkSize; 241 | this.clock = clock; 242 | this.prefix = prefix; 243 | this.timeout = timeout; 244 | if (indexDateFormat != null && indexDateFormat.length() > 0) { 245 | this.indexDateFormat = new SimpleDateFormat(indexDateFormat); 246 | } 247 | if (percolationNotifier != null && percolationFilter != null) { 248 | this.percolationFilter = percolationFilter; 249 | this.notifier = percolationNotifier; 250 | } 251 | if (timestampFieldname == null || timestampFieldname.trim().length() == 0) { 252 | LOGGER.error("Timestampfieldname {} is not valid, using default @timestamp", timestampFieldname); 253 | timestampFieldname = "@timestamp"; 254 | } 255 | 256 | objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); 257 | objectMapper.configure(SerializationFeature.CLOSE_CLOSEABLE, false); 258 | // auto closing means, that the objectmapper is closing after the first write call, which does not work for bulk requests 259 | objectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); 260 | objectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); 261 | objectMapper.registerModule(new AfterburnerModule()); 262 | objectMapper.registerModule(new MetricsElasticsearchModule(rateUnit, durationUnit, timestampFieldname, additionalFields)); 263 | writer = objectMapper.writer(); 264 | checkForIndexTemplate(); 265 | } 266 | 267 | @Override 268 | public void report(SortedMap gauges, 269 | SortedMap counters, 270 | SortedMap histograms, 271 | SortedMap meters, 272 | SortedMap timers) { 273 | 274 | // nothing to do if we dont have any metrics to report 275 | if (gauges.isEmpty() && counters.isEmpty() && histograms.isEmpty() && meters.isEmpty() && timers.isEmpty()) { 276 | LOGGER.info("All metrics empty, nothing to report"); 277 | return; 278 | } 279 | 280 | if (!checkedForIndexTemplate) { 281 | checkForIndexTemplate(); 282 | } 283 | final long timestamp = clock.getTime() / 1000; 284 | 285 | currentIndexName = index; 286 | if (indexDateFormat != null) { 287 | currentIndexName += "-" + indexDateFormat.format(new Date(timestamp * 1000)); 288 | } 289 | 290 | try { 291 | HttpURLConnection connection = openConnection("/_bulk", "POST"); 292 | if (connection == null) { 293 | LOGGER.error("Could not connect to any configured elasticsearch instances: {}", Arrays.asList(hosts)); 294 | return; 295 | } 296 | 297 | List percolationMetrics = new ArrayList<>(); 298 | AtomicInteger entriesWritten = new AtomicInteger(0); 299 | 300 | for (Map.Entry entry : gauges.entrySet()) { 301 | if (entry.getValue().getValue() != null) { 302 | JsonMetric jsonMetric = new JsonGauge(name(prefix, entry.getKey()), timestamp, entry.getValue()); 303 | connection = writeJsonMetricAndRecreateConnectionIfNeeded(jsonMetric, connection, entriesWritten); 304 | addJsonMetricToPercolationIfMatching(jsonMetric, percolationMetrics); 305 | } 306 | } 307 | 308 | for (Map.Entry entry : counters.entrySet()) { 309 | JsonCounter jsonMetric = new JsonCounter(name(prefix, entry.getKey()), timestamp, entry.getValue()); 310 | connection = writeJsonMetricAndRecreateConnectionIfNeeded(jsonMetric, connection, entriesWritten); 311 | addJsonMetricToPercolationIfMatching(jsonMetric, percolationMetrics); 312 | } 313 | 314 | for (Map.Entry entry : histograms.entrySet()) { 315 | JsonHistogram jsonMetric = new JsonHistogram(name(prefix, entry.getKey()), timestamp, entry.getValue()); 316 | connection = writeJsonMetricAndRecreateConnectionIfNeeded(jsonMetric, connection, entriesWritten); 317 | addJsonMetricToPercolationIfMatching(jsonMetric, percolationMetrics); 318 | } 319 | 320 | for (Map.Entry entry : meters.entrySet()) { 321 | JsonMeter jsonMetric = new JsonMeter(name(prefix, entry.getKey()), timestamp, entry.getValue()); 322 | connection = writeJsonMetricAndRecreateConnectionIfNeeded(jsonMetric, connection, entriesWritten); 323 | addJsonMetricToPercolationIfMatching(jsonMetric, percolationMetrics); 324 | } 325 | 326 | for (Map.Entry entry : timers.entrySet()) { 327 | JsonTimer jsonMetric = new JsonTimer(name(prefix, entry.getKey()), timestamp, entry.getValue()); 328 | connection = writeJsonMetricAndRecreateConnectionIfNeeded(jsonMetric, connection, entriesWritten); 329 | addJsonMetricToPercolationIfMatching(jsonMetric, percolationMetrics); 330 | } 331 | 332 | closeConnection(connection); 333 | 334 | // execute the notifier impl, in case percolation found matches 335 | if (percolationMetrics.size() > 0 && notifier != null) { 336 | for (JsonMetric jsonMetric : percolationMetrics) { 337 | List matches = getPercolationMatches(jsonMetric); 338 | for (String match : matches) { 339 | notifier.notify(jsonMetric, match); 340 | } 341 | } 342 | } 343 | // catch the exception to make sure we do not interrupt the live application 344 | } catch (IOException e) { 345 | LOGGER.error("Couldnt report to elasticsearch server", e); 346 | } 347 | } 348 | 349 | /** 350 | * Execute a percolation request for the specified metric 351 | */ 352 | private List getPercolationMatches(JsonMetric jsonMetric) throws IOException { 353 | HttpURLConnection connection = openConnection("/" + currentIndexName + "/" + jsonMetric.type() + "/_percolate", "POST"); 354 | if (connection == null) { 355 | LOGGER.error("Could not connect to any configured elasticsearch instances for percolation: {}", Arrays.asList(hosts)); 356 | return Collections.emptyList(); 357 | } 358 | 359 | Map data = new HashMap<>(1); 360 | data.put("doc", jsonMetric); 361 | objectMapper.writeValue(connection.getOutputStream(), data); 362 | closeConnection(connection); 363 | 364 | if (connection.getResponseCode() != 200) { 365 | throw new RuntimeException("Error percolating " + jsonMetric); 366 | } 367 | 368 | Map input = objectMapper.readValue(connection.getInputStream(), new TypeReference>() {}); 369 | List matches = new ArrayList<>(); 370 | if (input.containsKey("matches") && input.get("matches") instanceof List) { 371 | List> foundMatches = (List>) input.get("matches"); 372 | for (Map entry : foundMatches) { 373 | if (entry.containsKey("_id")) { 374 | matches.add(entry.get("_id")); 375 | } 376 | } 377 | } 378 | 379 | return matches; 380 | } 381 | 382 | /** 383 | * Add metric to list of matched percolation if needed 384 | */ 385 | private void addJsonMetricToPercolationIfMatching(JsonMetric jsonMetric, List percolationMetrics) { 386 | if (percolationFilter != null && percolationFilter.matches(jsonMetric.name(), jsonMetric.value())) { 387 | percolationMetrics.add(jsonMetric); 388 | } 389 | } 390 | 391 | private HttpURLConnection writeJsonMetricAndRecreateConnectionIfNeeded(JsonMetric jsonMetric, HttpURLConnection connection, 392 | AtomicInteger entriesWritten) throws IOException { 393 | writeJsonMetric(jsonMetric, writer, connection.getOutputStream()); 394 | return createNewConnectionIfBulkSizeReached(connection, entriesWritten.incrementAndGet()); 395 | } 396 | 397 | private void closeConnection(HttpURLConnection connection) throws IOException { 398 | connection.getOutputStream().close(); 399 | connection.disconnect(); 400 | 401 | // we have to call this, otherwise out HTTP data does not get send, even though close()/disconnect was called 402 | // Ceterum censeo HttpUrlConnection esse delendam 403 | if (connection.getResponseCode() != 200) { 404 | LOGGER.error("Reporting returned code {} {}: {}", connection.getResponseCode(), connection.getResponseMessage()); 405 | } 406 | } 407 | 408 | /** 409 | * Create a new connection when the bulk size has hit the limit 410 | * Checked on every write of a metric 411 | */ 412 | private HttpURLConnection createNewConnectionIfBulkSizeReached(HttpURLConnection connection, int entriesWritten) throws IOException { 413 | if (entriesWritten % bulkSize == 0) { 414 | closeConnection(connection); 415 | return openConnection("/_bulk", "POST"); 416 | } 417 | 418 | return connection; 419 | } 420 | 421 | /** 422 | * serialize a JSON metric over the outputstream in a bulk request 423 | */ 424 | private void writeJsonMetric(JsonMetric jsonMetric, ObjectWriter writer, OutputStream out) throws IOException { 425 | writer.writeValue(out, new BulkIndexOperationHeader(currentIndexName, jsonMetric.type())); 426 | out.write("\n".getBytes()); 427 | writer.writeValue(out, jsonMetric); 428 | out.write("\n".getBytes()); 429 | 430 | out.flush(); 431 | } 432 | 433 | /** 434 | * Open a new HttpUrlConnection, in case it fails it tries for the next host in the configured list 435 | */ 436 | private HttpURLConnection openConnection(String uri, String method) { 437 | for (String host : hosts) { 438 | try { 439 | URL templateUrl = new URL("http://" + host + uri); 440 | HttpURLConnection connection = ( HttpURLConnection ) templateUrl.openConnection(); 441 | connection.setRequestMethod(method); 442 | connection.setConnectTimeout(timeout); 443 | connection.setUseCaches(false); 444 | if (method.equalsIgnoreCase("POST") || method.equalsIgnoreCase("PUT")) { 445 | connection.setDoOutput(true); 446 | } 447 | connection.connect(); 448 | 449 | return connection; 450 | } catch (IOException e) { 451 | LOGGER.error("Error connecting to {}: {}", host, e); 452 | } 453 | } 454 | 455 | return null; 456 | } 457 | 458 | /** 459 | * This index template is automatically applied to all indices which start with the index name 460 | * The index template simply configures the name not to be analyzed 461 | */ 462 | private void checkForIndexTemplate() { 463 | try { 464 | HttpURLConnection connection = openConnection( "/_template/metrics_template", "HEAD"); 465 | if (connection == null) { 466 | LOGGER.error("Could not connect to any configured elasticsearch instances: {}", Arrays.asList(hosts)); 467 | return; 468 | } 469 | connection.disconnect(); 470 | 471 | boolean isTemplateMissing = connection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND; 472 | 473 | // nothing there, lets create it 474 | if (isTemplateMissing) { 475 | LOGGER.debug("No metrics template found in elasticsearch. Adding..."); 476 | HttpURLConnection putTemplateConnection = openConnection( "/_template/metrics_template", "PUT"); 477 | if(putTemplateConnection == null) { 478 | LOGGER.error("Error adding metrics template to elasticsearch"); 479 | return; 480 | } 481 | 482 | JsonGenerator json = new JsonFactory().createGenerator(putTemplateConnection.getOutputStream()); 483 | json.writeStartObject(); 484 | json.writeStringField("template", index + "*"); 485 | json.writeObjectFieldStart("mappings"); 486 | 487 | json.writeObjectFieldStart("_default_"); 488 | json.writeObjectFieldStart("_all"); 489 | json.writeBooleanField("enabled", false); 490 | json.writeEndObject(); 491 | json.writeObjectFieldStart("properties"); 492 | json.writeObjectFieldStart("name"); 493 | json.writeObjectField("type", "string"); 494 | json.writeObjectField("index", "not_analyzed"); 495 | json.writeEndObject(); 496 | json.writeEndObject(); 497 | json.writeEndObject(); 498 | 499 | json.writeEndObject(); 500 | json.writeEndObject(); 501 | json.flush(); 502 | 503 | putTemplateConnection.disconnect(); 504 | if (putTemplateConnection.getResponseCode() != 200) { 505 | LOGGER.error("Error adding metrics template to elasticsearch: {}/{}" + putTemplateConnection.getResponseCode(), putTemplateConnection.getResponseMessage()); 506 | } 507 | } 508 | checkedForIndexTemplate = true; 509 | } catch (IOException e) { 510 | LOGGER.error("Error when checking/adding metrics template to elasticsearch", e); 511 | } 512 | } 513 | } 514 | --------------------------------------------------------------------------------