threadInfoPredicate) {
71 | return Collections2.filter(getAllThreadsInState(lockedMonitors, lockedSynchronizers, state), threadInfoPredicate);
72 | }
73 | }
74 |
75 |
--------------------------------------------------------------------------------
/visualization/graphite_dump.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | from optparse import OptionParser
3 | import requests
4 |
5 | def get_arg_parser():
6 | parser = OptionParser()
7 | parser.add_option('-o', '--host', dest='host', help='Hostname of graphite server', metavar='HOST')
8 | parser.add_option('-r', '--port', dest='port', help='Port for graphite server', metavar='PORT')
9 | parser.add_option('-p', '--prefix', dest='prefix', help='Prefix of metric for which to dump the tree', metavar='PREFIX')
10 | parser.add_option('-s', '--start', dest='start', help='Start date for query', metavar='DATE')
11 | parser.add_option('-e', '--end', dest='end', help='End date for query', metavar='DATE')
12 |
13 | return parser
14 |
15 |
16 | def get_children(host, prefix, min, max):
17 | url = 'http://%s/metrics/expand' % host
18 | leaves = []
19 | for i in range(min, max + 1):
20 | params = {'query': '%s%s' % (prefix, '.*' * i), 'leavesOnly': '1'}
21 | json_url = requests.get(url, params=params)
22 | json_results = json_url.json()
23 | leaves.extend(json_results['results'])
24 |
25 | return leaves
26 |
27 |
28 | def get_bounds(host, prefix):
29 | params = {'query': '%s.*' % prefix, 'leavesOnly': '1'}
30 | url = 'http://%s/metrics/expand' % host
31 | json_url = requests.get(url, params=params)
32 | json_results = json_url.json()
33 | bounds = []
34 | for bound in json_results['results']:
35 | boundresult = bound.replace(prefix + '.', '')
36 | if boundresult.isdigit():
37 | bounds.append(int(boundresult))
38 |
39 | return (min(bounds), max(bounds))
40 |
41 |
42 | def get_max_metric(host, metric, start, end):
43 | params = {'target': 'keepLastValue(%s)' % metric, 'format': 'json', 'from': start, 'until': end}
44 | url = 'http://%s/render' % host
45 | json_url = requests.get(url, params=params)
46 | json_results = json_url.json()
47 | if not json_results:
48 | return None
49 | else:
50 | return max([point[0] for point in json_results[0]['datapoints']])
51 |
52 |
53 | def get_tree(host, prefix, start, end):
54 | nodes_to_process = [prefix]
55 | (min, max) = get_bounds(host, prefix)
56 | leaves = get_children(host, prefix, min, max)
57 |
58 | results = {}
59 | for leaf in leaves:
60 | leafmetric = get_max_metric(host, leaf, start, end)
61 | if leafmetric is not None:
62 | results[leaf] = leafmetric
63 |
64 | return results
65 |
66 |
67 | def format_metric(metric, prefix):
68 | no_prefix = metric.replace(prefix + '.', '')
69 | frames = no_prefix.split('.')
70 | frames.reverse()
71 | return ';'.join(frames).replace('-', '.')
72 |
73 |
74 | def format_output(prefix, results):
75 | for metric, value in results.iteritems():
76 | if value is not None:
77 | print '%s %d' % (format_metric(metric, prefix), value)
78 |
79 |
80 | if __name__ == '__main__':
81 | parser = get_arg_parser()
82 | args, _ = parser.parse_args()
83 | if not(args.host and args.prefix and args.start and args.end):
84 | parser.print_help()
85 | sys.exit(255)
86 |
87 | host = args.host
88 | if args.port is not None:
89 | host = '%s:%s' % (host, args.port)
90 |
91 | results = get_tree(host, args.prefix, args.start, args.end)
92 | format_output(args.prefix, results)
93 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/profilers/CPULoadProfiler.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.profilers;
2 |
3 | import com.google.common.collect.ImmutableMap;
4 |
5 | import com.etsy.statsd.profiler.Arguments;
6 | import com.etsy.statsd.profiler.Profiler;
7 | import com.etsy.statsd.profiler.reporter.Reporter;
8 |
9 | import java.lang.management.ManagementFactory;
10 | import java.util.Map;
11 | import java.util.concurrent.TimeUnit;
12 | import javax.management.Attribute;
13 | import javax.management.AttributeList;
14 | import javax.management.InstanceNotFoundException;
15 | import javax.management.MBeanServer;
16 | import javax.management.MalformedObjectNameException;
17 | import javax.management.ObjectName;
18 | import javax.management.ReflectionException;
19 |
20 | /**
21 | * This profiler retrieves CPU values for the JVM and System from the "OperatingSystem" JMX Bean.
22 | *
23 | * This profiler relies on a JMX bean that might not be available in all JVM implementations.
24 | * We know for sure it's available in Sun/Oracle's JRE 7+, but there are no guarantees it
25 | * will remain there for the foreseeable future.
26 | *
27 | * @see StackOverflow post
28 | *
29 | * @author Alejandro Rivera
30 | */
31 | public class CPULoadProfiler extends Profiler {
32 |
33 | public static final long PERIOD = 10;
34 | private static final Map ATTRIBUTES_MAP = ImmutableMap.of("ProcessCpuLoad", "cpu.jvm",
35 | "SystemCpuLoad", "cpu.system");
36 |
37 | private AttributeList list;
38 |
39 | public CPULoadProfiler(Reporter reporter, Arguments arguments) {
40 | super(reporter, arguments);
41 | try {
42 | MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
43 | ObjectName os = ObjectName.getInstance("java.lang:type=OperatingSystem");
44 | list = mbs.getAttributes(os, ATTRIBUTES_MAP.keySet().toArray(new String[ATTRIBUTES_MAP.size()]));
45 | } catch (InstanceNotFoundException | ReflectionException | MalformedObjectNameException e) {
46 | list = null;
47 | }
48 |
49 | }
50 |
51 | /**
52 | * Profile memory usage and GC statistics
53 | */
54 | @Override
55 | public void profile() {
56 | recordStats();
57 | }
58 |
59 | @Override
60 | public void flushData() {
61 | recordStats();
62 | }
63 |
64 | @Override
65 | public long getPeriod() {
66 | return PERIOD;
67 | }
68 |
69 | @Override
70 | public TimeUnit getTimeUnit() {
71 | return TimeUnit.SECONDS;
72 | }
73 |
74 | @Override
75 | protected void handleArguments(Arguments arguments) { /* No arguments needed */ }
76 |
77 | /**
78 | * Records all memory statistics
79 | */
80 | private void recordStats() {
81 | if (list == null) {
82 | return;
83 | }
84 |
85 | Attribute att;
86 | Double value;
87 | String metric;
88 | for (Object o : list) {
89 | att = (Attribute) o;
90 | value = (Double) att.getValue();
91 |
92 | if (value == null || value == -1.0) {
93 | continue;
94 | }
95 |
96 | metric = ATTRIBUTES_MAP.get(att.getName());
97 | if (metric == null) {
98 | continue;
99 | }
100 |
101 | value = ((int) (value * 1000)) / 10.0d; // 0-100 with 1-decimal precision
102 | recordGaugeValue(metric, value);
103 | }
104 | }
105 | }
106 |
--------------------------------------------------------------------------------
/visualization/influxdb_dump.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | from optparse import OptionParser
3 | from influxdb import InfluxDBClient
4 | import sys
5 |
6 | class InfluxDBDump:
7 | def __init__(self, host, port, username, password, database, prefix, tag_mapping):
8 | self.host = host
9 | self.port = port
10 | self.username = username
11 | self.password = password
12 | self.database = database
13 | self.prefix = prefix
14 | self.tag_mapping = tag_mapping
15 | self.client = InfluxDBClient(self.host, self.port, self.username, self.password, self.database)
16 | self.mapped_tags = self._construct_tag_mapping(prefix, tag_mapping)
17 |
18 | def run(self):
19 | clauses = ["%s ='%s'" % (tag, value) for (tag, value) in self.mapped_tags.iteritems()]
20 | query = 'select value from /^cpu.trace.*/ where %s' % " and ".join(clauses)
21 | metrics = self.client.query(query).raw['series']
22 | for metric in metrics:
23 | name = self._format_metric_name(metric['name'], 'cpu.trace.')
24 | value = sum([v[1] for v in metric['values']])
25 | if name != str(value):
26 | print '%s %d' % (name, value)
27 |
28 | def _format_metric_name(self, name, prefix):
29 | tokens = name.replace(prefix, '').split('.')
30 | reverse = reversed(tokens)
31 | line_numbers = [':'.join(r.rsplit('-', 1)) for r in reverse]
32 | return ';'.join(line_numbers).replace('-', '.')
33 |
34 | def _construct_tag_mapping(self, prefix, tag_mapping):
35 | mapped_tags = {}
36 | if tag_mapping:
37 | tag_names = tag_mapping.split('.')
38 | prefix_components = prefix.split('.')
39 | if len(tag_names) != len(prefix_components):
40 | raise Exception('Invalid tag mapping %s' % tag_mapping)
41 |
42 | zipped = zip(tag_names, prefix_components)
43 | for entry in zipped:
44 | if entry[0] != 'SKIP':
45 | mapped_tags[entry[0]] = entry[1]
46 | else:
47 | mapped_tags['prefix'] = prefix
48 |
49 | return mapped_tags
50 |
51 |
52 | def get_arg_parser():
53 | parser = OptionParser()
54 | parser.add_option('-o', '--host', dest='host', help='Hostname of InfluxDB server', metavar='HOST')
55 | parser.add_option('-r', '--port', dest='port', help='Port for InfluxDB HTTP API (defaults to 8086)', metavar='PORT')
56 | parser.add_option('-u', '--username', dest='username', help='Username with which to connect to InfluxDB', metavar='USER')
57 | parser.add_option('-p', '--password', dest='password', help='Password with which to connect to InfluxDB', metavar='PASSWORD')
58 | parser.add_option('-d', '--database', dest='database', help='InfluxDB database which contains profiler data', metavar='DB')
59 | parser.add_option('-e', '--prefix', dest='prefix', help='Metric prefix', metavar='PREFIX')
60 | parser.add_option('-t', '--tag-mapping', dest='mapping', help='Tag mapping for metric prefix', metavar='MAPPING')
61 |
62 | return parser
63 |
64 | if __name__ == '__main__':
65 | parser = get_arg_parser()
66 | args, _ = parser.parse_args()
67 | if not(args.host and args.username and args.password and args.database and args.prefix):
68 | parser.print_help()
69 | sys.exit(255)
70 | port = args.port or 8086
71 | tag_mapping = args.mapping or None
72 | dumper = InfluxDBDump(args.host, port, args.username, args.password, args.database, args.prefix, tag_mapping)
73 | dumper.run()
74 |
75 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/Profiler.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler;
2 |
3 | import com.etsy.statsd.profiler.reporter.Reporter;
4 | import com.google.common.base.Preconditions;
5 |
6 | import java.util.Map;
7 | import java.util.concurrent.TimeUnit;
8 |
9 | /**
10 | * Interface for profilers
11 | *
12 | * @author Andrew Johnson
13 | */
14 | public abstract class Profiler {
15 | public static final Class[] CONSTRUCTOR_PARAM_TYPES = new Class[]{Reporter.class, Arguments.class};
16 |
17 | private final Reporter reporter;
18 | // CONTAINER_ID=container_1486158130664_0002_01_000157
19 | private String[] tags = {};
20 |
21 | private long recordedStats = 0;
22 | public Profiler(Reporter reporter, Arguments arguments) {
23 | Preconditions.checkNotNull(reporter);
24 | this.reporter = reporter;
25 | handleArguments(arguments);
26 |
27 | String containerId = System.getenv("CONTAINER_ID");
28 | if (containerId != null) {
29 | String applicationId = getApplicationId(containerId);
30 | tags = new String[]{"container_id:" + containerId, "application_id:" + applicationId};
31 | }
32 | }
33 |
34 | public static String getApplicationId(String containerId) {
35 | String[] parts = containerId.replace("container_", "").split("_");
36 | return "application_" + parts[0] + "_" + parts[1];
37 | }
38 |
39 | /**
40 | * Perform profiling
41 | */
42 | public abstract void profile();
43 |
44 | /**
45 | * Hook to flush any remaining data cached by the profiler at JVM shutdown
46 | */
47 | public abstract void flushData();
48 |
49 | /**
50 | * Get the period to use for this profiler in the ScheduledExecutorService
51 | *
52 | * @return The ScheduledExecutorThread period for this profiler
53 | */
54 | public abstract long getPeriod();
55 |
56 | /**
57 | * Get the unit of time that corresponds to the period for this profiler
58 | *
59 | * @return A TimeUnit corresponding the the period for this profiler
60 | */
61 | public abstract TimeUnit getTimeUnit();
62 |
63 | /**
64 | * CPUTracingProfiler can emit some metrics that indicate the upper and lower bound on the length of stack traces
65 | * This is helpful for querying this data for some backends (such as Graphite) that do not have rich query languages
66 | * Reporters can override this to disable these metrics
67 | *
68 | * @return true if the bounds metrics should be emitted, false otherwise
69 | */
70 | protected boolean emitBounds() {
71 | return reporter.emitBounds();
72 | }
73 |
74 | /**
75 | * Handle any additional arguments necessary for this profiler
76 | *
77 | * @param arguments The arguments given to the profiler
78 | */
79 | protected abstract void handleArguments(Arguments arguments);
80 |
81 | /**
82 | * Record a gauge value
83 | *
84 | * @param key The key for the gauge
85 | * @param value The value of the gauge
86 | */
87 | protected void recordGaugeValue(String key, long value) {
88 | recordedStats++;
89 | reporter.recordGaugeValue(key, value);
90 | }
91 |
92 | /**
93 | * @see #recordGaugeValue(String, long)
94 | */
95 | protected void recordGaugeValue(String key, double value) {
96 | recordedStats++;
97 | reporter.recordGaugeValue(key, value);
98 | }
99 |
100 | /**
101 | * Record multiple gauge values
102 | * This is useful for reporters that can send points in batch
103 | *
104 | * @param gauges A map of gauge names to values
105 | */
106 | protected void recordGaugeValues(Map gauges) {
107 | recordedStats++;
108 | reporter.recordGaugeValues(gauges, tags);
109 | }
110 |
111 | public long getRecordedStats() { return recordedStats; }
112 | }
113 |
--------------------------------------------------------------------------------
/example/StatsDProfilerFlowListener.scala:
--------------------------------------------------------------------------------
1 | package com.etsy.cascading.flow
2 |
3 | import java.util.Properties
4 |
5 | import cascading.flow.{Flow, FlowListener, FlowStep}
6 | import org.apache.hadoop.mapred.JobConf
7 |
8 | import scala.collection.JavaConversions._
9 |
10 | /**
11 | * Flow listener for setting up JobConf to enable statsd-jvm-profiler
12 | */
13 | class StatsDProfilerFlowListener extends FlowListener {
14 | val baseParamsFormat = "-javaagent:%s=server=%s,port=%s,prefix=bigdata.profiler.%s.%s.%s.%%s.%%s,packageWhitelist=%s,packageBlacklist=%s,username=%s,password=%s,database=%s,reporter=%s,tagMapping=%s"
15 |
16 | override def onStarting(flow: Flow[_]): Unit = {
17 | val profilerProps = loadProperties("statsd-jvm-profiler.properties")
18 |
19 | val jarPath = profilerProps.getProperty("jar.location")
20 | val host = profilerProps.getProperty("host")
21 | val port = profilerProps.getProperty("port")
22 | val userName = System.getProperty("user.name")
23 | val flowId = flow.getID
24 | val jobName = flow.getName.replace(".", "-")
25 | val reporter = profilerProps.getProperty("reporter")
26 | val packageBlacklist = profilerProps.getProperty("package.blacklist")
27 | val packageWhiteList = profilerProps.getProperty("package.whitelist")
28 | val influxdbUser = profilerProps.getProperty("influxdb.user")
29 | val influxdbPassword = profilerProps.getProperty("influxdb.password")
30 | val influxdbDatabase = profilerProps.getProperty("influxdb.database")
31 | val dashboardUrl = profilerProps.getProperty("dashboard.url")
32 | val tagMapping = profilerProps.getProperty("tagMapping")
33 |
34 | val baseParams = baseParamsFormat.format(jarPath, host, port, userName, jobName, flowId, packageWhiteList, packageBlacklist, influxdbUser, influxdbPassword, influxdbDatabase, reporter, tagMapping)
35 |
36 | flow.getFlowSteps.toList foreach { fs: FlowStep[_] =>
37 | val stepNum = fs.getStepNum.toString
38 | val conf = fs.getConfig.asInstanceOf[JobConf]
39 | val numReduceTasks = conf.get("mapreduce.job.reduces")
40 |
41 | conf.setBoolean("mapreduce.task.profile", true)
42 | conf.set("mapreduce.task.profile.map.params", baseParams.format(stepNum, "map"))
43 | conf.set("mapreduce.task.profile.reduce.params", baseParams.format(stepNum, "reduce"))
44 | // In newer versions of Cascading/Scalding it seems to no longer be possible to retrieve the correct
45 | // number of map or reduce tasks from the flow.
46 | // As such we have to profile a predetermined task every time, rather than picking a random one
47 | conf.set("mapreduce.task.profile.maps", getTaskToProfile(stepNum, "map", conf))
48 | conf.set("mapreduce.task.profile.reduces", getTaskToProfile(stepNum, "reduce", conf))
49 |
50 | // If you use https://github.com/etsy/Sahale this will cause links to the profiler dashboard to appear in Sahale
51 | // for jobs that are profiled
52 | val additionalLinksOrig = conf.get("sahale.additional.links", "")
53 | val additionalLinks = numReduceTasks.toInt match {
54 | case i: Int if i <= 0 => "%s;Profiler Dashboard - Map|%s".format(additionalLinksOrig, dashboardUrl.format("map"))
55 | case _ => "%s;Profiler Dashboard - Map|%s;Profiler Dashboard - Reduce|%s".format(additionalLinksOrig, dashboardUrl.format("map"), dashboardUrl.format("reduce"))
56 | }
57 | conf.set("sahale.additional.links", additionalLinks)
58 | }
59 | }
60 |
61 | private def getTaskToProfile(stage: String, phase: String, conf: JobConf): String = {
62 | val prop = "profiler.stage%s.%s".format(stage, phase)
63 | conf.get(prop, "0")
64 | }
65 |
66 | private def loadProperties(resourceName: String): Properties = {
67 | val props = new Properties()
68 | props.load(Thread.currentThread.getContextClassLoader.getResourceAsStream(resourceName))
69 |
70 | props
71 | }
72 |
73 | override def onThrowable(flow: Flow[_], t: Throwable): Boolean = false
74 |
75 | override def onStopping(flow: Flow[_]): Unit = ()
76 |
77 | override def onCompleted(flow: Flow[_]): Unit = ()
78 | }
79 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/util/StackTraceFilter.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.util;
2 |
3 | import com.google.common.base.Function;
4 | import com.google.common.base.Joiner;
5 | import com.google.common.collect.Lists;
6 |
7 | import java.util.List;
8 | import java.util.regex.Matcher;
9 | import java.util.regex.Pattern;
10 |
11 | /**
12 | * Utility class for filtering stack traces
13 | * Assumes a string representation like that produced by @link{StackTraceFormatter}
14 | *
15 | * @author Andrew Johnson
16 | */
17 | public class StackTraceFilter {
18 | public static final Pattern MATCH_EVERYTHING = Pattern.compile("^.*$");
19 | public static final Pattern MATCH_NOTHING = Pattern.compile("$^");
20 |
21 | private final Pattern includePattern;
22 | private final Pattern excludePattern;
23 | // This allows us to shortcut doing regex matching for performance
24 | private final boolean arePatternsDefault;
25 |
26 | public StackTraceFilter(List includePackages, List excludePackages) {
27 | includePattern = getPackagePattern(includePackages, MATCH_EVERYTHING);
28 | excludePattern = getPackagePattern(excludePackages, MATCH_NOTHING);
29 | arePatternsDefault = includePattern == MATCH_EVERYTHING && excludePattern == MATCH_NOTHING;
30 | }
31 |
32 | /**
33 | * Indicate if this stack trace should be included in the filter
34 | * Checks if the stack trace matches the include pattern and does not match the exclude pattern
35 | *
36 | * @param formattedStackTrace The stack trace to check for inclusion in the filter
37 | * @return True if it should be included, false otherwise
38 | */
39 | public boolean includeStackTrace(String formattedStackTrace) {
40 | return arePatternsDefault || includeMatches(formattedStackTrace) && !excludeMatches(formattedStackTrace);
41 | }
42 |
43 | /**
44 | * Indicate if this stack trace matches one of the included packages
45 | *
46 | * @param formattedStackTrace The stack trace to check against the included packages
47 | * @return True if it matches an included package, false otherwise
48 | */
49 | public boolean includeMatches(String formattedStackTrace) {
50 | return matches(includePattern, formattedStackTrace);
51 | }
52 |
53 | /**
54 | * Indicate if this stack trace matches one of the excluded packages
55 | *
56 | * @param formattedStackTrace The stack trace to check against the excluded packages
57 | * @return True if it matches an excluded package, false otherwise
58 | */
59 | public boolean excludeMatches(String formattedStackTrace) {
60 | return matches(excludePattern, formattedStackTrace);
61 | }
62 |
63 | /**
64 | * Indicate if this stack trace matches the given pattern
65 | *
66 | * @param pattern The pattern to match against the stack trace
67 | * @param formattedStackTrace The stack trace to check against the pattern
68 | * @return True if the stack trace matches the pattern, false otherwise
69 | */
70 | private static boolean matches(Pattern pattern, String formattedStackTrace) {
71 | Matcher matcher = pattern.matcher(formattedStackTrace);
72 |
73 | return matcher.matches();
74 | }
75 |
76 | /**
77 | * Construct a Pattern that matches any of the given packages
78 | *
79 | * @param packageWhitelist The packages to match in this Pattern
80 | * @return A Pattern object that matches any of the given packages
81 | */
82 | public Pattern getPackagePattern(List packageWhitelist, Pattern defaultPattern) {
83 | if (packageWhitelist == null || packageWhitelist.isEmpty()) {
84 | return defaultPattern;
85 | }
86 | else {
87 | return Pattern.compile(String.format("(.*\\.|^)(%s).*",
88 | Joiner.on("|").join(Lists.transform(packageWhitelist, new Function() {
89 | @Override
90 | public String apply(String s) {
91 | return String.format("(%s)", s.replace(".", "-"));
92 | }
93 | }))));
94 | }
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/visualization/influxdb-dashboard/public/scripts/influxdb.js:
--------------------------------------------------------------------------------
1 | var influx = require('influx')
2 | var config = require('./config.js')
3 |
4 | var client = null;
5 | config.getConfig(function(conf) {
6 | client = influx(conf['influxdb']);
7 | });
8 |
9 | // This measurement will always exist and can be used to query for valid tags
10 | var canaryMeasurement = "heap.total.max";
11 |
12 | exports.getData = function(user, job, flow, stage, phase, jvmName, metric, callback) {
13 | var optionalJvmName = jvmName ? "' and jvmName = '" + jvmName + "'": "'";
14 | var query = "select value from /^" + metric + ".*/ where username = '" + user + "' and job = '" + job + "' and flow = '" + flow + "' and stage = '" + stage + "' and phase = '" + phase + optionalJvmName;
15 | client.queryRaw(query, function(err, res) {
16 | var series = res[0].series;
17 | var results = {};
18 | if (series) {
19 | results = series.map(function(series) {
20 | var name = series.name.replace(metric + '.', '');
21 | var points = series.values.map(function(value) {
22 | return {time: new Date(value[0]).getTime(), value: value[1]};
23 | });
24 |
25 | return {metric: name, values: points};
26 | });
27 | }
28 | callback(results);
29 | });
30 | }
31 |
32 |
33 | exports.getFlameGraphData = function(user, job, flow, stage, phase, jvmName, prefix, callback) {
34 | var optionalJvmName = jvmName ? "' and jvmName = '" + jvmName + "'": "'";
35 | var query = "select value from /^" + prefix + ".*/ where username = '" + user + "' and job = '" + job + "' and flow = '" + flow + "' and stage = '" + stage + "' and phase = '" + phase + optionalJvmName;
36 | client.queryRaw(query, function(err, res) {
37 | var series = res[0].series;
38 | var results = {};
39 | if (series) {
40 | results = series.map(function(series) {
41 | var name = formatMetric(series.name, prefix);
42 | var value = (null, series.values.map(function(value) {
43 | return value[1];
44 | })).reduce(function(total, curr) {
45 | return total + curr;
46 | }, 0);
47 |
48 | return {metric: name, value: value};
49 | });
50 | }
51 | callback(results);
52 | });
53 | }
54 |
55 | exports.getOptions = function(prefix, callback) {
56 | client.getSeries("heap.total.max", function(err, seriesNames) {
57 | var result = {};
58 | if (seriesNames !== undefined) {
59 | var series = seriesNames[0]
60 | var columns = series.values.map(function(value) {
61 | var tokens = value[0].split(',');
62 | column = {}
63 | tokens.forEach(function(token) {
64 | if (token.indexOf("=") != -1) {
65 | var split = token.split("=");
66 | column[split[0]] = split[1];
67 | }
68 | })
69 | return column;
70 | });
71 |
72 | columns.map(function(values) {
73 | var user = values['username'];
74 | var job = values['job'];
75 | var flow = values['flow'];
76 | var stage = values['stage'];
77 | var phase = values['phase'];
78 | var jvmName = values['jvmName'];
79 |
80 | var userVal = result[user]
81 | if (!userVal) {
82 | userVal = {};
83 | }
84 |
85 | var jobVal = userVal[job];
86 | if (!jobVal) {
87 | jobVal = {};
88 | }
89 |
90 | var flowVal = jobVal[flow];
91 | if(!flowVal) {
92 | flowVal = {};
93 | }
94 |
95 | var stageVal = flowVal[stage];
96 | if(!stageVal) {
97 | stageVal = {};
98 | }
99 |
100 |
101 | var phaseVal = stageVal[phase];
102 | if(!phaseVal) {
103 | phaseVal = [];
104 | }
105 |
106 | if (phaseVal.indexOf(jvmName) == -1) {
107 | phaseVal.push(jvmName);
108 | }
109 |
110 | stageVal[phase] = phaseVal;
111 | flowVal[stage] = stageVal;
112 | jobVal[flow] = flowVal;
113 | userVal[job] = jobVal;
114 | result[user] = userVal;
115 | });
116 | callback(result);
117 | }
118 | });
119 | }
120 |
121 | function formatMetric(metric, prefix) {
122 | var noPrefix = metric.replace(prefix, '');
123 | var frames = noPrefix.split('.');
124 | frames.splice(0, 1);
125 | frames.reverse();
126 | var lineNumFrames = frames.map(function(frame) {
127 | var lineNumSepIndex = frame.lastIndexOf('-');
128 | return frame.substring(0, lineNumSepIndex) + ':' + frame.substring(lineNumSepIndex+1);
129 | });
130 |
131 | return lineNumFrames.join(';').replace(/-/g, '.');
132 | }
133 |
--------------------------------------------------------------------------------
/visualization/influxdb-dashboard/public/scripts/render-client.js:
--------------------------------------------------------------------------------
1 | $(document).ready(function() {
2 | $("#toc").toc({
3 | 'highlightOffset': 1,
4 | });
5 | var params = URI(window.location.href).search(true);
6 |
7 | var user = params.user;
8 | var job = params.job;
9 | var flow = params.flow;
10 | var stage = params.stage;
11 | var phase = params.phase;
12 | var jvmName = params.jvm;
13 | var base = params.prefix || $('#prefix').text() || 'bigdata.profiler';
14 | var refresh = params.refresh || $('#refresh').text() || 60;
15 | var optionalJvmName = jvmName && jvmName !== '' ? '/' + jvmName + '/' : '/';
16 |
17 | var prefix = base + '.' + user + '.' + job + '.' + flow + '.' + stage + '.' + phase;
18 | var cpuPrefix = 'cpu.trace';
19 | var heapPrefix = 'heap';
20 | var nonHeapPrefix = 'nonheap';
21 | var finalizePrefix = 'pending-finalization-count';
22 | var gcPrefix = 'gc';
23 | var classLoadingPrefix = '.*class-count';
24 |
25 | var memoryMetrics = [{metric:'init', alias:'Initial'},{metric:'committed', alias:'Committed'},{metric:'max', alias:'Maximum'},{metric:'used', alias:'Used'}];
26 | var finalizeMetrics = [{metric: finalizePrefix, alias: 'Objects Pending Finalization'}];
27 | var gcCountMetrics = [{metric:'PS_MarkSweep.count', alias:'PS_MarkSweep'},{metric:'PS_Scavenge.count', alias:'PS_Scavenge'}];
28 | var gcTimeMetrics = [{metric:'PS_MarkSweep.time', alias:'PS_MarkSweep'},{metric:'PS_Scavenge.time', alias:'PS_Scavenge'}];
29 | var gcRuntimeMetrics = [{metric:'PS_MarkSweep.runtime', alias:'PS_MarkSweep'},{metric:'PS_Scavenge.runtime', alias:'PS_Scavenge'}];
30 | var classLoadingMetrics = [{metric:'loaded-class-count', alias:'Classes Loaded'},{metric:'total-loaded-class-count', alias:'Total Classes Loaded'},{metric:'unloaded-class-count', alias:'Classes Unloaded'}];
31 | var heapPools = [{pool:'ps-eden-space', selector:'#eden', title:'Eden'}, {pool:'ps-old-gen', selector:'#oldgen', title:'Old Generation'}, {pool:'ps-survivor-space', selector:'#survivor', title:'Survivor Space'}];
32 | var nonHeapPools = [{pool:'code-cache', selector:'#codecache', title:'Code Cache'}, {pool:'ps-perm-gen', selector:'#permgen', title:'Permgen'}];
33 |
34 | $("#toc ul").append('Flame Graph');
35 | $('#toc').affix({
36 | offset: {
37 | top: $('.navbar navbar-default').height()
38 | }
39 | });
40 |
41 | var heapGet = $.get('/data/' + user + '/' + job + '/' + flow + '/' + stage + '/' + phase + optionalJvmName + heapPrefix);
42 | var nonHeapGet = $.get('/data/' + user + '/' + job + '/' + flow + '/' + stage + '/' + phase + optionalJvmName + nonHeapPrefix);
43 | var finalizeGet = $.get('/data/' + user + '/' + job + '/' + flow + '/' + stage + '/' + phase + optionalJvmName + finalizePrefix);
44 | var gcGet = $.get('/data/' + user + '/' + job + '/' + flow + '/' + stage + '/' + phase + optionalJvmName + gcPrefix);
45 | var classLoadingGet = $.get('/data/' + user + '/' + job + '/' + flow + '/' + stage + '/' + phase + optionalJvmName + classLoadingPrefix);
46 |
47 | $.when(heapGet).done(function() {
48 | var heapResults = heapGet['responseJSON'];
49 | ViewUtil.renderGraph(heapResults, 'Heap Usage', '#heap', ViewUtil.getMetricsForPool('total', memoryMetrics));
50 | heapPools.forEach(function(pool) {
51 | ViewUtil.renderGraph(heapResults, pool.title, pool.selector, ViewUtil.getMetricsForPool(pool.pool, memoryMetrics));
52 | });
53 | });
54 |
55 | $.when(nonHeapGet).done(function() {
56 | var nonHeapResults = nonHeapGet['responseJSON'];
57 | ViewUtil.renderGraph(nonHeapResults, 'Non-Heap Usage', '#nonheap', ViewUtil.getMetricsForPool('total', memoryMetrics));
58 |
59 | nonHeapPools.forEach(function(pool) {
60 | ViewUtil.renderGraph(nonHeapResults, pool.title, pool.selector, ViewUtil.getMetricsForPool(pool.pool, memoryMetrics));
61 | });
62 | });
63 |
64 | $.when(finalizeGet).done(function() {
65 | var finalizeResults = finalizeGet['responseJSON'];
66 | ViewUtil.renderGraph(finalizeResults, 'Objects Pending Finalization', '#finalize', finalizeMetrics);
67 | });
68 |
69 | $.when(gcGet).done(function() {
70 | var gcResults = gcGet['responseJSON'];
71 | ViewUtil.renderGraph(gcResults, 'Garbage Collection', '#count', gcCountMetrics);
72 | ViewUtil.renderGraph(gcResults, 'Garbage Collection', '#time', gcTimeMetrics);
73 | ViewUtil.renderGraph(gcResults, 'Garbage Collection', '#runtime', gcRuntimeMetrics);
74 | });
75 |
76 | $.when(classLoadingGet).done(function() {
77 | var classLoadingResults = classLoadingGet['responseJSON'];
78 | ViewUtil.renderGraph(classLoadingResults, 'Class Loading', '#classloading', classLoadingMetrics);
79 | });
80 |
81 | if (refresh > 0) {
82 | setTimeout(function() {
83 | location.reload();
84 | }, refresh * 1000);
85 | }
86 | });
87 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/reporter/InfluxDBReporter.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.reporter;
2 |
3 | import com.etsy.statsd.profiler.Arguments;
4 | import com.etsy.statsd.profiler.util.TagUtil;
5 | import com.google.common.base.Preconditions;
6 | import com.google.common.collect.ImmutableMap;
7 | import org.influxdb.InfluxDB;
8 | import org.influxdb.InfluxDBFactory;
9 | import org.influxdb.dto.BatchPoints;
10 | import org.influxdb.dto.Point;
11 |
12 | import java.util.Map;
13 | import java.util.concurrent.TimeUnit;
14 |
15 | /**
16 | * Reporter that sends data to InfluxDB
17 | *
18 | * @author Andrew Johnson
19 | */
20 | public class InfluxDBReporter extends Reporter {
21 | public static final String VALUE_COLUMN = "value";
22 | public static final String USERNAME_ARG = "username";
23 | public static final String PASSWORD_ARG = "password";
24 | public static final String DATABASE_ARG = "database";
25 | public static final String TAG_MAPPING_ARG = "tagMapping";
26 |
27 | private String username;
28 | private String password;
29 | private String database;
30 | private String tagMapping;
31 | private final Map tags;
32 |
33 | public InfluxDBReporter(Arguments arguments) {
34 | super(arguments);
35 | String prefix = arguments.metricsPrefix;
36 | // If we have a tag mapping it must match the number of components of the prefix
37 | Preconditions.checkArgument(tagMapping == null || tagMapping.split("\\.").length == prefix.split("\\.").length);
38 | tags = TagUtil.getTags(tagMapping, prefix, true);
39 | }
40 |
41 | /**
42 | * Record a gauge value in InfluxDB
43 | *
44 | * @param key The key for the gauge
45 | * @param value The value of the gauge
46 | */
47 | @Override
48 | public void recordGaugeValue(String key, long value, String... tags) {
49 | Map gauges = ImmutableMap.of(key, value);
50 | recordGaugeValues(gauges);
51 | }
52 |
53 | /**
54 | * @see #recordGaugeValue(String, long, String...)
55 | */
56 | @Override
57 | public void recordGaugeValue(String key, double value, String... tags) {
58 | Map gauges = ImmutableMap.of(key, value);
59 | recordGaugeValues(gauges);
60 | }
61 |
62 | /**
63 | * Record multiple gauge values in InfluxDB
64 | *
65 | * @param gauges A map of gauge names to values
66 | */
67 | @Override
68 | public void recordGaugeValues(Map gauges, String... tags) {
69 | long time = System.currentTimeMillis();
70 | BatchPoints batchPoints = BatchPoints.database(database).build();
71 | for (Map.Entry gauge: gauges.entrySet()) {
72 | batchPoints.point(constructPoint(time, gauge.getKey(), gauge.getValue()));
73 | }
74 | client.write(batchPoints);
75 | }
76 |
77 | /**
78 | * InfluxDB has a rich query language and does not need the bounds metrics emitted by CPUTracingProfiler
79 | * As such we can disable emitting these metrics
80 | *
81 | * @return false
82 | */
83 | @Override
84 | public boolean emitBounds() {
85 | return false;
86 | }
87 |
88 | /**
89 | *
90 | * @param server The server to which to report data
91 | * @param port The port on which the server is running
92 | * @param prefix The prefix for metrics
93 | * @return An InfluxDB client
94 | */
95 | @Override
96 | protected InfluxDB createClient(String server, int port, String prefix) {
97 | return InfluxDBFactory.connect(String.format("http://%s:%d", server, port), username, password);
98 | }
99 |
100 | /**
101 | * Handle remaining arguments
102 | *
103 | * @param arguments The arguments given to the profiler agent
104 | */
105 | @Override
106 | protected void handleArguments(Arguments arguments) {
107 | username = arguments.remainingArgs.get(USERNAME_ARG);
108 | password = arguments.remainingArgs.get(PASSWORD_ARG);
109 | database = arguments.remainingArgs.get(DATABASE_ARG);
110 | tagMapping = arguments.remainingArgs.get(TAG_MAPPING_ARG);
111 |
112 | Preconditions.checkNotNull(username);
113 | Preconditions.checkNotNull(password);
114 | Preconditions.checkNotNull(database);
115 | }
116 |
117 | private Point constructPoint(long time, String key, Number value) {
118 | Point.Builder builder = Point.measurement(key)
119 | .time(time, TimeUnit.MILLISECONDS)
120 | .field(VALUE_COLUMN, value);
121 | for (Map.Entry entry : tags.entrySet()) {
122 | builder = builder.tag(entry.getKey(), entry.getValue());
123 | }
124 |
125 | return builder.build();
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/profilers/CPUTracingProfiler.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.profilers;
2 |
3 | import com.etsy.statsd.profiler.Arguments;
4 | import com.etsy.statsd.profiler.Profiler;
5 | import com.etsy.statsd.profiler.reporter.Reporter;
6 | import com.etsy.statsd.profiler.util.*;
7 | import com.etsy.statsd.profiler.worker.ProfilerThreadFactory;
8 | import com.google.common.base.Predicate;
9 | import com.google.common.collect.Iterables;
10 | import com.google.common.collect.Lists;
11 |
12 | import java.lang.management.ThreadInfo;
13 | import java.util.ArrayList;
14 | import java.util.Arrays;
15 | import java.util.Collection;
16 | import java.util.List;
17 | import java.util.concurrent.TimeUnit;
18 |
19 | /**
20 | * Profiles CPU time spent in each method
21 | *
22 | * @author Andrew Johnson
23 | */
24 | public class CPUTracingProfiler extends Profiler {
25 | private static final String PACKAGE_WHITELIST_ARG = "packageWhitelist";
26 | private static final String PACKAGE_BLACKLIST_ARG = "packageBlacklist";
27 |
28 | public static final long REPORTING_PERIOD = 1;
29 | public static final long PERIOD = 10;
30 | public static final List EXCLUDE_PACKAGES = Arrays.asList("com.etsy.statsd.profiler", "com.timgroup.statsd");
31 |
32 | private final CPUTraces traces;
33 | private long profileCount;
34 | private StackTraceFilter filter;
35 | private final long reportingFrequency;
36 |
37 |
38 | public CPUTracingProfiler(Reporter reporter, Arguments arguments) {
39 | super(reporter, arguments);
40 | traces = new CPUTraces();
41 | profileCount = 0;
42 | reportingFrequency = TimeUtil.convertReportingPeriod(getPeriod(), getTimeUnit(), REPORTING_PERIOD, TimeUnit.SECONDS);
43 | }
44 |
45 | /**
46 | * Profile CPU time by method call
47 | */
48 | @Override
49 | public void profile() {
50 | profileCount++;
51 |
52 | for (ThreadInfo thread : getAllRunnableThreads()) {
53 | // certain threads do not have stack traces
54 | if (thread.getStackTrace().length > 0) {
55 | String traceKey = StackTraceFormatter.formatStackTrace(thread.getStackTrace());
56 | if (filter.includeStackTrace(traceKey)) {
57 | traces.increment(traceKey, 1);
58 | }
59 | }
60 | }
61 |
62 | // To keep from overwhelming StatsD, we only report statistics every second
63 | if (profileCount == reportingFrequency) {
64 | profileCount = 0;
65 | recordMethodCounts();
66 | }
67 | }
68 |
69 | /**
70 | * Flush methodCounts data on shutdown
71 | */
72 | @Override
73 | public void flushData() {
74 | recordMethodCounts();
75 | // These bounds are recorded to help speed up generating flame graphs for certain backends
76 | if (emitBounds()) {
77 | Range bounds = traces.getBounds();
78 | recordGaugeValue("cpu.trace." + bounds.getLeft(), bounds.getLeft());
79 | recordGaugeValue("cpu.trace." + bounds.getRight(), bounds.getRight());
80 | }
81 | }
82 |
83 | @Override
84 | public long getPeriod() {
85 | return PERIOD;
86 | }
87 |
88 | @Override
89 | public TimeUnit getTimeUnit() {
90 | return TimeUnit.MILLISECONDS;
91 | }
92 |
93 | @Override
94 | protected void handleArguments(Arguments arguments) {
95 | List packageWhitelist = parsePackageList(arguments.remainingArgs.get(PACKAGE_WHITELIST_ARG));
96 | List packageBlacklist = parsePackageList(arguments.remainingArgs.get(PACKAGE_BLACKLIST_ARG));
97 | filter = new StackTraceFilter(packageWhitelist, Lists.newArrayList(Iterables.concat(EXCLUDE_PACKAGES, packageBlacklist)));
98 | }
99 |
100 | /**
101 | * Parses a colon-delimited list of packages
102 | *
103 | * @param packages A string containing a colon-delimited list of packages
104 | * @return A List of packages
105 | */
106 | private static List parsePackageList(String packages) {
107 | if (packages == null) {
108 | return new ArrayList<>();
109 | } else {
110 | return Arrays.asList(packages.split(":"));
111 | }
112 | }
113 |
114 | /**
115 | * Records method CPU time in StatsD
116 | */
117 | private void recordMethodCounts() {
118 | recordGaugeValues(traces.getDataToFlush());
119 | }
120 |
121 | /**
122 | * Gets all runnable threads, excluding profiler threads
123 | *
124 | * @return A Collection representing current thread state
125 | */
126 | private static Collection getAllRunnableThreads() {
127 | return ThreadDumper.filterAllThreadsInState(false, false, Thread.State.RUNNABLE, new Predicate() {
128 | @Override
129 | public boolean apply(ThreadInfo input) {
130 | return !input.getThreadName().startsWith(ProfilerThreadFactory.NAME_PREFIX);
131 | }
132 | });
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/checkstyle.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
--------------------------------------------------------------------------------
/src/test/java/com/etsy/statsd/profiler/server/ProfilerServerTest.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.server;
2 |
3 | import com.etsy.statsd.profiler.Profiler;
4 | import com.etsy.statsd.profiler.profilers.MockProfiler1;
5 | import com.etsy.statsd.profiler.profilers.MockProfiler2;
6 | import com.etsy.statsd.profiler.worker.ProfilerThreadFactory;
7 | import com.etsy.statsd.profiler.worker.ProfilerWorkerThread;
8 | import com.google.common.base.Joiner;
9 | import com.google.common.util.concurrent.MoreExecutors;
10 | import org.apache.http.HttpEntity;
11 | import org.apache.http.client.methods.CloseableHttpResponse;
12 | import org.apache.http.client.methods.HttpGet;
13 | import org.apache.http.client.methods.HttpRequestBase;
14 | import org.apache.http.impl.client.CloseableHttpClient;
15 | import org.apache.http.impl.client.HttpClients;
16 | import org.apache.http.util.EntityUtils;
17 | import org.junit.Before;
18 | import org.junit.Test;
19 |
20 | import java.io.IOException;
21 | import java.util.*;
22 | import java.util.concurrent.Executors;
23 | import java.util.concurrent.ScheduledExecutorService;
24 | import java.util.concurrent.ScheduledFuture;
25 | import java.util.concurrent.ScheduledThreadPoolExecutor;
26 | import java.util.concurrent.atomic.AtomicReference;
27 |
28 | import static org.junit.Assert.assertEquals;
29 | import static org.junit.Assert.assertNotNull;
30 |
31 | public class ProfilerServerTest {
32 | private Map activeProfilers;
33 | private int port;
34 | private AtomicReference isRunning;
35 | private List errors;
36 | private CloseableHttpClient client;
37 |
38 | @Before
39 | public void setup() throws IOException {
40 | MockProfiler1 profiler1 = new MockProfiler1(new HashSet());
41 | MockProfiler2 profiler2 = new MockProfiler2(new HashSet());
42 |
43 | activeProfilers = new HashMap<>();
44 | activeProfilers.put("MockProfiler1", profiler1);
45 | activeProfilers.put("MockProfiler2", profiler2);
46 |
47 | port = 8080;
48 |
49 | isRunning = new AtomicReference<>(true);
50 | errors = new ArrayList<>();
51 | errors.add("example error");
52 |
53 | Map> runningProfilers = new HashMap<>();
54 | ProfilerWorkerThread worker1 = new ProfilerWorkerThread(profiler1, errors);
55 | ProfilerWorkerThread worker2 = new ProfilerWorkerThread(profiler2, errors);
56 | ScheduledExecutorService scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(
57 | (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2, new ProfilerThreadFactory()));
58 | ScheduledFuture future1 = scheduledExecutorService.scheduleAtFixedRate(worker1, 0, profiler1.getPeriod(), profiler1.getTimeUnit());
59 | ScheduledFuture future2 = scheduledExecutorService.scheduleAtFixedRate(worker2, 0, profiler2.getPeriod(), profiler2.getTimeUnit());
60 | runningProfilers.put("MockProfiler1", future1);
61 | runningProfilers.put("MockProfiler2", future2);
62 |
63 | ProfilerServer.startServer(runningProfilers, activeProfilers, port, isRunning, errors);
64 | client = HttpClients.createDefault();
65 | }
66 |
67 | @Test
68 | public void testDisableProfiler() throws IOException {
69 | String profilerString = "MockProfiler1\nMockProfiler2";
70 | httpRequestTest("profilers", profilerString);
71 |
72 | String profilerToDisable = "MockProfiler1";
73 | httpRequestTest(String.format("disable/%s", profilerToDisable), String.format("Disabled profiler %s", profilerToDisable));
74 |
75 | profilerString = "MockProfiler2";
76 | httpRequestTest("profilers", profilerString);
77 | }
78 |
79 | @Test
80 | public void testProfilerStatus() throws IOException {
81 | String profilerName = "MockProfiler1";
82 | long recordedStats = activeProfilers.get(profilerName).getRecordedStats();
83 | httpRequestTest(String.format("status/profiler/%s", profilerName), String.format("Recorded stats %d\n", recordedStats));
84 | }
85 |
86 | @Test
87 | public void testProfilers() throws IOException {
88 | String profilerString = "MockProfiler1\nMockProfiler2";
89 | httpRequestTest("profilers", profilerString);
90 | }
91 |
92 | @Test
93 | public void testErrors() throws IOException {
94 | String errorString = Joiner.on("\n").join(errors);
95 | httpRequestTest("errors", String.format("Errors: %s", errorString));
96 | }
97 |
98 | @Test
99 | public void testIsRunning() throws IOException {
100 | httpRequestTest("isRunning", String.format("isRunning: %s", isRunning.get().toString()));
101 | }
102 |
103 | private void httpRequestTest(String path, String expectedBody) throws IOException {
104 | HttpRequestBase get = new HttpGet(String.format("http://localhost:%d/%s", port, path));
105 | CloseableHttpResponse response = client.execute(get);
106 |
107 | int statusCode = response.getStatusLine().getStatusCode();
108 | assertEquals(200, statusCode);
109 |
110 | HttpEntity entity = response.getEntity();
111 | assertNotNull(entity);
112 | String responseBody = EntityUtils.toString(entity);
113 | assertEquals(expectedBody, responseBody);
114 |
115 | response.close();
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/Agent.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler;
2 |
3 | import com.etsy.statsd.profiler.reporter.Reporter;
4 | import com.etsy.statsd.profiler.server.ProfilerServer;
5 | import com.etsy.statsd.profiler.worker.ProfilerShutdownHookWorker;
6 | import com.etsy.statsd.profiler.worker.ProfilerThreadFactory;
7 | import com.etsy.statsd.profiler.worker.ProfilerWorkerThread;
8 | import com.google.common.util.concurrent.MoreExecutors;
9 |
10 | import java.lang.instrument.Instrumentation;
11 | import java.lang.reflect.Constructor;
12 | import java.lang.reflect.InvocationTargetException;
13 | import java.util.*;
14 | import java.util.concurrent.Executors;
15 | import java.util.concurrent.ScheduledExecutorService;
16 | import java.util.concurrent.ScheduledFuture;
17 | import java.util.concurrent.ScheduledThreadPoolExecutor;
18 | import java.util.concurrent.atomic.AtomicReference;
19 |
20 | /**
21 | * javaagent profiler using StatsD as a backend
22 | *
23 | * @author Andrew Johnson
24 | */
25 | public final class Agent {
26 | public static final int EXECUTOR_DELAY = 0;
27 |
28 | static AtomicReference isRunning = new AtomicReference<>(true);
29 | static LinkedList errors = new LinkedList<>();
30 |
31 | private Agent() { }
32 |
33 | public static void agentmain(final String args, final Instrumentation instrumentation) {
34 | premain(args, instrumentation);
35 | }
36 |
37 | /**
38 | * Start the profiler
39 | *
40 | * @param args Profiler arguments
41 | * @param instrumentation Instrumentation agent
42 | */
43 | public static void premain(final String args, final Instrumentation instrumentation) {
44 | Arguments arguments = Arguments.parseArgs(args);
45 |
46 | Reporter reporter = instantiate(arguments.reporter, Reporter.CONSTRUCTOR_PARAM_TYPES, arguments);
47 |
48 | Collection profilers = new ArrayList<>();
49 | for (Class profiler : arguments.profilers) {
50 | profilers.add(instantiate(profiler, Profiler.CONSTRUCTOR_PARAM_TYPES, reporter, arguments));
51 | }
52 |
53 | scheduleProfilers(profilers, arguments);
54 | registerShutdownHook(profilers);
55 | }
56 |
57 | /**
58 | * Schedule profilers with a SchedulerExecutorService
59 | *
60 | * @param profilers Collection of profilers to schedule
61 | * @param arguments
62 | */
63 | private static void scheduleProfilers(Collection profilers, Arguments arguments) {
64 | // We need to convert to an ExitingScheduledExecutorService so the JVM shuts down
65 | // when the main thread finishes
66 | ScheduledExecutorService scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(
67 | (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(profilers.size(), new ProfilerThreadFactory()));
68 |
69 | Map> runningProfilers = new HashMap<>(profilers.size());
70 | Map activeProfilers = new HashMap<>(profilers.size());
71 | for (Profiler profiler : profilers) {
72 | activeProfilers.put(profiler.getClass().getSimpleName(), profiler);
73 | ProfilerWorkerThread worker = new ProfilerWorkerThread(profiler, errors);
74 | ScheduledFuture future = scheduledExecutorService.scheduleAtFixedRate(worker, EXECUTOR_DELAY, profiler.getPeriod(), profiler.getTimeUnit());
75 | runningProfilers.put(profiler.getClass().getSimpleName(), future);
76 | }
77 |
78 | if (arguments.httpServerEnabled) {
79 | ProfilerServer.startServer(runningProfilers, activeProfilers, arguments.httpPort, isRunning, errors);
80 | }
81 | }
82 |
83 | /**
84 | * Register a shutdown hook to flush profiler data to StatsD
85 | *
86 | * @param profilers The profilers to flush at shutdown
87 | */
88 | private static void registerShutdownHook(Collection profilers) {
89 | Thread shutdownHook = new Thread(new ProfilerShutdownHookWorker(profilers, isRunning));
90 | Runtime.getRuntime().addShutdownHook(shutdownHook);
91 | }
92 |
93 | /**
94 | * Uniformed handling of initialization exception
95 | *
96 | * @param clazz The class that could not be instantiated
97 | * @param cause The underlying exception
98 | */
99 | private static void handleInitializationException(final Class clazz, final Exception cause) {
100 | throw new RuntimeException("Unable to instantiate " + clazz.getSimpleName(), cause);
101 | }
102 |
103 | /**
104 | * Instantiate an object
105 | *
106 | * @param clazz A Class representing the type of object to instantiate
107 | * @param parameterTypes The parameter types for the constructor
108 | * @param initArgs The values to pass to the constructor
109 | * @param The type of the object to instantiate
110 | * @return A new instance of type T
111 | */
112 | private static T instantiate(final Class clazz, Class[] parameterTypes, Object... initArgs) {
113 | try {
114 | Constructor constructor = clazz.getConstructor(parameterTypes);
115 | return constructor.newInstance(initArgs);
116 | } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
117 | handleInitializationException(clazz, e);
118 | }
119 |
120 | return null;
121 | }
122 | }
123 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/Arguments.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler;
2 |
3 | import com.etsy.statsd.profiler.profilers.CPUTracingProfiler;
4 | import com.etsy.statsd.profiler.profilers.MemoryProfiler;
5 | import com.etsy.statsd.profiler.reporter.Reporter;
6 | import com.etsy.statsd.profiler.reporter.StatsDReporter;
7 | import com.google.common.base.Optional;
8 |
9 | import java.util.*;
10 |
11 | /**
12 | * Represents arguments to the profiler
13 | *
14 | * @author Andrew Johnson
15 | */
16 | public final class Arguments {
17 | private static final String SERVER = "server";
18 | private static final String PORT = "port";
19 | private static final String METRICS_PREFIX = "prefix";
20 | private static final String PROFILERS = "profilers";
21 | private static final String REPORTER = "reporter";
22 | private static final String HTTP_PORT = "httpPort";
23 | private static final String HTTP_SEVER_ENABLED = "httpServerEnabled";
24 |
25 | private static final Collection REQUIRED = Arrays.asList(SERVER, PORT);
26 |
27 | public String server;
28 | public int port;
29 | public String metricsPrefix;
30 | public Set> profilers;
31 | public Map remainingArgs;
32 | public Class> reporter;
33 | public int httpPort;
34 | public boolean httpServerEnabled;
35 |
36 | private Arguments(Map parsedArgs) {
37 | server = parsedArgs.get(SERVER);
38 | port = Integer.parseInt(parsedArgs.get(PORT));
39 | metricsPrefix = Optional.fromNullable(parsedArgs.get(METRICS_PREFIX)).or("statsd-jvm-profiler");
40 | profilers = parseProfilerArg(parsedArgs.get(PROFILERS));
41 | reporter = parserReporterArg(parsedArgs.get(REPORTER));
42 | httpPort = Integer.parseInt(Optional.fromNullable(parsedArgs.get(HTTP_PORT)).or("5005"));
43 | httpServerEnabled = Boolean.parseBoolean(Optional.fromNullable(parsedArgs.get(HTTP_SEVER_ENABLED)).or("true"));
44 |
45 | parsedArgs.remove(SERVER);
46 | parsedArgs.remove(PORT);
47 | parsedArgs.remove(METRICS_PREFIX);
48 | parsedArgs.remove(PROFILERS);
49 | remainingArgs = parsedArgs;
50 | }
51 |
52 | /**
53 | * Parses arguments into an Arguments object
54 | *
55 | * @param args A String containing comma-delimited args in k=v form
56 | * @return An Arguments object representing the given arguments
57 | */
58 | public static Arguments parseArgs(final String args) {
59 | Map parsed = new HashMap<>();
60 | for (String argPair : args.split(",")) {
61 | String[] tokens = argPair.split("=");
62 | if (tokens.length != 2) {
63 | throw new IllegalArgumentException("statsd-jvm-profiler takes a comma-delimited list of arguments in k=v form");
64 | }
65 |
66 | parsed.put(tokens[0], tokens[1]);
67 | }
68 |
69 | for (String requiredArg : REQUIRED) {
70 | if (!parsed.containsKey(requiredArg)) {
71 | throw new IllegalArgumentException(String.format("%s argument was not supplied", requiredArg));
72 | }
73 | }
74 |
75 | return new Arguments(parsed);
76 | }
77 |
78 | @SuppressWarnings("unchecked")
79 | private static Class> parserReporterArg(String reporterArg) {
80 | if (reporterArg == null) {
81 | return StatsDReporter.class;
82 | } else {
83 | try {
84 | return (Class>) Class.forName(reporterArg);
85 | } catch (ClassNotFoundException e) {
86 | // This might indicate the package was left off, so we'll try with the default package
87 | try {
88 | return (Class>) Class.forName("com.etsy.statsd.profiler.reporter." + reporterArg);
89 | } catch (ClassNotFoundException inner) {
90 | throw new IllegalArgumentException("Reporter " + reporterArg + " not found", inner);
91 | }
92 | }
93 | }
94 | }
95 |
96 | @SuppressWarnings("unchecked")
97 | private static Set> parseProfilerArg(String profilerArg) {
98 | Set> parsedProfilers = new HashSet<>();
99 | if (profilerArg == null) {
100 | parsedProfilers.add(CPUTracingProfiler.class);
101 | parsedProfilers.add(MemoryProfiler.class);
102 | } else {
103 | for (String p : profilerArg.split(":")) {
104 | try {
105 | parsedProfilers.add((Class) Class.forName(p));
106 | } catch (ClassNotFoundException e) {
107 | // This might indicate the package was left off, so we'll try with the default package
108 | try {
109 | parsedProfilers.add((Class) Class.forName("com.etsy.statsd.profiler.profilers." + p));
110 | } catch (ClassNotFoundException inner) {
111 | throw new IllegalArgumentException("Profiler " + p + " not found", inner);
112 | }
113 | }
114 | }
115 | }
116 |
117 | if (parsedProfilers.isEmpty()) {
118 | throw new IllegalArgumentException("At least one profiler must be specified");
119 | }
120 |
121 | return parsedProfilers;
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/src/test/java/com/etsy/statsd/profiler/ArgumentsTest.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler;
2 |
3 | import com.etsy.statsd.profiler.profilers.CPUTracingProfiler;
4 | import com.etsy.statsd.profiler.profilers.MemoryProfiler;
5 | import com.etsy.statsd.profiler.reporter.InfluxDBReporter;
6 | import com.etsy.statsd.profiler.reporter.StatsDReporter;
7 | import org.junit.Rule;
8 | import org.junit.Test;
9 | import org.junit.rules.ExpectedException;
10 |
11 | import java.util.HashSet;
12 | import java.util.Set;
13 |
14 | import static org.junit.Assert.assertEquals;
15 | import static org.junit.Assert.assertFalse;
16 | import static org.junit.Assert.assertTrue;
17 |
18 | public class ArgumentsTest {
19 | @Rule
20 | public ExpectedException exception = ExpectedException.none();
21 |
22 | @Test
23 | public void testInvalidArgument() {
24 | String args = "key=value,key2";
25 |
26 | exception.expect(IllegalArgumentException.class);
27 | Arguments.parseArgs(args);
28 | }
29 |
30 | @Test
31 | public void testMissingRequiredArgument() {
32 | String args = "server=localhost,prefix=prefix";
33 |
34 | exception.expect(IllegalArgumentException.class);
35 | Arguments.parseArgs(args);
36 | }
37 |
38 | @Test
39 | public void testNonNumericPort() {
40 | String args = "server=localhost,port=abcd";
41 |
42 | exception.expect(NumberFormatException.class);
43 | Arguments.parseArgs(args);
44 | }
45 |
46 | @Test
47 | public void testNoOptionalArguments() {
48 | String args = "server=localhost,port=8125";
49 | Arguments arguments = Arguments.parseArgs(args);
50 |
51 | assertEquals("localhost", arguments.server);
52 | assertEquals(8125, arguments.port);
53 | assertEquals("statsd-jvm-profiler", arguments.metricsPrefix);
54 | }
55 |
56 | @Test
57 | public void testOptionalArguments() {
58 | String args = "server=localhost,port=8125,prefix=i.am.a.prefix,packageWhitelist=com.etsy";
59 | Arguments arguments = Arguments.parseArgs(args);
60 |
61 | assertEquals("localhost", arguments.server);
62 | assertEquals(8125, arguments.port);
63 | assertEquals("i.am.a.prefix", arguments.metricsPrefix);
64 | }
65 |
66 | @Test
67 | public void testDefaultProfilers() {
68 | String args = "server=localhost,port=8125";
69 | Arguments arguments = Arguments.parseArgs(args);
70 |
71 | Set> expected = new HashSet<>();
72 | expected.add(CPUTracingProfiler.class);
73 | expected.add(MemoryProfiler.class);
74 |
75 | assertEquals(expected, arguments.profilers);
76 | }
77 |
78 | @Test
79 | public void testProfilerWithPackage() {
80 | String args = "server=localhost,port=8125,profilers=com.etsy.statsd.profiler.profilers.CPUTracingProfiler";
81 | Arguments arguments = Arguments.parseArgs(args);
82 |
83 | Set> expected = new HashSet<>();
84 | expected.add(CPUTracingProfiler.class);
85 |
86 | assertEquals(expected, arguments.profilers);
87 | }
88 |
89 | @Test
90 | public void testProfilerWithoutPackage() {
91 | String args = "server=localhost,port=8125,profilers=MemoryProfiler";
92 | Arguments arguments = Arguments.parseArgs(args);
93 |
94 | Set> expected = new HashSet<>();
95 | expected.add(MemoryProfiler.class);
96 |
97 | assertEquals(expected, arguments.profilers);
98 | }
99 |
100 | @Test
101 | public void testMultipleProfilers() {
102 | String args = "server=localhost,port=8125,profilers=CPUTracingProfiler:MemoryProfiler";
103 | Arguments arguments = Arguments.parseArgs(args);
104 |
105 | Set> expected = new HashSet<>();
106 | expected.add(CPUTracingProfiler.class);
107 | expected.add(MemoryProfiler.class);
108 |
109 | assertEquals(expected, arguments.profilers);
110 | }
111 |
112 | @Test
113 | public void testProfilerNotFound() {
114 | String args = "server=localhost,port=8125,profilers=FakeProfiler";
115 |
116 | exception.expect(IllegalArgumentException.class);
117 | Arguments.parseArgs(args);
118 | }
119 |
120 | @Test
121 | public void testReporterWithoutPackage() {
122 | String args = "server=localhost,port=8125,reporter=InfluxDBReporter";
123 | Arguments arguments = Arguments.parseArgs(args);
124 |
125 | assertEquals(InfluxDBReporter.class, arguments.reporter);
126 | }
127 |
128 | @Test
129 | public void testReporterWithPackage() {
130 | String args = "server=localhost,port=8125,reporter=com.etsy.statsd.profiler.reporter.InfluxDBReporter";
131 | Arguments arguments = Arguments.parseArgs(args);
132 |
133 | assertEquals(InfluxDBReporter.class, arguments.reporter);
134 | }
135 |
136 | @Test(expected = IllegalArgumentException.class)
137 | public void testReporterNotFound() {
138 | String args = "server=localhost,port=8125,reporter=NotRealReporter";
139 | Arguments.parseArgs(args);
140 | }
141 |
142 | @Test
143 | public void testDefaultReporter() {
144 | String args = "server=localhost,port=8125";
145 | Arguments arguments = Arguments.parseArgs(args);
146 |
147 | assertEquals(StatsDReporter.class, arguments.reporter);
148 | }
149 |
150 | @Test
151 | public void testHttpServerEnabledByDefault() throws Exception {
152 | String args = "server=localhost,port=8125";
153 | Arguments arguments = Arguments.parseArgs(args);
154 |
155 | assertTrue(arguments.httpServerEnabled);
156 | }
157 |
158 | @Test
159 | public void testHttpServerDisabled() throws Exception {
160 | String args = "server=localhost,port=8125,httpServerEnabled=false";
161 | Arguments arguments = Arguments.parseArgs(args);
162 |
163 | assertFalse(arguments.httpServerEnabled);
164 | }
165 | }
166 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/server/RequestHandler.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.server;
2 |
3 | import com.etsy.statsd.profiler.Profiler;
4 | import com.google.common.base.Function;
5 | import com.google.common.base.Joiner;
6 | import com.google.common.base.Predicate;
7 | import com.google.common.collect.Collections2;
8 | import org.vertx.java.core.Handler;
9 | import org.vertx.java.core.http.HttpServerRequest;
10 | import org.vertx.java.core.http.RouteMatcher;
11 |
12 | import java.util.*;
13 | import java.util.concurrent.ScheduledFuture;
14 | import java.util.concurrent.atomic.AtomicReference;
15 |
16 | /**
17 | * Handler for HTTP requests to the profiler server
18 | *
19 | * @author Andrew Johnson
20 | */
21 | public final class RequestHandler {
22 | private RequestHandler() { }
23 |
24 | /**
25 | * Construct a RouteMatcher for the supported routes
26 | *
27 | * @param activeProfilers The active profilers
28 | * @return A RouteMatcher that matches all supported routes
29 | */
30 | public static RouteMatcher getMatcher(final Map> runningProfilers, Map activeProfilers, AtomicReference isRunning, List errors) {
31 | RouteMatcher matcher = new RouteMatcher();
32 | matcher.get("/profilers", RequestHandler.handleGetProfilers(runningProfilers));
33 | matcher.get("/disable/:profiler", RequestHandler.handleDisableProfiler(runningProfilers));
34 | matcher.get("/status/profiler/:profiler", RequestHandler.handleProfilerStatus(activeProfilers));
35 | matcher.get("/errors", RequestHandler.handleErrorMessages(errors));
36 | matcher.get("/isRunning", RequestHandler.isRunning(isRunning));
37 | return matcher;
38 | }
39 |
40 | /**
41 | * Handle a GET to /isRunning
42 | *
43 | * @return A Handler that returns all running profilers
44 | */
45 | public static Handler isRunning(final AtomicReference isRunning) {
46 | return new Handler() {
47 | @Override
48 | public void handle(HttpServerRequest httpServerRequest) {
49 | httpServerRequest.response().end(String.format("isRunning: %b", isRunning.get()));
50 | }
51 | };
52 | }
53 |
54 | /**
55 | * Handle a GET to /profilers
56 | *
57 | * @return A Handler that handles a request to the /profilers endpoint
58 | */
59 | public static Handler handleGetProfilers(final Map> runningProfilers) {
60 | return new Handler() {
61 | @Override
62 | public void handle(HttpServerRequest httpServerRequest) {
63 | httpServerRequest.response().end(Joiner.on("\n").join(getEnabledProfilers(runningProfilers)));
64 | }
65 | };
66 | }
67 |
68 | /**
69 | * Handle a GET to /errors
70 | *
71 | * @return The last 10 error stacktraces
72 | */
73 | public static Handler handleErrorMessages(final List errors) {
74 | return new Handler() {
75 | @Override
76 | public void handle(HttpServerRequest httpServerRequest) {
77 | httpServerRequest.response().end("Errors: " + Joiner.on("\n").join(errors));
78 | }
79 | };
80 | }
81 |
82 | /**
83 | * Handle a GET to /disable/:profiler
84 | *
85 | * @param activeProfilers The active profilers
86 | * @return A Handler that handles a request to the /disable/:profiler endpoint
87 | */
88 | public static Handler handleDisableProfiler(final Map> activeProfilers) {
89 | return new Handler() {
90 | @Override
91 | public void handle(HttpServerRequest httpServerRequest) {
92 | String profilerToDisable = httpServerRequest.params().get("profiler");
93 | ScheduledFuture future = activeProfilers.get(profilerToDisable);
94 | future.cancel(false);
95 | httpServerRequest.response().end(String.format("Disabled profiler %s", profilerToDisable));
96 | }
97 | };
98 | }
99 |
100 | /**
101 | * Handle a GET to /status/profiler/:profiler
102 | *
103 | * @param activeProfilers The active profilers
104 | * @return A Handler that handles a request to the /disable/:profiler endpoint
105 | */
106 | public static Handler handleProfilerStatus(final Map activeProfilers) {
107 | return new Handler() {
108 | @Override
109 | public void handle(HttpServerRequest httpServerRequest) {
110 | String profilerName = httpServerRequest.params().get("profiler");
111 | Profiler profiler = activeProfilers.get(profilerName);
112 | httpServerRequest.response().end(String.format("Recorded stats %d\n", profiler.getRecordedStats()));
113 | }
114 | };
115 | }
116 |
117 | /**
118 | * Get all enabled profilers
119 | * @param activeProfilers The active profilers
120 | * @return A sorted List containing the names of profilers that are currently running
121 | */
122 | private static List getEnabledProfilers(final Map> activeProfilers) {
123 | Collection profilers = Collections2.transform(Collections2.filter(activeProfilers.entrySet(), new Predicate>>() {
124 | @Override
125 | public boolean apply(Map.Entry> input) {
126 | return !input.getValue().isDone();
127 | }
128 | }), new Function>, String>() {
129 | @Override
130 | public String apply(Map.Entry> input) {
131 | return input.getKey();
132 | }
133 | });
134 |
135 | List result = new ArrayList<>(profilers);
136 | Collections.sort(result);
137 | return result;
138 |
139 | }
140 | }
141 |
--------------------------------------------------------------------------------
/src/main/java/com/etsy/statsd/profiler/profilers/MemoryProfiler.java:
--------------------------------------------------------------------------------
1 | package com.etsy.statsd.profiler.profilers;
2 |
3 | import com.etsy.statsd.profiler.Arguments;
4 | import com.etsy.statsd.profiler.Profiler;
5 | import com.etsy.statsd.profiler.reporter.Reporter;
6 | import com.google.common.collect.Maps;
7 |
8 | import java.io.BufferedReader;
9 | import java.io.IOException;
10 | import java.io.InputStreamReader;
11 | import java.lang.management.*;
12 | import java.util.HashMap;
13 | import java.util.List;
14 | import java.util.Map;
15 | import java.util.concurrent.TimeUnit;
16 | import java.util.concurrent.atomic.AtomicLong;
17 | import java.util.logging.Logger;
18 |
19 | /**
20 | * Profiles memory usage and GC statistics
21 | *
22 | * @author Andrew Johnson
23 | */
24 | public class MemoryProfiler extends Profiler {
25 | private static final Logger LOGGER = Logger.getLogger(MemoryProfiler.class.getName());
26 |
27 | private static final long PERIOD = 10;
28 |
29 | private final MemoryMXBean memoryMXBean;
30 | private final List gcMXBeans;
31 | private final HashMap gcTimes = new HashMap<>();
32 | private final ClassLoadingMXBean classLoadingMXBean;
33 | private final List memoryPoolMXBeans;
34 | private final Integer pid;
35 |
36 | public MemoryProfiler(Reporter reporter, Arguments arguments) {
37 | super(reporter, arguments);
38 | memoryMXBean = ManagementFactory.getMemoryMXBean();
39 | gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
40 | classLoadingMXBean = ManagementFactory.getClassLoadingMXBean();
41 | memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
42 |
43 | for (GarbageCollectorMXBean b : gcMXBeans) {
44 | gcTimes.put(b, new AtomicLong());
45 | }
46 |
47 | String processName = ManagementFactory.getRuntimeMXBean().getName();
48 | pid = Integer.parseInt(processName.split("@")[0]);
49 | LOGGER.info("Process id: " + pid);
50 | }
51 |
52 | /**
53 | * Profile memory usage and GC statistics
54 | */
55 | @Override
56 | public void profile() {
57 | recordStats();
58 | }
59 |
60 | @Override
61 | public void flushData() {
62 | recordStats();
63 | }
64 |
65 | @Override
66 | public long getPeriod() {
67 | return PERIOD;
68 | }
69 |
70 | @Override
71 | public TimeUnit getTimeUnit() {
72 | return TimeUnit.SECONDS;
73 | }
74 |
75 | @Override
76 | protected void handleArguments(Arguments arguments) { /* No arguments needed */ }
77 |
78 | /**
79 | * Records all memory statistics
80 | */
81 | private void recordStats() {
82 | long finalizationPendingCount = memoryMXBean.getObjectPendingFinalizationCount();
83 | MemoryUsage heap = memoryMXBean.getHeapMemoryUsage();
84 | MemoryUsage nonHeap = memoryMXBean.getNonHeapMemoryUsage();
85 | Map metrics = Maps.newHashMap();
86 |
87 | metrics.put("pending-finalization-count", finalizationPendingCount);
88 | recordMemoryUsage("heap.total", heap, metrics);
89 | recordMemoryUsage("nonheap.total", nonHeap, metrics);
90 |
91 | for (GarbageCollectorMXBean gcMXBean : gcMXBeans) {
92 | String gcName = gcMXBean.getName().replace(" ", "_");
93 | metrics.put("gc." + gcName + ".count", gcMXBean.getCollectionCount());
94 |
95 | final long time = gcMXBean.getCollectionTime();
96 | final long prevTime = gcTimes.get(gcMXBean).get();
97 | final long runtime = time - prevTime;
98 |
99 | metrics.put("gc." + gcName + ".time", time);
100 | metrics.put("gc." + gcName + ".runtime", runtime);
101 |
102 | if (runtime > 0) {
103 | gcTimes.get(gcMXBean).set(time);
104 | }
105 | }
106 |
107 | long loadedClassCount = classLoadingMXBean.getLoadedClassCount();
108 | long totalLoadedClassCount = classLoadingMXBean.getTotalLoadedClassCount();
109 | long unloadedClassCount = classLoadingMXBean.getUnloadedClassCount();
110 |
111 | metrics.put("loaded-class-count", loadedClassCount);
112 | metrics.put("total-loaded-class-count", totalLoadedClassCount);
113 | metrics.put("unloaded-class-count", unloadedClassCount);
114 |
115 | for (MemoryPoolMXBean memoryPoolMXBean: memoryPoolMXBeans) {
116 | String type = poolTypeToMetricName(memoryPoolMXBean.getType());
117 | String name = poolNameToMetricName(memoryPoolMXBean.getName());
118 | String prefix = type + '.' + name;
119 | MemoryUsage usage = memoryPoolMXBean.getUsage();
120 |
121 | recordMemoryUsage(prefix, usage, metrics);
122 | }
123 |
124 | try {
125 | Process process = Runtime.getRuntime().exec("cat /proc/" + pid + "/status");
126 | process.waitFor();
127 | BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
128 | String line;
129 | while ((line = reader.readLine()) != null) {
130 | if (line.startsWith("VmRSS:")) {
131 | metrics.put("process.rss", getRssMemory(line));
132 | break;
133 | }
134 | }
135 | } catch (IOException | InterruptedException e) {
136 | LOGGER.severe(e.getMessage());
137 | }
138 |
139 | recordGaugeValues(metrics);
140 | }
141 |
142 | static long getRssMemory(String line) {
143 | return 1024 * Long.parseLong(
144 | line.replace("VmRSS:", "").replace(" kB", "").replace(" ", "").replace("\t", "")
145 | );
146 | }
147 |
148 | /**
149 | * Records memory usage
150 | *
151 | * @param prefix The prefix to use for this object
152 | * @param memory The MemoryUsage object containing the memory usage info
153 | */
154 | private static void recordMemoryUsage(String prefix, MemoryUsage memory, Map metrics) {
155 | metrics.put(prefix + ".init", memory.getInit());
156 | metrics.put(prefix + ".used", memory.getUsed());
157 | metrics.put(prefix + ".committed", memory.getCommitted());
158 | metrics.put(prefix + ".max", memory.getMax());
159 | }
160 |
161 | /**
162 | * Formats a MemoryType into a valid metric name
163 | *
164 | * @param memoryType a MemoryType
165 | * @return a valid metric name
166 | */
167 | private static String poolTypeToMetricName(MemoryType memoryType) {
168 | switch (memoryType) {
169 | case HEAP:
170 | return "heap";
171 | case NON_HEAP:
172 | return "nonheap";
173 | default:
174 | return "unknown";
175 | }
176 | }
177 |
178 | /**
179 | * Formats a pool name into a valid metric name
180 | *
181 | * @param poolName a pool name
182 | * @return a valid metric name
183 | */
184 | private static String poolNameToMetricName(String poolName) {
185 | return poolName.toLowerCase().replaceAll("\\s+", "-");
186 | }
187 | }
188 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
3 | 4.0.0
4 |
5 | com.datadoghq
6 | spark-jvm-profiler
7 | 1.0.0-SNAPSHOT
8 | jar
9 |
10 | spark-jvm-profiler
11 | Simple JVM profiler using StatsD
12 | https://github.com/datadoghq/spark-jvm-profiler
13 |
14 |
15 |
16 | MIT License
17 | http://opensource.org/licenses/MIT
18 | repo
19 |
20 |
21 |
22 |
23 | git@github.com:datadog/spark-jvm-profiler.git
24 | scm:git:git@github.com:datadog/spark-jvm-profiler.git
25 |
26 |
27 |
28 |
29 | ajsquared
30 | Andrew Johnson
31 | github.com/ajsquared
32 |
33 |
34 | buryat
35 | Vadim Semenov
36 | github.com/buryat
37 |
38 |
39 |
40 |
41 | UTF-8
42 |
43 |
44 |
45 |
46 | ossrh
47 | https://oss.sonatype.org/content/repositories/snapshots
48 |
49 |
50 |
51 |
52 |
53 | sonatype
54 | https://oss.sonatype.org/content/repositories/releases
55 |
56 |
57 |
58 |
59 |
60 | com.datadoghq
61 | java-dogstatsd-client
62 | 2.6.1
63 |
64 |
65 | com.google.guava
66 | guava
67 | 18.0
68 |
69 |
70 | org.influxdb
71 | influxdb-java
72 | 2.0
73 |
74 |
75 | io.vertx
76 | vertx-core
77 | 2.1.5
78 |
79 |
80 | junit
81 | junit
82 | 4.11
83 | test
84 |
85 |
86 | org.mockito
87 | mockito-core
88 | 1.10.19
89 | test
90 |
91 |
92 | org.apache.httpcomponents
93 | httpclient
94 | 4.5.2
95 | test
96 |
97 |
98 |
99 |
100 |
101 | release
102 |
103 |
104 |
105 | org.apache.maven.plugins
106 | maven-gpg-plugin
107 | 1.5
108 |
109 |
110 | sign-artifacts
111 | verify
112 |
113 | sign
114 |
115 |
116 |
117 |
118 | true
119 |
120 |
121 |
122 | org.apache.maven.plugins
123 | maven-source-plugin
124 | 2.4
125 |
126 |
127 | attach-sources
128 |
129 | jar-no-fork
130 |
131 |
132 |
133 |
134 |
135 | org.apache.maven.plugins
136 | maven-javadoc-plugin
137 | 2.10.3
138 |
139 |
140 | attach-javadocs
141 |
142 | jar
143 |
144 |
145 |
146 |
147 |
148 | org.sonatype.plugins
149 | nexus-staging-maven-plugin
150 | 1.6.3
151 | true
152 |
153 | ossrh
154 | https://oss.sonatype.org/
155 | true
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 | org.apache.maven.plugins
167 | maven-compiler-plugin
168 | 3.3
169 |
170 | 1.7
171 | 1.7
172 |
173 |
174 |
175 | org.apache.maven.plugins
176 | maven-jar-plugin
177 | 2.6
178 |
179 |
180 |
181 | com.etsy.statsd.profiler.Agent
182 | com.etsy.statsd.profiler.Agent
183 |
184 |
185 |
186 |
187 |
188 | org.apache.maven.plugins
189 | maven-shade-plugin
190 | 2.4.1
191 |
192 |
193 | package
194 |
195 | shade
196 |
197 |
198 | true
199 | jar-with-dependencies
200 |
201 |
202 |
203 |
204 |
205 | org.apache.maven.plugins
206 | maven-checkstyle-plugin
207 | 2.17
208 |
209 | checkstyle.xml
210 | false
211 | true
212 |
213 |
214 |
215 | com.puppycrawl.tools
216 | checkstyle
217 | 6.14.1
218 |
219 |
220 |
221 |
222 | checkstyle
223 | test
224 |
225 | check
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # spark-jvm-profiler [](https://travis-ci.org/DataDog/spark-jvm-profiler)
2 |
3 | spark-jvm-profiler is a JVM agent profiler that sends profiling data to StatsD. Inspired by [riemann-jvm-profiler](https://github.com/riemann/riemann-jvm-profiler), it was primarily built for profiling Hadoop jobs, but can be used with any JVM process.
4 |
5 | Read [the blog post](https://codeascraft.com/2015/01/14/introducing-statsd-jvm-profiler-a-jvm-profiler-for-hadoop/) that introduced statsd-jvm-profiler on [Code as Craft](https://codeascraft.com/), Etsy's engineering blog.
6 |
7 | Also check out [the blog post](https://codeascraft.com/2015/05/12/four-months-of-statsd-jvm-profiler-a-retrospective/) reflecting on the experience of open-sourcing the project.
8 |
9 | ## Mailing List
10 | There is a mailing list for this project at https://groups.google.com/forum/#!forum/statsd-jvm-profiler. If you have questions or suggestions for the project send them here!
11 |
12 | ## Installation
13 |
14 | You will need the statsd-jvm-profiler JAR on the machine where the JVM will be running. If you are profiling Hadoop jobs, that means the JAR will need to be on all of the datanodes.
15 |
16 | The JAR can be built with `mvn package`. You will need a relatively recent Maven (at least Maven 3).
17 |
18 | statsd-jvm-profiler is available in Maven Central:
19 | ```xml
20 |
21 | com.etsy
22 | statsd-jvm-profiler
23 | 2.0.0
24 |
25 | ```
26 |
27 | If you would like an uberjar containing all of the dependencies instead of the standard JAR, use the `jar-with-dependencies` classifier:
28 | ```xml
29 |
30 | com.etsy
31 | statsd-jvm-profiler
32 | 2.0.0
33 | jar-with-dependencies
34 |
35 | ```
36 |
37 | ## Usage
38 |
39 | The profiler is enabled using the JVM's `-javaagent` argument. You are required to specify at least the StatsD host and port number to use. You can also specify the prefix for metrics and a whitelist of packages to be included in the CPU profiling. Arguments can be specified like so:
40 | ```
41 | -javaagent:/usr/etsy/statsd-jvm-profiler/statsd-jvm-profiler.jar=server=hostname,port=num
42 | ```
43 |
44 | You should use the uberjar when starting the profiler in this manner so that all the profiler's dependencies are available.
45 |
46 | The profiler can also be loaded dynamically (after the JVM has already started), but this technique requires relying on Sun's `tools.jar`, meaning it's an implementation-specific solution that might not work for all JVMs. For more information see the [Dynamic Loading section](#dynamic-loading-of-agent).
47 |
48 | An example of setting up Cascading/Scalding jobs to use the profiler can be found in the `example` directory.
49 |
50 | ### Global Options
51 |
52 | Name | Meaning
53 | ---------------- | -------
54 | server | The hostname to which the reporter should send data (required)
55 | port | The port number for the server to which the reporter should send data (required)
56 | prefix | The prefix for metrics (optional, defaults to statsd-jvm-profiler)
57 | packageWhitelist | Colon-delimited whitelist for packages to include (optional, defaults to include everything)
58 | packageBlacklist | Colon-delimited whitelist for packages to exclude (optional, defaults to exclude nothing)
59 | profilers | Colon-delimited list of profiler class names (optional, defaults to `CPUTracingProfiler` and `MemoryProfiler`)
60 | reporter | Class name of the reporter to use (optional, defaults to StatsDReporter)
61 | httpServerEnabled| Determines if the embedded HTTP server should be started. (optional, defaults to `true`)
62 | httpPort | The port on which to bind the embedded HTTP server (optional, defaults to 5005). If this port is already in use, the next free port will be taken.
63 |
64 | ### Embedded HTTP Server
65 | statsd-jvm-profiler embeds an HTTP server to support simple interactions with the profiler while it is in operation.
66 | You can configure the port on which this server runs with the `httpPort` option.
67 | You can disable it altogether using the `httpServerEnabled=false` argument.
68 |
69 | Endpoint | Usage
70 | --------------- | -----
71 | /profilers | List the currently enabled profilers
72 | /isRunning | List the running profilers. This should be the same as /profilers.
73 | /disable/:profiler | Disable the profiler specified by `:profiler`. The name must match what is returned by `/profilers`.
74 | /errors | List the past 10 errors from the running profilers and reporters.
75 | /status/profiler/:profiler | Displays a status message with the number of recorded stats for the requested profiler.
76 |
77 | ### Reporters
78 | statsd-jvm-profiler supports multiple backends. StatsD is the default, but InfluxDB is also supported. You can select the backend to use by passing the `reporter` argument to the profiler; `StatsDReporter` and `InfluxDBReporter` are the supported values.
79 |
80 | Some reporters may require additional arguments.
81 |
82 | #### StatsDReporter
83 | This reporter does not have any additional arguments.
84 |
85 | #### InfluxDBReporter
86 |
87 | Name | Meaning
88 | ----------- | -------
89 | username | The username with which to connect to InfluxDB (required)
90 | password | The password with which to connect to InfluxDB (required)
91 | database | The database to which to write metrics (required)
92 | tagMapping | A mapping of tag names from the metric prefix (optional, defaults to no mapping)
93 |
94 | ##### Tag Mapping
95 | InfluxDB 0.9 supports tagging measurements and querying based on those tags. statsd-jvm-profilers uses these tags to support richer querying of the produced data. For compatibility with other metric backends, the tags are extracted from the metric prefix.
96 |
97 | If the `tagMapping` argument is not defined, only the `prefix` tag will be added, with the value of the entire prefix.
98 |
99 | `tagMapping` should be a period-delimited set of tag names. It must have the same number of components as `prefix`, or else an exception would be thrown. Each component of `tagMapping` is the name of the tag. The component in the corresponding position of `prefix` will be the value.
100 |
101 | If you do not want to include a component of `prefix` as a tag, use the special name `SKIP` in `tagMapping` for that position.
102 |
103 | ## Profilers
104 |
105 | `statsd-jvm-profiler` offers 3 profilers: `MemoryProfiler`, `CPUTracingProfiler` and `CPULoadProfiler`.
106 |
107 | The metrics for all these profilers will prefixed with the value from the `prefix` argument or it's default value: `statsd-jvm-profiler`.
108 |
109 | You can enable specific profilers through the `profilers` argument like so:
110 | 1. Memory metrics only: `profilers=MemoryProfiler`
111 | 2. CPU Tracing metrics only: `profilers=CPUTracingProfiler`
112 | 3. JVM/System CPU load metrics only: `profilers=CPULoadProfiler`
113 |
114 | Default value: `profilers=MemoryProfiler:CPUTracingProfiler`
115 |
116 | ### Garbage Collector and Memory Profiler: `MemoryProfiler`
117 | This profiler will record:
118 |
119 | 1. Heap and non-heap memory usage
120 | 2. Number of GC pauses and GC time
121 |
122 | Assuming you use the default prefix of `statsd-jvm-profiler`,
123 | the memory usage metrics will be under `statsd-jvm-profiler.heap` and `statsd-jvm-profiler.nonheap`,
124 | the GC metrics will be under `statsd-jvm-profiler.gc`.
125 |
126 | Memory and GC metrics are reported once every 10 seconds.
127 |
128 | ### CPU Tracing Profiler: `CPUTracingProfiler`
129 | This profiler records the time spent in each function across all Threads.
130 |
131 | Assuming you use the default prefix of `statsd-jvm-profiler`, the the CPU time metrics will be under `statsd-jvm-profiler.cpu.trace`.
132 |
133 | The CPU time is sampled every millisecond, but only reported every 10 seconds.
134 | The CPU time metrics represent the total time spent in that function.
135 |
136 | Profiling a long-running process or a lot of processes simultaneously will produce a lot of data, so be careful with the
137 | capacity of your StatsD instance. The `packageWhitelist` and `packageBlacklist` arguments can be used to limit the number
138 | of functions that are reported. Any function whose stack trace contains a function in one of the whitelisted packages will be included.
139 |
140 | The `visualization` directory contains some utilities for visualizing the output of this profiler.
141 |
142 | ### JVM And System CPU Load Profiler: `CPULoadProfiler`
143 |
144 | This profiler will record the JVM's and the overall system's CPU load, if the JVM is capable of providing this information.
145 |
146 | Assuming you use the default prefix of `statsd-jvm-profiler`, the JVM CPU load metrics will be under `statsd-jvm-profiler.cpu.jvm`,
147 | and the System CPU load wil be under `statsd-jvm-profiler.cpu.system`.
148 |
149 | The reported metrics will be percentages in the range of [0, 100] with 1 decimal precision.
150 |
151 | CPU load metrics are sampled and reported once every 10 seconds.
152 |
153 | Important notes:
154 | * This Profiler is not enabled by default. To enable use the argument `profilers=CPULoadProfiler`
155 | * This Profiler relies on Sun/Oracle-specific JVM implementations that offer a JMX bean that might not be available in other JVMs.
156 | Even if you are using the right JVM, there's no guarantee this JMX bean will remain there in the future.
157 | * The minimum required JVM version that offers support for this is for Java 7.
158 | * See [com.sun.management.OperatingSystemMXBean](https://docs.oracle.com/javase/7/docs/jre/api/management/extension/com/sun/management/OperatingSystemMXBean.html#getProcessCpuLoad())
159 | for more information.
160 | * If the JVM doesn't support the required operations, the metrics above won't be reported at all.
161 |
162 | ## Dynamic Loading of Agent
163 |
164 | 1. Make sure you have the `tools.jar` available in your classpath during compilation and runtime. This JAR is usually found in the JAVA_HOME directory under the `/lib` folder for Oracle Java installations.
165 | 2. Make sure the `jvm-profiler` JAR is available during runtime.
166 | 3. During your application boostrap process, do the following:
167 |
168 | ```scala
169 | val jarPath: String = s"$ABSOLUTE_PATH_TO/com.etsy.statsd-jvm-profiler-$VERSION.jar"
170 | val agentArgs: String = s"server=$SERVER,port=$PORT"
171 | attachJvmAgent(jarPath, agentArgs)
172 |
173 | def attachJvmAgent(profilerJarPath: String, agentArgs: String): Unit = {
174 | val nameOfRunningVM: String = java.lang.management.ManagementFactory.getRuntimeMXBean.getName
175 | val p: Integer = nameOfRunningVM.indexOf('@')
176 | val pid: String = nameOfRunningVM.substring(0, p)
177 |
178 | try {
179 | val vm: com.sun.tools.attach.VirtualMachine = com.sun.tools.attach.VirtualMachine.attach(pid)
180 | vm.loadAgent(profilerJarPath, agentArgs)
181 | vm.detach()
182 | LOGGER.info("Dynamically loaded StatsD JVM Profiler Agent...");
183 | } catch {
184 | case e: Exception => LOGGER.warn(s"Could not dynamically load StatsD JVM Profiler Agent ($profilerJarPath)", e);
185 | }
186 | }
187 | ```
188 |
189 |
190 | ## Contributing
191 | Contributions are highly encouraged! Check out [the contribution guidlines](https://github.com/etsy/statsd-jvm-profiler/blob/master/CONTRIBUTING.md).
192 |
193 | Any ideas you have are welcome, but check out [some ideas](https://github.com/etsy/statsd-jvm-profiler/wiki/Contribution-Ideas) for contributions.
194 |
--------------------------------------------------------------------------------
/visualization/influxdb-dashboard/public/css/bootstrap-theme.min.css:
--------------------------------------------------------------------------------
1 | /*!
2 | * Bootstrap v3.3.5 (http://getbootstrap.com)
3 | * Copyright 2011-2015 Twitter, Inc.
4 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
5 | */.btn-danger,.btn-default,.btn-info,.btn-primary,.btn-success,.btn-warning{text-shadow:0 -1px 0 rgba(0,0,0,.2);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 1px rgba(0,0,0,.075)}.btn-danger.active,.btn-danger:active,.btn-default.active,.btn-default:active,.btn-info.active,.btn-info:active,.btn-primary.active,.btn-primary:active,.btn-success.active,.btn-success:active,.btn-warning.active,.btn-warning:active{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn-danger.disabled,.btn-danger[disabled],.btn-default.disabled,.btn-default[disabled],.btn-info.disabled,.btn-info[disabled],.btn-primary.disabled,.btn-primary[disabled],.btn-success.disabled,.btn-success[disabled],.btn-warning.disabled,.btn-warning[disabled],fieldset[disabled] .btn-danger,fieldset[disabled] .btn-default,fieldset[disabled] .btn-info,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-success,fieldset[disabled] .btn-warning{-webkit-box-shadow:none;box-shadow:none}.btn-danger .badge,.btn-default .badge,.btn-info .badge,.btn-primary .badge,.btn-success .badge,.btn-warning .badge{text-shadow:none}.btn.active,.btn:active{background-image:none}.btn-default{text-shadow:0 1px 0 #fff;background-image:-webkit-linear-gradient(top,#fff 0,#e0e0e0 100%);background-image:-o-linear-gradient(top,#fff 0,#e0e0e0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fff),to(#e0e0e0));background-image:linear-gradient(to bottom,#fff 0,#e0e0e0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe0e0e0', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#dbdbdb;border-color:#ccc}.btn-default:focus,.btn-default:hover{background-color:#e0e0e0;background-position:0 -15px}.btn-default.active,.btn-default:active{background-color:#e0e0e0;border-color:#dbdbdb}.btn-default.disabled,.btn-default.disabled.active,.btn-default.disabled.focus,.btn-default.disabled:active,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled],.btn-default[disabled].active,.btn-default[disabled].focus,.btn-default[disabled]:active,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default,fieldset[disabled] .btn-default.active,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:active,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#e0e0e0;background-image:none}.btn-primary{background-image:-webkit-linear-gradient(top,#337ab7 0,#265a88 100%);background-image:-o-linear-gradient(top,#337ab7 0,#265a88 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#265a88));background-image:linear-gradient(to bottom,#337ab7 0,#265a88 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff265a88', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#245580}.btn-primary:focus,.btn-primary:hover{background-color:#265a88;background-position:0 -15px}.btn-primary.active,.btn-primary:active{background-color:#265a88;border-color:#245580}.btn-primary.disabled,.btn-primary.disabled.active,.btn-primary.disabled.focus,.btn-primary.disabled:active,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled],.btn-primary[disabled].active,.btn-primary[disabled].focus,.btn-primary[disabled]:active,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-primary.active,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:active,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#265a88;background-image:none}.btn-success{background-image:-webkit-linear-gradient(top,#5cb85c 0,#419641 100%);background-image:-o-linear-gradient(top,#5cb85c 0,#419641 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5cb85c),to(#419641));background-image:linear-gradient(to bottom,#5cb85c 0,#419641 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff419641', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#3e8f3e}.btn-success:focus,.btn-success:hover{background-color:#419641;background-position:0 -15px}.btn-success.active,.btn-success:active{background-color:#419641;border-color:#3e8f3e}.btn-success.disabled,.btn-success.disabled.active,.btn-success.disabled.focus,.btn-success.disabled:active,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled],.btn-success[disabled].active,.btn-success[disabled].focus,.btn-success[disabled]:active,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success,fieldset[disabled] .btn-success.active,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:active,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#419641;background-image:none}.btn-info{background-image:-webkit-linear-gradient(top,#5bc0de 0,#2aabd2 100%);background-image:-o-linear-gradient(top,#5bc0de 0,#2aabd2 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5bc0de),to(#2aabd2));background-image:linear-gradient(to bottom,#5bc0de 0,#2aabd2 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2aabd2', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#28a4c9}.btn-info:focus,.btn-info:hover{background-color:#2aabd2;background-position:0 -15px}.btn-info.active,.btn-info:active{background-color:#2aabd2;border-color:#28a4c9}.btn-info.disabled,.btn-info.disabled.active,.btn-info.disabled.focus,.btn-info.disabled:active,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled],.btn-info[disabled].active,.btn-info[disabled].focus,.btn-info[disabled]:active,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info,fieldset[disabled] .btn-info.active,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:active,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#2aabd2;background-image:none}.btn-warning{background-image:-webkit-linear-gradient(top,#f0ad4e 0,#eb9316 100%);background-image:-o-linear-gradient(top,#f0ad4e 0,#eb9316 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f0ad4e),to(#eb9316));background-image:linear-gradient(to bottom,#f0ad4e 0,#eb9316 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffeb9316', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#e38d13}.btn-warning:focus,.btn-warning:hover{background-color:#eb9316;background-position:0 -15px}.btn-warning.active,.btn-warning:active{background-color:#eb9316;border-color:#e38d13}.btn-warning.disabled,.btn-warning.disabled.active,.btn-warning.disabled.focus,.btn-warning.disabled:active,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled],.btn-warning[disabled].active,.btn-warning[disabled].focus,.btn-warning[disabled]:active,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning,fieldset[disabled] .btn-warning.active,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:active,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#eb9316;background-image:none}.btn-danger{background-image:-webkit-linear-gradient(top,#d9534f 0,#c12e2a 100%);background-image:-o-linear-gradient(top,#d9534f 0,#c12e2a 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9534f),to(#c12e2a));background-image:linear-gradient(to bottom,#d9534f 0,#c12e2a 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc12e2a', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#b92c28}.btn-danger:focus,.btn-danger:hover{background-color:#c12e2a;background-position:0 -15px}.btn-danger.active,.btn-danger:active{background-color:#c12e2a;border-color:#b92c28}.btn-danger.disabled,.btn-danger.disabled.active,.btn-danger.disabled.focus,.btn-danger.disabled:active,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled],.btn-danger[disabled].active,.btn-danger[disabled].focus,.btn-danger[disabled]:active,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger,fieldset[disabled] .btn-danger.active,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:active,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#c12e2a;background-image:none}.img-thumbnail,.thumbnail{-webkit-box-shadow:0 1px 2px rgba(0,0,0,.075);box-shadow:0 1px 2px rgba(0,0,0,.075)}.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{background-color:#e8e8e8;background-image:-webkit-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f5f5f5),to(#e8e8e8));background-image:linear-gradient(to bottom,#f5f5f5 0,#e8e8e8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);background-repeat:repeat-x}.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{background-color:#2e6da4;background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}.navbar-default{background-image:-webkit-linear-gradient(top,#fff 0,#f8f8f8 100%);background-image:-o-linear-gradient(top,#fff 0,#f8f8f8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fff),to(#f8f8f8));background-image:linear-gradient(to bottom,#fff 0,#f8f8f8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff8f8f8', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-radius:4px;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 5px rgba(0,0,0,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 5px rgba(0,0,0,.075)}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.open>a{background-image:-webkit-linear-gradient(top,#dbdbdb 0,#e2e2e2 100%);background-image:-o-linear-gradient(top,#dbdbdb 0,#e2e2e2 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dbdbdb),to(#e2e2e2));background-image:linear-gradient(to bottom,#dbdbdb 0,#e2e2e2 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdbdbdb', endColorstr='#ffe2e2e2', GradientType=0);background-repeat:repeat-x;-webkit-box-shadow:inset 0 3px 9px rgba(0,0,0,.075);box-shadow:inset 0 3px 9px rgba(0,0,0,.075)}.navbar-brand,.navbar-nav>li>a{text-shadow:0 1px 0 rgba(255,255,255,.25)}.navbar-inverse{background-image:-webkit-linear-gradient(top,#3c3c3c 0,#222 100%);background-image:-o-linear-gradient(top,#3c3c3c 0,#222 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#3c3c3c),to(#222));background-image:linear-gradient(to bottom,#3c3c3c 0,#222 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff3c3c3c', endColorstr='#ff222222', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-radius:4px}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.open>a{background-image:-webkit-linear-gradient(top,#080808 0,#0f0f0f 100%);background-image:-o-linear-gradient(top,#080808 0,#0f0f0f 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#080808),to(#0f0f0f));background-image:linear-gradient(to bottom,#080808 0,#0f0f0f 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff080808', endColorstr='#ff0f0f0f', GradientType=0);background-repeat:repeat-x;-webkit-box-shadow:inset 0 3px 9px rgba(0,0,0,.25);box-shadow:inset 0 3px 9px rgba(0,0,0,.25)}.navbar-inverse .navbar-brand,.navbar-inverse .navbar-nav>li>a{text-shadow:0 -1px 0 rgba(0,0,0,.25)}.navbar-fixed-bottom,.navbar-fixed-top,.navbar-static-top{border-radius:0}@media (max-width:767px){.navbar .navbar-nav .open .dropdown-menu>.active>a,.navbar .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}}.alert{text-shadow:0 1px 0 rgba(255,255,255,.2);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.25),0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 1px 0 rgba(255,255,255,.25),0 1px 2px rgba(0,0,0,.05)}.alert-success{background-image:-webkit-linear-gradient(top,#dff0d8 0,#c8e5bc 100%);background-image:-o-linear-gradient(top,#dff0d8 0,#c8e5bc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dff0d8),to(#c8e5bc));background-image:linear-gradient(to bottom,#dff0d8 0,#c8e5bc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffc8e5bc', GradientType=0);background-repeat:repeat-x;border-color:#b2dba1}.alert-info{background-image:-webkit-linear-gradient(top,#d9edf7 0,#b9def0 100%);background-image:-o-linear-gradient(top,#d9edf7 0,#b9def0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9edf7),to(#b9def0));background-image:linear-gradient(to bottom,#d9edf7 0,#b9def0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffb9def0', GradientType=0);background-repeat:repeat-x;border-color:#9acfea}.alert-warning{background-image:-webkit-linear-gradient(top,#fcf8e3 0,#f8efc0 100%);background-image:-o-linear-gradient(top,#fcf8e3 0,#f8efc0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fcf8e3),to(#f8efc0));background-image:linear-gradient(to bottom,#fcf8e3 0,#f8efc0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fff8efc0', GradientType=0);background-repeat:repeat-x;border-color:#f5e79e}.alert-danger{background-image:-webkit-linear-gradient(top,#f2dede 0,#e7c3c3 100%);background-image:-o-linear-gradient(top,#f2dede 0,#e7c3c3 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f2dede),to(#e7c3c3));background-image:linear-gradient(to bottom,#f2dede 0,#e7c3c3 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffe7c3c3', GradientType=0);background-repeat:repeat-x;border-color:#dca7a7}.progress{background-image:-webkit-linear-gradient(top,#ebebeb 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#ebebeb 0,#f5f5f5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#ebebeb),to(#f5f5f5));background-image:linear-gradient(to bottom,#ebebeb 0,#f5f5f5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffebebeb', endColorstr='#fff5f5f5', GradientType=0);background-repeat:repeat-x}.progress-bar{background-image:-webkit-linear-gradient(top,#337ab7 0,#286090 100%);background-image:-o-linear-gradient(top,#337ab7 0,#286090 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#286090));background-image:linear-gradient(to bottom,#337ab7 0,#286090 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff286090', GradientType=0);background-repeat:repeat-x}.progress-bar-success{background-image:-webkit-linear-gradient(top,#5cb85c 0,#449d44 100%);background-image:-o-linear-gradient(top,#5cb85c 0,#449d44 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5cb85c),to(#449d44));background-image:linear-gradient(to bottom,#5cb85c 0,#449d44 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff449d44', GradientType=0);background-repeat:repeat-x}.progress-bar-info{background-image:-webkit-linear-gradient(top,#5bc0de 0,#31b0d5 100%);background-image:-o-linear-gradient(top,#5bc0de 0,#31b0d5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5bc0de),to(#31b0d5));background-image:linear-gradient(to bottom,#5bc0de 0,#31b0d5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff31b0d5', GradientType=0);background-repeat:repeat-x}.progress-bar-warning{background-image:-webkit-linear-gradient(top,#f0ad4e 0,#ec971f 100%);background-image:-o-linear-gradient(top,#f0ad4e 0,#ec971f 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f0ad4e),to(#ec971f));background-image:linear-gradient(to bottom,#f0ad4e 0,#ec971f 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffec971f', GradientType=0);background-repeat:repeat-x}.progress-bar-danger{background-image:-webkit-linear-gradient(top,#d9534f 0,#c9302c 100%);background-image:-o-linear-gradient(top,#d9534f 0,#c9302c 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9534f),to(#c9302c));background-image:linear-gradient(to bottom,#d9534f 0,#c9302c 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc9302c', GradientType=0);background-repeat:repeat-x}.progress-bar-striped{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.list-group{border-radius:4px;-webkit-box-shadow:0 1px 2px rgba(0,0,0,.075);box-shadow:0 1px 2px rgba(0,0,0,.075)}.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{text-shadow:0 -1px 0 #286090;background-image:-webkit-linear-gradient(top,#337ab7 0,#2b669a 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2b669a 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2b669a));background-image:linear-gradient(to bottom,#337ab7 0,#2b669a 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2b669a', GradientType=0);background-repeat:repeat-x;border-color:#2b669a}.list-group-item.active .badge,.list-group-item.active:focus .badge,.list-group-item.active:hover .badge{text-shadow:none}.panel{-webkit-box-shadow:0 1px 2px rgba(0,0,0,.05);box-shadow:0 1px 2px rgba(0,0,0,.05)}.panel-default>.panel-heading{background-image:-webkit-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f5f5f5),to(#e8e8e8));background-image:linear-gradient(to bottom,#f5f5f5 0,#e8e8e8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);background-repeat:repeat-x}.panel-primary>.panel-heading{background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}.panel-success>.panel-heading{background-image:-webkit-linear-gradient(top,#dff0d8 0,#d0e9c6 100%);background-image:-o-linear-gradient(top,#dff0d8 0,#d0e9c6 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dff0d8),to(#d0e9c6));background-image:linear-gradient(to bottom,#dff0d8 0,#d0e9c6 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffd0e9c6', GradientType=0);background-repeat:repeat-x}.panel-info>.panel-heading{background-image:-webkit-linear-gradient(top,#d9edf7 0,#c4e3f3 100%);background-image:-o-linear-gradient(top,#d9edf7 0,#c4e3f3 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9edf7),to(#c4e3f3));background-image:linear-gradient(to bottom,#d9edf7 0,#c4e3f3 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffc4e3f3', GradientType=0);background-repeat:repeat-x}.panel-warning>.panel-heading{background-image:-webkit-linear-gradient(top,#fcf8e3 0,#faf2cc 100%);background-image:-o-linear-gradient(top,#fcf8e3 0,#faf2cc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fcf8e3),to(#faf2cc));background-image:linear-gradient(to bottom,#fcf8e3 0,#faf2cc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fffaf2cc', GradientType=0);background-repeat:repeat-x}.panel-danger>.panel-heading{background-image:-webkit-linear-gradient(top,#f2dede 0,#ebcccc 100%);background-image:-o-linear-gradient(top,#f2dede 0,#ebcccc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f2dede),to(#ebcccc));background-image:linear-gradient(to bottom,#f2dede 0,#ebcccc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffebcccc', GradientType=0);background-repeat:repeat-x}.well{background-image:-webkit-linear-gradient(top,#e8e8e8 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#e8e8e8 0,#f5f5f5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#e8e8e8),to(#f5f5f5));background-image:linear-gradient(to bottom,#e8e8e8 0,#f5f5f5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffe8e8e8', endColorstr='#fff5f5f5', GradientType=0);background-repeat:repeat-x;border-color:#dcdcdc;-webkit-box-shadow:inset 0 1px 3px rgba(0,0,0,.05),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 3px rgba(0,0,0,.05),0 1px 0 rgba(255,255,255,.1)}
--------------------------------------------------------------------------------