├── .gitignore ├── .travis.yml ├── LICENSE ├── NOTICE ├── README.md ├── epl-v10.txt ├── metricDetails.md ├── pom.xml ├── src ├── main │ ├── java │ │ └── com │ │ │ └── uber │ │ │ └── profiling │ │ │ ├── Agent.java │ │ │ ├── AgentImpl.java │ │ │ ├── AgentThreadFactory.java │ │ │ ├── ArgumentUtils.java │ │ │ ├── Arguments.java │ │ │ ├── ConfigProvider.java │ │ │ ├── Profiler.java │ │ │ ├── ProfilerGroup.java │ │ │ ├── ProfilerRunner.java │ │ │ ├── Reporter.java │ │ │ ├── ShutdownHookRunner.java │ │ │ ├── YamlConfigProvider.java │ │ │ ├── examples │ │ │ └── HelloWorldApplication.java │ │ │ ├── profilers │ │ │ ├── Constants.java │ │ │ ├── CpuAndMemoryProfiler.java │ │ │ ├── IOProfiler.java │ │ │ ├── MethodArgumentCollector.java │ │ │ ├── MethodArgumentProfiler.java │ │ │ ├── MethodDurationCollector.java │ │ │ ├── MethodDurationProfiler.java │ │ │ ├── ProcessInfoProfiler.java │ │ │ ├── ProfilerBase.java │ │ │ ├── StacktraceCollectorProfiler.java │ │ │ ├── StacktraceReporterProfiler.java │ │ │ └── ThreadInfoProfiler.java │ │ │ ├── reporters │ │ │ ├── ConsoleOutputReporter.java │ │ │ ├── FileOutputReporter.java │ │ │ ├── FlattenKafkaOutputReporter.java │ │ │ ├── GraphiteOutputReporter.java │ │ │ └── KafkaOutputReporter.java │ │ │ ├── transformers │ │ │ ├── JavaAgentFileTransformer.java │ │ │ └── MethodProfilerStaticProxy.java │ │ │ └── util │ │ │ ├── AgentLogger.java │ │ │ ├── ClassAndMethod.java │ │ │ ├── ClassAndMethodFilter.java │ │ │ ├── ClassAndMethodLongMetricBuffer.java │ │ │ ├── ClassAndMethodMetricKey.java │ │ │ ├── ClassMethodArgument.java │ │ │ ├── ClassMethodArgumentFilter.java │ │ │ ├── ClassMethodArgumentMetricBuffer.java │ │ │ ├── DummyConfigProvider.java │ │ │ ├── ErrorLogReporter.java │ │ │ ├── ExponentialBackoffRetryPolicy.java │ │ │ ├── Histogram.java │ │ │ ├── IOUtils.java │ │ │ ├── JsonUtils.java │ │ │ ├── NetworkUtils.java │ │ │ ├── NoopConfigProvider.java │ │ │ ├── ProcFileUtils.java │ │ │ ├── ProcessUtils.java │ │ │ ├── ReflectionUtils.java │ │ │ ├── SparkAppCmdInfo.java │ │ │ ├── SparkUtils.java │ │ │ ├── Stacktrace.java │ │ │ ├── StacktraceMetricBuffer.java │ │ │ └── StringUtils.java │ ├── java_datadog │ │ └── com │ │ │ └── uber │ │ │ └── profiling │ │ │ └── reporters │ │ │ └── DatadogOutputReporter.java │ ├── java_graphite │ │ └── graphite.yaml │ ├── java_influxdb │ │ └── com │ │ │ └── uber │ │ │ └── profiling │ │ │ └── reporters │ │ │ └── InfluxDBOutputReporter.java │ ├── java_jdbc │ │ └── com │ │ │ └── uber │ │ │ └── profiling │ │ │ └── reporters │ │ │ ├── CpuAndMemoryProfilerMetric.java │ │ │ ├── CpuAndMemoryProfilerMetricDao.java │ │ │ ├── JdbcOutputReporter.java │ │ │ └── util │ │ │ ├── BaseJdbcDao.java │ │ │ ├── BeanUtils.java │ │ │ ├── DateTimeUtils.java │ │ │ ├── DbConnectionProvider.java │ │ │ ├── JdbcUtils.java │ │ │ ├── SingleTableJdbcWriter.java │ │ │ └── SqlUtils.java │ └── java_redis │ │ └── com │ │ └── uber │ │ └── profiling │ │ └── RedisOutputReporter.java └── test │ ├── java │ └── com │ │ └── uber │ │ └── profiling │ │ ├── AgentITCase.java │ │ ├── AgentThreadFactoryTest.java │ │ ├── ArgumentsTest.java │ │ ├── ProfilerRunnableTest.java │ │ ├── YamlConfigProviderTest.java │ │ ├── profilers │ │ ├── CpuAndMemoryProfilerTest.java │ │ ├── MethodArgumentProfilerTest.java │ │ ├── MethodDurationProfilerTest.java │ │ ├── ProcessInfoProfilerTest.java │ │ ├── StacktraceCollectorProfilerTest.java │ │ ├── StacktraceReporterProfilerTest.java │ │ └── ThreadInfoProfilerTest.java │ │ ├── reporters │ │ ├── ConsoleOutputReporterTest.java │ │ └── GraphiteOutputReporterTest.java │ │ ├── transformers │ │ └── MethodProfilerStaticProxyTest.java │ │ └── util │ │ ├── ClassAndMethodFilterTest.java │ │ ├── ClassAndMethodMetricBufferTest.java │ │ ├── ClassMethodArgmentMetricBufferTest.java │ │ ├── ClassMethodArgumentFilterTest.java │ │ ├── IOUtilsTest.java │ │ ├── ProcFileUtilsTest.java │ │ ├── ReflectionUtilsTest.java │ │ ├── SparkUtilsTest.java │ │ ├── StacktraceMetricBufferTest.java │ │ └── StringUtilsTest.java │ └── java_jdbc │ └── com │ └── uber │ └── profiling │ └── reporters │ ├── CpuAndMemoryProfilerMetricDaoTest.java │ └── JdbcOutputReporterTest.java └── stackcollapse.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | *.iml 4 | 5 | dependency-reduced-pom.xml 6 | 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | dist: trusty 3 | jdk: 4 | - oraclejdk8 5 | - openjdk8 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Uber JVM Profiler 2 | Copyright (c) 2018 Uber Technologies, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | Uber JVM Profiler depends on the following library: 17 | 18 | statsd-jvm-profiler 19 | Copyright (c) 2014-2016 Etsy 20 | 21 | Permission is hereby granted, free of charge, to any person obtaining a copy 22 | of this software and associated documentation files (the "Software"), to deal 23 | in the Software without restriction, including without limitation the rights 24 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 25 | copies of the Software, and to permit persons to whom the Software is 26 | furnished to do so, subject to the following conditions: 27 | 28 | The above copyright notice and this permission notice shall be included in all 29 | copies or substantial portions of the Software. 30 | 31 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 32 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 33 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 34 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 35 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 36 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 37 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Uber JVM Profiler 2 | Copyright (c) 2018 Uber Technologies, Inc. 3 | 4 | I. Included Software 5 | 6 | This product may include software developed at 7 | The Apache Software Foundation (https://www.apache.org/). 8 | Licensed under the Apache License. 9 | 10 | This product may include software developed by 11 | etsy (https://github.com/etsy/statsd-jvm-profiler). 12 | Licensed under the MIT License. 13 | 14 | This product depends on software developed by 15 | Javassist (http://jboss-javassist.github.io/javassist/). 16 | Licensed under the Apache License. 17 | 18 | This product depends on software developed by 19 | Javassist (http://jboss-javassist.github.io/javassist/). 20 | Licensed under the Apache License. 21 | 22 | This product depends on software developed by 23 | FasterXML (https://github.com/FasterXML). 24 | Licensed under the Apache License. 25 | 26 | This product depends on software developed by 27 | JUnit (http://junit.org/junit4). 28 | Licensed under the Eclipse Public License. 29 | 30 | This product depends on software developed by 31 | Andrey Somov (https://bitbucket.org/asomov/snakeyaml). 32 | Licensed under the Apache License. 33 | 34 | II. License Summary 35 | - Apache License 2.0 -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/Agent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import java.lang.instrument.Instrumentation; 20 | 21 | public final class Agent { 22 | 23 | private static AgentImpl agentImpl = new AgentImpl(); 24 | 25 | private Agent() { 26 | } 27 | 28 | public static void agentmain(final String args, final Instrumentation instrumentation) { 29 | premain(args, instrumentation); 30 | } 31 | 32 | public static void premain(final String args, final Instrumentation instrumentation) { 33 | System.out.println("Java Agent " + AgentImpl.VERSION + " premain args: " + args); 34 | 35 | Arguments arguments = Arguments.parseArgs(args); 36 | arguments.runConfigProvider(); 37 | agentImpl.run(arguments, instrumentation, null); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/AgentThreadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import java.util.concurrent.Executors; 20 | import java.util.concurrent.ThreadFactory; 21 | 22 | public class AgentThreadFactory implements ThreadFactory { 23 | public static final String NAME_PREFIX = "uber_java_agent"; 24 | 25 | private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); 26 | 27 | @Override 28 | public Thread newThread(Runnable r) { 29 | Thread thread = defaultThreadFactory.newThread(r); 30 | if (thread != null) { 31 | thread.setDaemon(true); 32 | thread.setName(String.format("%s-%s", NAME_PREFIX, thread.getName())); 33 | } 34 | 35 | return thread; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/ArgumentUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | public class ArgumentUtils { 24 | 25 | public static boolean needToUpdateArg(String argValue) { 26 | return argValue != null && !argValue.isEmpty(); 27 | } 28 | 29 | public static boolean needToUpdateRollingArg(String enableRolling) { 30 | return enableRolling != null && !enableRolling.isEmpty() && Boolean.parseBoolean(enableRolling); 31 | } 32 | 33 | public static String getArgumentSingleValue(Map> parsedArgs, String argName) { 34 | List list = parsedArgs.get(argName); 35 | if (list == null) { 36 | return null; 37 | } 38 | 39 | if (list.isEmpty()) { 40 | return ""; 41 | } 42 | 43 | return list.get(list.size() - 1); 44 | } 45 | 46 | public static List getArgumentMultiValues(Map> parsedArgs, String argName) { 47 | List list = parsedArgs.get(argName); 48 | if (list == null) { 49 | return new ArrayList<>(); 50 | } 51 | return list; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/ConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | public interface ConfigProvider { 23 | /*** 24 | * Returns a map containing multiple child maps. Each child map contains a group of key/value pairs 25 | * for a specific jvm profiler user (identified by tag value when start the profiler). The child map 26 | * for empty tag value contains the default configuration for all profilers. 27 | * @return 28 | */ 29 | Map>> getConfig(); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/Profiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | public interface Profiler { 20 | long getIntervalMillis(); 21 | 22 | void setReporter(Reporter reporter); 23 | 24 | void profile(); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/ProfilerGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class ProfilerGroup { 23 | private List oneTimeProfilers; 24 | private List periodicProfilers; 25 | 26 | public ProfilerGroup(List oneTimeProfilers, List periodicProfilers) { 27 | this.oneTimeProfilers = new ArrayList<>(oneTimeProfilers); 28 | this.periodicProfilers = new ArrayList<>(periodicProfilers); 29 | } 30 | 31 | public List getOneTimeProfilers() { 32 | return oneTimeProfilers; 33 | } 34 | 35 | public List getPeriodicProfilers() { 36 | return periodicProfilers; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/ProfilerRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import com.uber.profiling.util.AgentLogger; 20 | 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | public class ProfilerRunner implements Runnable { 24 | private static final AgentLogger logger = AgentLogger.getLogger(ProfilerRunner.class.getName()); 25 | 26 | private static final int MAX_ERROR_COUNT_TO_LOG = 100; 27 | 28 | private final Profiler profiler; 29 | private final AtomicLong errorCounter = new AtomicLong(0); 30 | 31 | public ProfilerRunner(Profiler profiler) { 32 | this.profiler = profiler; 33 | } 34 | 35 | @Override 36 | public void run() { 37 | try { 38 | profiler.profile(); 39 | } catch (Throwable e) { 40 | long errorCountValue = errorCounter.incrementAndGet(); 41 | if (errorCountValue <= MAX_ERROR_COUNT_TO_LOG) { 42 | logger.warn("Failed to run profile: " + profiler, e); 43 | } else { 44 | e.printStackTrace(); 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/Reporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | public interface Reporter { 23 | 24 | default void updateArguments(Map> parsedArgs) { 25 | } 26 | 27 | void report(String profilerName, Map metrics); 28 | 29 | void close(); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/ShutdownHookRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import com.uber.profiling.util.AgentLogger; 20 | 21 | import java.util.ArrayList; 22 | import java.util.Collection; 23 | import java.util.Date; 24 | import java.util.List; 25 | 26 | public class ShutdownHookRunner implements Runnable { 27 | private static final AgentLogger logger = AgentLogger.getLogger(ShutdownHookRunner.class.getName()); 28 | 29 | private List profilers; 30 | private List reporters; 31 | private List closeables; 32 | 33 | public ShutdownHookRunner(Collection profilers, Collection reporters, Collection objectsToCloseOnShutdown) { 34 | this.profilers = profilers == null ? new ArrayList<>() : new ArrayList<>(profilers); 35 | this.reporters = reporters == null ? new ArrayList<>() : new ArrayList<>(reporters); 36 | this.closeables = objectsToCloseOnShutdown == null ? new ArrayList<>() : new ArrayList<>(objectsToCloseOnShutdown); 37 | } 38 | 39 | @Override 40 | public void run() { 41 | logShutdownMessage("Running java agent shutdown"); 42 | 43 | for (Profiler profiler : profilers) { 44 | try { 45 | logShutdownMessage("Running periodic profiler (last run): " + profiler); 46 | profiler.profile(); 47 | logShutdownMessage("Ran periodic profiler (last run): " + profiler); 48 | } catch (Throwable ex) { 49 | logger.warn("Failed to run periodic profiler (last run): " + profiler, ex); 50 | } 51 | } 52 | 53 | for (Reporter r : reporters) { 54 | try { 55 | logShutdownMessage("Closing reporter " + r); 56 | r.close(); 57 | logShutdownMessage("Closed reporter " + r); 58 | } catch (Throwable ex) { 59 | logger.warn("Failed to close reporter " + r + ", " + new Date() + ", " + System.currentTimeMillis(), ex); 60 | } 61 | } 62 | 63 | for (AutoCloseable closeable : closeables) { 64 | // Do not use logger.warn here because the logger may depend on error log reporter which will be already closed here. 65 | // So we use logShutdownMessage (System.out.println) to print out logs. 66 | try { 67 | logShutdownMessage("Closing object " + closeable); 68 | closeable.close(); 69 | logShutdownMessage("Closed object " + closeable ); 70 | } catch (Throwable ex) { 71 | logShutdownMessage("Failed to close object " + closeable); 72 | ex.printStackTrace(); 73 | } 74 | } 75 | } 76 | 77 | private void logShutdownMessage(String msg) { 78 | // Sometime spark log in console output seems not fully collected, thus log to error output as well to make sure 79 | // we capture this shutdown hook execution. This is to help debug some issue when shutdown hook seems not executed. 80 | String log = System.currentTimeMillis() + " " + msg; 81 | System.out.println(log); 82 | System.err.println(log); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/examples/HelloWorldApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.examples; 18 | 19 | import java.util.Random; 20 | import java.util.concurrent.atomic.AtomicLong; 21 | 22 | public class HelloWorldApplication { 23 | 24 | /** 25 | * This application could be used to test the java agent. 26 | * For example, you could run it with following argument: 27 | * -javaagent:target/uber-java-agent-0.0.1-jar-with-dependencies.jar=reporter=com.uber.profiling.reporters.ConsoleOutputReporter,tag=tag1,metricInterval=10000,durationProfiling=com.uber.profiling.examples.HelloWorldApplication.* 28 | */ 29 | public static void main(String[] args) throws Throwable { 30 | long totalRunningMillis = 1 * 60 * 1000; 31 | long sleepMillis = 1000; 32 | 33 | if (args.length >= 1) { 34 | totalRunningMillis = Long.parseLong(args[0]); 35 | } 36 | 37 | if (args.length >= 2) { 38 | sleepMillis = Long.parseLong(args[1]); 39 | } 40 | 41 | long startMillis = System.currentTimeMillis(); 42 | long lastPrintMillis = 0; 43 | 44 | Random random = new Random(); 45 | 46 | while (System.currentTimeMillis() - startMillis < totalRunningMillis) { 47 | if (System.currentTimeMillis() - lastPrintMillis >= 10000) { 48 | System.out.println("Hello World " + System.currentTimeMillis()); 49 | lastPrintMillis = System.currentTimeMillis(); 50 | } 51 | 52 | sleepMillis += random.nextInt(100); 53 | sleepMillis -= random.nextInt(100); 54 | 55 | privateSleepMethod(sleepMillis); 56 | 57 | AtomicLong atomicLong = new AtomicLong(sleepMillis); 58 | publicSleepMethod(atomicLong); 59 | } 60 | } 61 | 62 | private static void privateSleepMethod(long millis) { 63 | try { 64 | Thread.sleep(millis); 65 | } catch (InterruptedException e) { 66 | throw new RuntimeException(e); 67 | } 68 | } 69 | 70 | public static void publicSleepMethod(AtomicLong millis) { 71 | try { 72 | Thread.sleep(millis.get()); 73 | } catch (InterruptedException e) { 74 | throw new RuntimeException(e); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | public class Constants { 20 | public final static long DEFAULT_METRIC_INTERVAL = 60000; 21 | 22 | public static final int MAX_STRING_LENGTH = 800000; 23 | 24 | public final static String EXECUTOR_ROLE = "executor"; 25 | public final static String DRIVER_ROLE = "driver"; 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/IOProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Profiler; 20 | import com.uber.profiling.Reporter; 21 | import com.uber.profiling.util.ProcFileUtils; 22 | 23 | import java.util.HashMap; 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class IOProfiler extends ProfilerBase implements Profiler { 28 | public final static String PROFILER_NAME = "IO"; 29 | 30 | private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL; 31 | 32 | private Reporter reporter; 33 | 34 | public IOProfiler(Reporter reporter) { 35 | setReporter(reporter); 36 | } 37 | 38 | @Override 39 | public long getIntervalMillis() { 40 | return intervalMillis; 41 | } 42 | 43 | public void setIntervalMillis(long intervalMillis) { 44 | this.intervalMillis = intervalMillis; 45 | } 46 | 47 | @Override 48 | public void setReporter(Reporter reporter) { 49 | this.reporter = reporter; 50 | } 51 | 52 | @Override 53 | public synchronized void profile() { 54 | // See http://man7.org/linux/man-pages/man5/proc.5.html for details about /proc/[pid]/io 55 | Map procMap = ProcFileUtils.getProcIO(); 56 | Long rchar = ProcFileUtils.getBytesValue(procMap, "rchar"); 57 | Long wchar = ProcFileUtils.getBytesValue(procMap, "wchar"); 58 | Long read_bytes = ProcFileUtils.getBytesValue(procMap, "read_bytes"); 59 | Long write_bytes = ProcFileUtils.getBytesValue(procMap, "write_bytes"); 60 | 61 | List> cpuTime = ProcFileUtils.getProcStatCpuTime(); 62 | 63 | Map map = new HashMap(); 64 | 65 | map.put("epochMillis", System.currentTimeMillis()); 66 | map.put("name", getProcessName()); 67 | map.put("host", getHostName()); 68 | map.put("processUuid", getProcessUuid()); 69 | map.put("appId", getAppId()); 70 | 71 | if (getTag() != null) { 72 | map.put("tag", getTag()); 73 | } 74 | 75 | if (getCluster() != null) { 76 | map.put("cluster", getCluster()); 77 | } 78 | 79 | if (getRole() != null) { 80 | map.put("role", getRole()); 81 | } 82 | 83 | Map selfMap = new HashMap(); 84 | map.put("self", selfMap); 85 | 86 | Map ioMap = new HashMap(); 87 | selfMap.put("io", ioMap); 88 | 89 | ioMap.put("rchar", rchar); 90 | ioMap.put("wchar", wchar); 91 | ioMap.put("read_bytes", read_bytes); 92 | ioMap.put("write_bytes", write_bytes); 93 | 94 | map.put("stat", cpuTime); 95 | 96 | if (reporter != null) { 97 | reporter.report(PROFILER_NAME, map); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/MethodArgumentCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.util.ClassMethodArgumentMetricBuffer; 20 | 21 | public class MethodArgumentCollector { 22 | private ClassMethodArgumentMetricBuffer buffer; 23 | 24 | public MethodArgumentCollector(ClassMethodArgumentMetricBuffer buffer) { 25 | this.buffer = buffer; 26 | } 27 | 28 | public void collectMetric(String className, String methodName, String argument) { 29 | if (argument == null) { 30 | argument = ""; 31 | } 32 | 33 | if (argument.length() > Constants.MAX_STRING_LENGTH) { 34 | argument = argument.substring(0, Constants.MAX_STRING_LENGTH); 35 | } 36 | 37 | buffer.appendValue(className, methodName, argument); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/MethodArgumentProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Profiler; 20 | import com.uber.profiling.Reporter; 21 | import com.uber.profiling.reporters.ConsoleOutputReporter; 22 | import com.uber.profiling.util.ClassAndMethodMetricKey; 23 | import com.uber.profiling.util.ClassMethodArgumentMetricBuffer; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | import java.util.concurrent.atomic.AtomicLong; 28 | 29 | public class MethodArgumentProfiler extends ProfilerBase implements Profiler { 30 | public static final String PROFILER_NAME = "MethodArgument"; 31 | 32 | private ClassMethodArgumentMetricBuffer buffer; 33 | 34 | private Reporter reporter = new ConsoleOutputReporter(); 35 | 36 | private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL; 37 | 38 | public MethodArgumentProfiler(ClassMethodArgumentMetricBuffer buffer, Reporter reporter) { 39 | this.buffer = buffer; 40 | this.reporter = reporter; 41 | } 42 | 43 | @Override 44 | public long getIntervalMillis() { 45 | return intervalMillis; 46 | } 47 | 48 | public void setIntervalMillis(long intervalMillis) { 49 | this.intervalMillis = intervalMillis; 50 | } 51 | 52 | public void setReporter(Reporter reporter) { 53 | this.reporter = reporter; 54 | } 55 | 56 | @Override 57 | public void profile() { 58 | if (buffer == null) { 59 | return; 60 | } 61 | 62 | if (reporter == null) { 63 | return; 64 | } 65 | 66 | Map metrics = buffer.reset(); 67 | 68 | long epochMillis = System.currentTimeMillis(); 69 | 70 | for (Map.Entry entry : metrics.entrySet()) { 71 | Map commonMap = new HashMap<>(); 72 | 73 | commonMap.put("epochMillis", epochMillis); 74 | commonMap.put("processName", getProcessName()); 75 | commonMap.put("host", getHostName()); 76 | commonMap.put("processUuid", getProcessUuid()); 77 | commonMap.put("appId", getAppId()); 78 | 79 | commonMap.put("className", entry.getKey().getClassName()); 80 | commonMap.put("methodName", entry.getKey().getMethodName()); 81 | 82 | if (getTag() != null) { 83 | commonMap.put("tag", getTag()); 84 | } 85 | 86 | if (getCluster() != null) { 87 | commonMap.put("cluster", getCluster()); 88 | } 89 | 90 | if (getRole() != null) { 91 | commonMap.put("role", getRole()); 92 | } 93 | 94 | { 95 | Map metricMap = new HashMap<>(commonMap); 96 | metricMap.put("metricName", entry.getKey().getMetricName()); 97 | metricMap.put("metricValue", (double) entry.getValue().get()); 98 | reporter.report(PROFILER_NAME, metricMap); 99 | } 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/MethodDurationCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.util.ClassAndMethodLongMetricBuffer; 20 | 21 | public class MethodDurationCollector { 22 | private ClassAndMethodLongMetricBuffer buffer; 23 | 24 | public MethodDurationCollector(ClassAndMethodLongMetricBuffer buffer) { 25 | this.buffer = buffer; 26 | } 27 | 28 | public void collectLongMetric(String className, String methodName, String metricName, long metricValue) { 29 | buffer.appendValue(className, methodName, metricName, metricValue); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/MethodDurationProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Profiler; 20 | import com.uber.profiling.Reporter; 21 | import com.uber.profiling.reporters.ConsoleOutputReporter; 22 | import com.uber.profiling.util.ClassAndMethodLongMetricBuffer; 23 | import com.uber.profiling.util.ClassAndMethodMetricKey; 24 | import com.uber.profiling.util.Histogram; 25 | 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | 29 | public class MethodDurationProfiler extends ProfilerBase implements Profiler { 30 | public static final String PROFILER_NAME = "MethodDuration"; 31 | 32 | private ClassAndMethodLongMetricBuffer buffer; 33 | 34 | private Reporter reporter = new ConsoleOutputReporter(); 35 | 36 | private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL; 37 | 38 | public MethodDurationProfiler(ClassAndMethodLongMetricBuffer buffer, Reporter reporter) { 39 | this.buffer = buffer; 40 | this.reporter = reporter; 41 | } 42 | 43 | @Override 44 | public long getIntervalMillis() { 45 | return intervalMillis; 46 | } 47 | 48 | public void setIntervalMillis(long intervalMillis) { 49 | this.intervalMillis = intervalMillis; 50 | } 51 | 52 | public void setReporter(Reporter reporter) { 53 | this.reporter = reporter; 54 | } 55 | 56 | @Override 57 | public void profile() { 58 | if (buffer == null) { 59 | return; 60 | } 61 | 62 | if (reporter == null) { 63 | return; 64 | } 65 | 66 | Map metrics = buffer.reset(); 67 | 68 | long epochMillis = System.currentTimeMillis(); 69 | 70 | for (Map.Entry entry : metrics.entrySet()) { 71 | Map commonMap = new HashMap<>(); 72 | 73 | commonMap.put("epochMillis", epochMillis); 74 | commonMap.put("processName", getProcessName()); 75 | commonMap.put("host", getHostName()); 76 | commonMap.put("processUuid", getProcessUuid()); 77 | commonMap.put("appId", getAppId()); 78 | 79 | commonMap.put("className", entry.getKey().getClassName()); 80 | commonMap.put("methodName", entry.getKey().getMethodName()); 81 | 82 | if (getTag() != null) { 83 | commonMap.put("tag", getTag()); 84 | } 85 | 86 | if (getCluster() != null) { 87 | commonMap.put("cluster", getCluster()); 88 | } 89 | 90 | if (getRole() != null) { 91 | commonMap.put("role", getRole()); 92 | } 93 | 94 | { 95 | Map metricMap = new HashMap<>(commonMap); 96 | metricMap.put("metricName", entry.getKey().getMetricName() + ".count"); 97 | metricMap.put("metricValue", (double) entry.getValue().getCount()); 98 | reporter.report(PROFILER_NAME, metricMap); 99 | } 100 | { 101 | Map metricMap = new HashMap<>(commonMap); 102 | metricMap.put("metricName", entry.getKey().getMetricName() + ".sum"); 103 | metricMap.put("metricValue", (double) entry.getValue().getSum()); 104 | reporter.report(PROFILER_NAME, metricMap); 105 | } 106 | { 107 | Map metricMap = new HashMap<>(commonMap); 108 | metricMap.put("metricName", entry.getKey().getMetricName() + ".min"); 109 | metricMap.put("metricValue", (double) entry.getValue().getMin()); 110 | reporter.report(PROFILER_NAME, metricMap); 111 | } 112 | { 113 | Map metricMap = new HashMap<>(commonMap); 114 | metricMap.put("metricName", entry.getKey().getMetricName() + ".max"); 115 | metricMap.put("metricValue", (double) entry.getValue().getMax()); 116 | reporter.report(PROFILER_NAME, metricMap); 117 | } 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/ProfilerBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.util.NetworkUtils; 20 | import com.uber.profiling.util.ProcFileUtils; 21 | import com.uber.profiling.util.ProcessUtils; 22 | import com.uber.profiling.util.SparkUtils; 23 | 24 | import java.util.UUID; 25 | 26 | public class ProfilerBase { 27 | private String tag = null; 28 | private String cluster = null; 29 | private String hostName = null; 30 | private String processName = null; 31 | private String processUuid = UUID.randomUUID().toString(); 32 | 33 | private String jobId = null; 34 | private String appId = null; 35 | 36 | private String role = null; 37 | 38 | public ProfilerBase() { 39 | setHostName(NetworkUtils.getLocalHostName()); 40 | setProcessName(ProcessUtils.getCurrentProcessName()); 41 | } 42 | 43 | public String getTag() { 44 | return tag; 45 | } 46 | 47 | public void setTag(String tag) { 48 | this.tag = tag; 49 | } 50 | 51 | public String getCluster() { 52 | return cluster; 53 | } 54 | 55 | public void setCluster(String cluster) { 56 | this.cluster = cluster; 57 | } 58 | 59 | public String getHostName() { 60 | return hostName; 61 | } 62 | 63 | public void setHostName(String hostName) { 64 | this.hostName = hostName; 65 | } 66 | 67 | public String getProcessName() { 68 | return processName; 69 | } 70 | 71 | public void setProcessName(String processName) { 72 | this.processName = processName; 73 | } 74 | 75 | public String getProcessUuid() { 76 | return processUuid; 77 | } 78 | 79 | public void setProcessUuid(String processUuid) { 80 | this.processUuid = processUuid; 81 | } 82 | 83 | public String getJobId() { 84 | return jobId; 85 | } 86 | 87 | public void setJobId(String jobId) { 88 | this.jobId = jobId; 89 | } 90 | 91 | public String getAppId() { 92 | if (appId != null && !appId.isEmpty()) { 93 | return appId; 94 | } 95 | 96 | appId = SparkUtils.getSparkEnvAppId(); 97 | return appId; 98 | } 99 | 100 | public void setAppId(String appId) { 101 | this.appId = appId; 102 | } 103 | 104 | public void setRole(String role) { 105 | this.role = role; 106 | } 107 | 108 | public String getRole() { 109 | if (role != null && !role.isEmpty()) { 110 | return role; 111 | } 112 | 113 | String cmdline = ProcFileUtils.getCmdline(); 114 | role = SparkUtils.probeRole(cmdline); 115 | return role; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/StacktraceCollectorProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Profiler; 20 | import com.uber.profiling.Reporter; 21 | import com.uber.profiling.util.ClassAndMethod; 22 | import com.uber.profiling.util.Stacktrace; 23 | import com.uber.profiling.util.StacktraceMetricBuffer; 24 | 25 | import java.lang.management.ManagementFactory; 26 | import java.lang.management.ThreadInfo; 27 | import java.lang.management.ThreadMXBean; 28 | import java.util.ArrayList; 29 | import java.util.List; 30 | 31 | /** 32 | * This class collects stacktraces by getting thread dump via JMX, and stores the stacktraces into the given buffer. 33 | */ 34 | public class StacktraceCollectorProfiler implements Profiler { 35 | private long intervalMillis; 36 | private StacktraceMetricBuffer buffer; 37 | private String ignoreThreadNamePrefix = ""; 38 | private int maxStringLength = Constants.MAX_STRING_LENGTH; 39 | private ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); 40 | 41 | public StacktraceCollectorProfiler(StacktraceMetricBuffer buffer, String ignoreThreadNamePrefix) { 42 | this(buffer, ignoreThreadNamePrefix, Constants.MAX_STRING_LENGTH); 43 | } 44 | 45 | public StacktraceCollectorProfiler(StacktraceMetricBuffer buffer, String ignoreThreadNamePrefix, int maxStringLength) { 46 | this.buffer = buffer; 47 | this.ignoreThreadNamePrefix = ignoreThreadNamePrefix == null ? "" : ignoreThreadNamePrefix; 48 | this.maxStringLength = maxStringLength; 49 | } 50 | 51 | public void setIntervalMillis(long intervalMillis) { 52 | this.intervalMillis = intervalMillis; 53 | } 54 | 55 | @Override 56 | public long getIntervalMillis() { 57 | return this.intervalMillis; 58 | } 59 | 60 | @Override 61 | public void setReporter(Reporter reporter) { 62 | } 63 | 64 | @Override 65 | public void profile() { 66 | ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(false, false); 67 | if (threadInfos == null) { 68 | return; 69 | } 70 | 71 | for (ThreadInfo threadInfo : threadInfos) { 72 | String threadName = threadInfo.getThreadName(); 73 | if (threadName == null) { 74 | threadName = ""; 75 | } 76 | 77 | if (!ignoreThreadNamePrefix.isEmpty() 78 | && threadName.startsWith(ignoreThreadNamePrefix)) { 79 | continue; 80 | } 81 | 82 | StackTraceElement[] stackTraceElements = threadInfo.getStackTrace(); 83 | 84 | Stacktrace stacktrace = new Stacktrace(); 85 | stacktrace.setThreadName(threadName); 86 | stacktrace.setThreadState(String.valueOf(threadInfo.getThreadState())); 87 | 88 | // Start from bottom of the stacktrace so we could trim top method (most nested method) if the size is too large 89 | int totalLength = 0; 90 | List stack = new ArrayList<>(stackTraceElements.length); 91 | for (int i = stackTraceElements.length - 1; i >= 0; i--) { 92 | StackTraceElement stackTraceElement = stackTraceElements[i]; 93 | String className = String.valueOf(stackTraceElement.getClassName()); 94 | String methodName = String.valueOf(stackTraceElement.getMethodName()); 95 | stack.add(new ClassAndMethod(className, methodName)); 96 | 97 | totalLength += className.length() + methodName.length(); 98 | 99 | if (totalLength >= maxStringLength) { 100 | stack.add(new ClassAndMethod("_stack_", "_trimmed_")); 101 | break; 102 | } 103 | } 104 | 105 | // Reverse the stack so the top method (most nested method) is the first element of the array 106 | ClassAndMethod[] classAndMethodArray = new ClassAndMethod[stack.size()]; 107 | for (int i = 0; i < stack.size(); i++) { 108 | classAndMethodArray[classAndMethodArray.length - 1 - i] = stack.get(i); 109 | } 110 | 111 | stacktrace.setStack(classAndMethodArray); 112 | 113 | buffer.appendValue(stacktrace); 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/StacktraceReporterProfiler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Profiler; 20 | import com.uber.profiling.Reporter; 21 | import com.uber.profiling.reporters.ConsoleOutputReporter; 22 | import com.uber.profiling.util.ClassAndMethod; 23 | import com.uber.profiling.util.Stacktrace; 24 | import com.uber.profiling.util.StacktraceMetricBuffer; 25 | 26 | import java.util.ArrayList; 27 | import java.util.HashMap; 28 | import java.util.List; 29 | import java.util.Map; 30 | import java.util.concurrent.atomic.AtomicLong; 31 | 32 | /** 33 | * This class reads the stacktraces from the given buffer and send out via given reporter. 34 | */ 35 | public class StacktraceReporterProfiler extends ProfilerBase implements Profiler { 36 | public static final String PROFILER_NAME = "Stacktrace"; 37 | 38 | private StacktraceMetricBuffer buffer; 39 | 40 | private Reporter reporter = new ConsoleOutputReporter(); 41 | 42 | private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL; 43 | 44 | public StacktraceReporterProfiler(StacktraceMetricBuffer buffer, Reporter reporter) { 45 | this.buffer = buffer; 46 | this.reporter = reporter; 47 | } 48 | 49 | @Override 50 | public long getIntervalMillis() { 51 | return intervalMillis; 52 | } 53 | 54 | public void setIntervalMillis(long intervalMillis) { 55 | this.intervalMillis = intervalMillis; 56 | } 57 | 58 | public void setReporter(Reporter reporter) { 59 | this.reporter = reporter; 60 | } 61 | 62 | @Override 63 | public void profile() { 64 | if (buffer == null) { 65 | return; 66 | } 67 | 68 | if (reporter == null) { 69 | return; 70 | } 71 | 72 | long startEpoch = buffer.getLastResetMillis(); 73 | 74 | Map metrics = buffer.reset(); 75 | 76 | long endEpoch = buffer.getLastResetMillis(); 77 | 78 | for (Map.Entry entry : metrics.entrySet()) { 79 | Map map = new HashMap<>(); 80 | 81 | map.put("startEpoch", startEpoch); 82 | map.put("endEpoch", endEpoch); 83 | 84 | map.put("host", getHostName()); 85 | map.put("name", getProcessName()); 86 | map.put("processUuid", getProcessUuid()); 87 | map.put("appId", getAppId()); 88 | 89 | if (getTag() != null) { 90 | map.put("tag", getTag()); 91 | } 92 | 93 | if (getCluster() != null) { 94 | map.put("cluster", getCluster()); 95 | } 96 | 97 | if (getRole() != null) { 98 | map.put("role", getRole()); 99 | } 100 | 101 | Stacktrace stacktrace = entry.getKey(); 102 | 103 | map.put("threadName", stacktrace.getThreadName()); 104 | map.put("threadState", stacktrace.getThreadState()); 105 | 106 | ClassAndMethod[] classAndMethodArray = stacktrace.getStack(); 107 | if (classAndMethodArray!= null) { 108 | List stackArray = new ArrayList<>(classAndMethodArray.length); 109 | for (int i = 0; i < classAndMethodArray.length; i++) { 110 | ClassAndMethod classAndMethod = classAndMethodArray[i]; 111 | stackArray.add(classAndMethod.getClassName() + "." + classAndMethod.getMethodName()); 112 | } 113 | map.put("stacktrace", stackArray); 114 | } 115 | 116 | map.put("count", entry.getValue().get()); 117 | 118 | reporter.report(PROFILER_NAME, map); 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/profilers/ThreadInfoProfiler.java: -------------------------------------------------------------------------------- 1 | package com.uber.profiling.profilers; 2 | 3 | import com.uber.profiling.Profiler; 4 | import com.uber.profiling.Reporter; 5 | import com.uber.profiling.util.AgentLogger; 6 | 7 | import java.lang.management.ManagementFactory; 8 | import java.lang.management.ThreadMXBean; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | /** 13 | * ThreadInfoProfiler is used to Collects the Thread Related Metrics. 14 | */ 15 | public class ThreadInfoProfiler extends ProfilerBase implements Profiler { 16 | public final static String PROFILER_NAME = "ThreadInfo"; 17 | public final static AgentLogger logger = AgentLogger.getLogger(ThreadInfoProfiler.class.getName()); 18 | private long intervalMillis = Constants.DEFAULT_METRIC_INTERVAL; 19 | 20 | private ThreadMXBean threadMXBean; 21 | private long previousTotalStartedThreadCount = 0L; // to keep track of Total Thread. 22 | 23 | private Reporter reporter; 24 | 25 | public ThreadInfoProfiler(Reporter reporter) { 26 | setReporter(reporter); 27 | init(); 28 | } 29 | 30 | private void init() { 31 | try { 32 | this.threadMXBean = ManagementFactory.getThreadMXBean(); 33 | } 34 | catch (Throwable ex) { 35 | logger.warn("Failed to get Thread MXBean", ex); 36 | } 37 | 38 | } 39 | 40 | public void setIntervalMillis(long intervalMillis) { 41 | this.intervalMillis = intervalMillis; 42 | } 43 | 44 | @Override 45 | public long getIntervalMillis() { 46 | return intervalMillis; 47 | } 48 | 49 | @Override 50 | public void setReporter(Reporter reporter) { 51 | this.reporter = reporter; 52 | } 53 | 54 | @Override 55 | public void profile() { 56 | 57 | long totalStartedThreadCount = 0L; // total Thread created so far since JVm Launch. 58 | int liveThreadCount = 0; // Number of thread which are currently active. 59 | int peakThreadCount = 0; // the peak live thread count since the Java virtual machine started or peak was reset 60 | long newThreadCount = 0; // Number of new thread created since last time time the metrics was created. 61 | // This is a Derived metrics from previous data point. 62 | if (threadMXBean != null) { 63 | liveThreadCount = threadMXBean.getThreadCount(); 64 | peakThreadCount = threadMXBean.getPeakThreadCount(); 65 | totalStartedThreadCount = threadMXBean.getTotalStartedThreadCount(); 66 | newThreadCount = totalStartedThreadCount - this.previousTotalStartedThreadCount; 67 | this.previousTotalStartedThreadCount = totalStartedThreadCount; 68 | } 69 | 70 | Map map = new HashMap<>(); 71 | 72 | map.put("epochMillis", System.currentTimeMillis()); 73 | map.put("name", getProcessName()); 74 | map.put("host", getHostName()); 75 | map.put("processUuid", getProcessUuid()); 76 | map.put("appId", getAppId()); 77 | 78 | if (getTag() != null) { 79 | map.put("tag", getTag()); 80 | } 81 | 82 | if (getCluster() != null) { 83 | map.put("cluster", getCluster()); 84 | } 85 | 86 | if (getRole() != null) { 87 | map.put("role", getRole()); 88 | } 89 | 90 | map.put("totalStartedThreadCount", totalStartedThreadCount); 91 | map.put("newThreadCount", newThreadCount); 92 | map.put("liveThreadCount", liveThreadCount); 93 | map.put("peakThreadCount", peakThreadCount); 94 | 95 | if (reporter != null) { 96 | reporter.report(PROFILER_NAME, map); 97 | } 98 | } 99 | 100 | } -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/reporters/ConsoleOutputReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import com.uber.profiling.Reporter; 20 | import com.uber.profiling.util.JsonUtils; 21 | 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | public class ConsoleOutputReporter implements Reporter { 26 | @Override 27 | public void report(String profilerName, Map metrics) { 28 | System.out.println(String.format("ConsoleOutputReporter - %s: %s", profilerName, JsonUtils.serialize(metrics))); 29 | } 30 | 31 | @Override 32 | public void close() { 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/reporters/FileOutputReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import com.uber.profiling.ArgumentUtils; 20 | import com.uber.profiling.Reporter; 21 | import com.uber.profiling.util.AgentLogger; 22 | import com.uber.profiling.util.JsonUtils; 23 | import com.uber.profiling.util.StringUtils; 24 | 25 | import java.io.File; 26 | import java.io.FileWriter; 27 | import java.io.IOException; 28 | import java.nio.file.Files; 29 | import java.nio.file.Path; 30 | import java.nio.file.Paths; 31 | import java.util.List; 32 | import java.util.Map; 33 | 34 | public class FileOutputReporter implements Reporter { 35 | public final static String ARG_OUTPUT_DIR = "outputDir"; 36 | public final static String ARG_ENABLE_ROLLING = "enableRolling"; 37 | public final static String ARG_ROLLING_SIZE = "rollingSize"; 38 | 39 | private static final AgentLogger logger = AgentLogger.getLogger(FileOutputReporter.class.getName()); 40 | 41 | private String directory; 42 | private volatile boolean closed = false; 43 | private boolean enableRolling = false; 44 | private Long rollingSize = StringUtils.getBytesValueOrNull("128mb"); 45 | 46 | public FileOutputReporter() { 47 | } 48 | 49 | public String getDirectory() { 50 | return directory; 51 | } 52 | 53 | // This method sets the output directory. By default, this reporter will create a temporary directory 54 | // and use it as output directory. User could set the output director if want to use another one. But 55 | // the output directory can only be set at mose once. Setting it again will throw exception. 56 | public void setDirectory(String directory) { 57 | synchronized (this) { 58 | if (this.directory == null || this.directory.isEmpty()) { 59 | Path path = Paths.get(directory); 60 | try { 61 | if (!Files.exists(path)) { 62 | Files.createDirectory(path); 63 | } 64 | } catch (IOException e) { 65 | throw new RuntimeException("Failed to create directory: " + path, e); 66 | } 67 | 68 | this.directory = directory; 69 | } else { 70 | throw new RuntimeException(String.format("Cannot set directory to %s because it is already has value %s", directory, this.directory)); 71 | } 72 | } 73 | } 74 | 75 | @Override 76 | public void updateArguments(Map> parsedArgs) { 77 | String outputDir = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_OUTPUT_DIR); 78 | String enableRolling = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_ENABLE_ROLLING); 79 | String rollingSize = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_ROLLING_SIZE); 80 | if (ArgumentUtils.needToUpdateArg(outputDir)) { 81 | setDirectory(outputDir); 82 | logger.info("Got argument value for outputDir: " + outputDir); 83 | } 84 | 85 | if (ArgumentUtils.needToUpdateRollingArg(enableRolling)) { 86 | setAndCheckRollingArg(rollingSize); 87 | logger.info("Got argument value for rollingSize: " + rollingSize); 88 | } 89 | } 90 | 91 | private void setAndCheckRollingArg(String rollingSize) { 92 | synchronized (this) { 93 | this.enableRolling = true; 94 | if (rollingSize != null && !rollingSize.isEmpty()) { 95 | this.rollingSize = StringUtils.getBytesValueOrNull(rollingSize); 96 | } else { 97 | logger.info("Rolling size is default value: 128mb"); 98 | } 99 | } 100 | } 101 | 102 | @Override 103 | public synchronized void report(String profilerName, Map metrics) { 104 | if (closed) { 105 | logger.info("Report already closed, do not report metrics"); 106 | return; 107 | } 108 | ensureFile(); 109 | try (FileWriter writer = createFileWriter(profilerName, needRolling(profilerName))) { 110 | writer.write(JsonUtils.serialize(metrics)); 111 | writer.write(System.lineSeparator()); 112 | writer.flush(); 113 | } catch (IOException e) { 114 | e.printStackTrace(); 115 | } 116 | } 117 | 118 | private boolean needRolling(String profilerName) { 119 | synchronized (this) { 120 | File file = new File(Paths.get(directory, profilerName + ".json").toString()); 121 | return enableRolling && file.length() > rollingSize; 122 | } 123 | } 124 | 125 | @Override 126 | public synchronized void close() { 127 | logger.info("close file output reporter"); 128 | closed = true; 129 | } 130 | 131 | private void ensureFile() { 132 | synchronized (this) { 133 | if (directory == null || directory.isEmpty()) { 134 | try { 135 | directory = Files.createTempDirectory("jvm_profiler_").toString(); 136 | } catch (IOException e) { 137 | e.printStackTrace(); 138 | } 139 | } 140 | } 141 | } 142 | 143 | private FileWriter createFileWriter(String profilerName, boolean needRolling) { 144 | String path = Paths.get(directory, profilerName + ".json").toString(); 145 | try { 146 | return new FileWriter(path, !needRolling); 147 | } catch (IOException e) { 148 | throw new RuntimeException("Failed to create file writer: " + path, e); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/reporters/FlattenKafkaOutputReporter.java: -------------------------------------------------------------------------------- 1 | package com.uber.profiling.reporters; 2 | 3 | import com.uber.profiling.ArgumentUtils; 4 | import com.uber.profiling.util.AgentLogger; 5 | import org.apache.commons.lang3.StringUtils; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | /** 11 | * FlattenKafkaOutputReporter: a custom KafkaOutputReporter 12 | * This reporter will transform metrics as key=value output format before sending to kafka 13 | * It would be useful in case you want to report the metrics into kafka -> consume those metrics in somewhere & then writing to InfluxDB indirectly. 14 | * Use in case your JVM machine vs the influxdb instance are hosted in different cluster or different network. 15 | */ 16 | public class FlattenKafkaOutputReporter extends KafkaOutputReporter{ 17 | public final static String ARG_TOPIC_LOWERCASE = "topicLowercase"; 18 | 19 | private static final AgentLogger logger = AgentLogger.getLogger(FlattenKafkaOutputReporter.class.getName()); 20 | 21 | private Boolean topicLowercase; 22 | public Boolean getTopicLowercase() { 23 | return topicLowercase; 24 | } 25 | public void setTopicLowercase(Boolean topicLowercase) {this.topicLowercase = topicLowercase;} 26 | 27 | public FlattenKafkaOutputReporter(String brokerList, boolean syncMode, String topicPrefix, Boolean topicLowercase) { 28 | super(brokerList, syncMode, topicPrefix); 29 | this.topicLowercase = topicLowercase; 30 | } 31 | 32 | @Override 33 | public void updateArguments(Map> parsedArgs) { 34 | super.updateArguments(parsedArgs); 35 | String argValue = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_TOPIC_LOWERCASE); 36 | if (ArgumentUtils.needToUpdateArg(argValue)) { 37 | setTopicLowercase(Boolean.parseBoolean(argValue)); 38 | logger.info("Got argument value for topicLowercase: " + topicLowercase); 39 | } 40 | } 41 | 42 | @Override 43 | public void report(String profilerName, Map metrics) { 44 | Map formattedMetrics = getFormattedMetrics(metrics); 45 | super.report(profilerName, formattedMetrics); 46 | } 47 | 48 | @Override 49 | public String getTopic(String profilerName) { 50 | String topic = super.getTopic(profilerName); 51 | if (topicLowercase) { 52 | return topic.toLowerCase(); 53 | } 54 | return topic; 55 | } 56 | 57 | // Format metrics in key=value (line protocol) 58 | private Map getFormattedMetrics(Map metrics) { 59 | Map formattedMetrics = new HashMap<>(); 60 | for (Map.Entry entry : metrics.entrySet()) { 61 | String key = entry.getKey(); 62 | Object value = entry.getValue(); 63 | if (value instanceof List) { 64 | List listValue = (List) value; 65 | if (!listValue.isEmpty() && listValue.get(0) instanceof String) { 66 | List metricList = (List) listValue; 67 | formattedMetrics.put(key, String.join(",", metricList)); 68 | } else if (!listValue.isEmpty() && listValue.get(0) instanceof Map) { 69 | List> metricList = (List>) listValue; 70 | int num = 1; 71 | for (Map metricMap : metricList) { 72 | String name = null; 73 | if(metricMap.containsKey("name") && metricMap.get("name") != null && metricMap.get("name") instanceof String){ 74 | name = (String) metricMap.get("name"); 75 | name = name.replaceAll("\\s", ""); 76 | } 77 | for (Map.Entry entry1 : metricMap.entrySet()) { 78 | if(StringUtils.isNotEmpty(name)){ 79 | formattedMetrics.put(key + "-" + name + "-" + entry1.getKey(), entry1.getValue()); 80 | }else{ 81 | formattedMetrics.put(key + "-" + entry1.getKey() + "-" + num, entry1.getValue()); 82 | } 83 | } 84 | num++; 85 | } 86 | } 87 | } else if (value instanceof Map) { 88 | Map metricMap = (Map) value; 89 | for (Map.Entry entry1 : metricMap.entrySet()) { 90 | String key1 = entry1.getKey(); 91 | Object value1 = entry1.getValue(); 92 | if (value1 instanceof Map) { 93 | Map value2 = (Map) value1; 94 | int num = 1; 95 | for (Map.Entry entry2 : value2.entrySet()) { 96 | formattedMetrics.put(key + "-" + key1 + "-" + entry2.getKey() + "-" + num, entry2.getValue()); 97 | } 98 | num++; 99 | } 100 | } 101 | } else { 102 | formattedMetrics.put(key, value); 103 | } 104 | } 105 | return formattedMetrics; 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/reporters/KafkaOutputReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import com.uber.profiling.Reporter; 20 | import com.uber.profiling.ArgumentUtils; 21 | import com.uber.profiling.util.AgentLogger; 22 | import com.uber.profiling.util.JsonUtils; 23 | import org.apache.kafka.clients.producer.KafkaProducer; 24 | import org.apache.kafka.clients.producer.Producer; 25 | import org.apache.kafka.clients.producer.ProducerRecord; 26 | import org.apache.kafka.clients.producer.RecordMetadata; 27 | 28 | import java.nio.charset.StandardCharsets; 29 | import java.util.List; 30 | import java.util.Map; 31 | import java.util.Properties; 32 | import java.util.concurrent.ConcurrentHashMap; 33 | import java.util.concurrent.ExecutionException; 34 | import java.util.concurrent.Future; 35 | 36 | public class KafkaOutputReporter implements Reporter { 37 | public final static String ARG_BROKER_LIST = "brokerList"; 38 | public final static String ARG_SYNC_MODE = "syncMode"; 39 | public final static String ARG_TOPIC_PREFIX = "topicPrefix"; 40 | 41 | private static final AgentLogger logger = AgentLogger.getLogger(KafkaOutputReporter.class.getName()); 42 | 43 | private String brokerList = "localhost:9092"; 44 | private boolean syncMode = false; 45 | 46 | private String topicPrefix; 47 | 48 | private ConcurrentHashMap profilerTopics = new ConcurrentHashMap<>(); 49 | 50 | private Producer producer; 51 | 52 | public KafkaOutputReporter() { 53 | } 54 | 55 | public KafkaOutputReporter(String brokerList, boolean syncMode, String topicPrefix) { 56 | this.brokerList = brokerList; 57 | this.syncMode = syncMode; 58 | this.topicPrefix = topicPrefix; 59 | } 60 | 61 | @Override 62 | public void updateArguments(Map> parsedArgs) { 63 | String argValue = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_BROKER_LIST); 64 | if (ArgumentUtils.needToUpdateArg(argValue)) { 65 | setBrokerList(argValue); 66 | logger.info("Got argument value for brokerList: " + brokerList); 67 | } 68 | 69 | argValue = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_SYNC_MODE); 70 | if (ArgumentUtils.needToUpdateArg(argValue)) { 71 | setSyncMode(Boolean.parseBoolean(argValue)); 72 | logger.info("Got argument value for syncMode: " + syncMode); 73 | } 74 | 75 | argValue = ArgumentUtils.getArgumentSingleValue(parsedArgs, ARG_TOPIC_PREFIX); 76 | if (ArgumentUtils.needToUpdateArg(argValue)) { 77 | setTopicPrefix(argValue); 78 | logger.info("Got argument value for topicPrefix: " + topicPrefix); 79 | } 80 | } 81 | 82 | @Override 83 | public void report(String profilerName, Map metrics) { 84 | ensureProducer(); 85 | 86 | String topicName = getTopic(profilerName); 87 | 88 | String str = JsonUtils.serialize(metrics); 89 | byte[] message = str.getBytes(StandardCharsets.UTF_8); 90 | 91 | Future future = producer.send( 92 | new ProducerRecord(topicName, message)); 93 | 94 | if (syncMode) { 95 | producer.flush(); 96 | try { 97 | future.get(); 98 | } catch (InterruptedException | ExecutionException e) { 99 | throw new RuntimeException(e); 100 | } 101 | } 102 | } 103 | 104 | @Override 105 | public void close() { 106 | synchronized (this) { 107 | if (producer == null) { 108 | return; 109 | } 110 | 111 | producer.flush(); 112 | producer.close(); 113 | 114 | producer = null; 115 | } 116 | } 117 | 118 | public String getBrokerList() { 119 | return brokerList; 120 | } 121 | 122 | public void setBrokerList(String brokerList) { 123 | this.brokerList = brokerList; 124 | } 125 | 126 | public boolean isSyncMode() { 127 | return syncMode; 128 | } 129 | 130 | public void setSyncMode(boolean syncMode) { 131 | this.syncMode = syncMode; 132 | } 133 | 134 | public void setTopic(String profilerName, String topicName) { 135 | profilerTopics.put(profilerName, topicName); 136 | } 137 | 138 | public String getTopicPrefix() { 139 | return topicPrefix; 140 | } 141 | 142 | public void setTopicPrefix(String topicPrefix) { 143 | this.topicPrefix = topicPrefix; 144 | } 145 | 146 | public String getTopic(String profilerName) { 147 | String topic = profilerTopics.getOrDefault(profilerName, null); 148 | if (topic == null || topic.isEmpty()) { 149 | topic = topicPrefix == null ? "" : topicPrefix; 150 | topic += profilerName; 151 | } 152 | return topic; 153 | } 154 | 155 | private void ensureProducer() { 156 | synchronized (this) { 157 | if (producer != null) { 158 | return; 159 | } 160 | 161 | Properties props = new Properties(); 162 | props.put("bootstrap.servers", brokerList); 163 | props.put("retries", 10); 164 | props.put("batch.size", 16384); 165 | props.put("linger.ms", 0); 166 | props.put("buffer.memory", 16384000); 167 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 168 | props.put("value.serializer", org.apache.kafka.common.serialization.ByteArraySerializer.class.getName()); 169 | props.put("client.id", "jvm-profilers"); 170 | 171 | if (syncMode) { 172 | props.put("acks", "all"); 173 | } 174 | 175 | producer = new KafkaProducer<>(props); 176 | } 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/transformers/MethodProfilerStaticProxy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.transformers; 18 | 19 | import com.uber.profiling.profilers.MethodArgumentCollector; 20 | import com.uber.profiling.profilers.MethodDurationCollector; 21 | 22 | public class MethodProfilerStaticProxy { 23 | private static MethodDurationCollector collectorSingleton; 24 | private static MethodArgumentCollector argumentCollectorSingleton; 25 | 26 | private MethodProfilerStaticProxy() { 27 | } 28 | 29 | public static void setCollector(MethodDurationCollector collector) { 30 | collectorSingleton = collector; 31 | } 32 | 33 | public static void setArgumentCollector(MethodArgumentCollector collector) { 34 | argumentCollectorSingleton = collector; 35 | } 36 | 37 | public static void collectMethodDuration(String className, String methodName, long metricValue) { 38 | if (collectorSingleton == null) { 39 | return; 40 | } 41 | 42 | try { 43 | collectorSingleton.collectLongMetric(className, methodName, "duration", metricValue); 44 | } catch (Throwable ex) { 45 | ex.printStackTrace(); 46 | } 47 | } 48 | 49 | public static void collectMethodArgument(String className, String methodName, int argIndex, Object argValue) { 50 | if (argumentCollectorSingleton == null) { 51 | return; 52 | } 53 | 54 | try { 55 | String argument = "arg." + argIndex + "." + String.valueOf(argValue); 56 | argumentCollectorSingleton.collectMetric(className, methodName, argument); 57 | } catch (Throwable ex) { 58 | ex.printStackTrace(); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/AgentLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.apache.commons.lang3.exception.ExceptionUtils; 20 | 21 | public class AgentLogger { 22 | private static boolean debug = false; 23 | private static ErrorLogReporter errorLogReporter; 24 | 25 | private String prefix; 26 | 27 | public static AgentLogger getLogger(String name) { 28 | return new AgentLogger(name); 29 | } 30 | 31 | public static void setDebug(boolean enableDebug) { 32 | debug = enableDebug; 33 | } 34 | 35 | public static void setErrorLogReporter(ErrorLogReporter reporter) { 36 | errorLogReporter = reporter; 37 | } 38 | 39 | public AgentLogger(String name) { 40 | if (name == null) { 41 | this.prefix = ""; 42 | } else { 43 | this.prefix = name + ": "; 44 | } 45 | } 46 | 47 | public void log(String msg) { 48 | info(msg); 49 | } 50 | 51 | public void info(String msg) { 52 | System.out.println(System.currentTimeMillis() + " " + prefix + msg); 53 | } 54 | 55 | public void debug(String msg) { 56 | if (AgentLogger.debug) { 57 | info(msg); 58 | } 59 | } 60 | 61 | public void warn(String msg) { 62 | try { 63 | System.out.println("[WARNING] " + System.currentTimeMillis() + " " + prefix + msg); 64 | 65 | if (AgentLogger.errorLogReporter != null) { 66 | AgentLogger.errorLogReporter.report(msg, null); 67 | } 68 | } catch (Throwable ex) { 69 | ex.printStackTrace(); 70 | } 71 | } 72 | 73 | public void warn(String msg, Throwable ex) { 74 | try { 75 | System.out.println("[WARNING] " + System.currentTimeMillis() + " " + prefix + msg + " " + ExceptionUtils.getStackTrace(ex)); 76 | 77 | if (AgentLogger.errorLogReporter != null) { 78 | AgentLogger.errorLogReporter.report(msg, ex); 79 | } 80 | } catch (Throwable executionException) { 81 | executionException.printStackTrace(); 82 | } 83 | } 84 | 85 | // Handle log specially when shutdown, since we should not depend on other kafka to log these messages 86 | public void logShutdownMessage(String msg) { 87 | // Sometime spark log in console output seems not fully collected, thus log to error output as well to make sure 88 | // we capture this shutdown hook execution. This is to help debug some issue when shutdown hook seems not executed. 89 | String log = System.currentTimeMillis() + " " + prefix + msg; 90 | System.out.println(log); 91 | System.err.println(log); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassAndMethod.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | public class ClassAndMethod { 20 | private final String className; 21 | private final String methodName; 22 | 23 | public ClassAndMethod(String className, String methodName) { 24 | if (className == null) { 25 | throw new NullPointerException("className"); 26 | } 27 | 28 | if (methodName == null) { 29 | throw new NullPointerException("methodName"); 30 | } 31 | 32 | this.className = className; 33 | this.methodName = methodName; 34 | } 35 | 36 | public String getClassName() { 37 | return className; 38 | } 39 | 40 | public String getMethodName() { 41 | return methodName; 42 | } 43 | 44 | @Override 45 | public boolean equals(Object o) { 46 | if (this == o) return true; 47 | if (o == null || getClass() != o.getClass()) return false; 48 | 49 | ClassAndMethod that = (ClassAndMethod) o; 50 | 51 | if (className != null ? !className.equals(that.className) : that.className != null) return false; 52 | return methodName != null ? methodName.equals(that.methodName) : that.methodName == null; 53 | } 54 | 55 | @Override 56 | public int hashCode() { 57 | int result = className != null ? className.hashCode() : 0; 58 | result = 31 * result + (methodName != null ? methodName.hashCode() : 0); 59 | return result; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return className + '.' + methodName; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassAndMethodFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.List; 20 | 21 | public class ClassAndMethodFilter { 22 | private static final String METHOD_NAME_WILDCARD = "*"; 23 | 24 | private ClassAndMethod[] classAndMethods = new ClassAndMethod[0]; 25 | 26 | public ClassAndMethodFilter(List classMethodNames) { 27 | if (classMethodNames != null) { 28 | this.classAndMethods = new ClassAndMethod[classMethodNames.size()]; 29 | for (int i = 0; i < classMethodNames.size(); i++) { 30 | this.classAndMethods[i] = classMethodNames.get(i); 31 | } 32 | } 33 | } 34 | 35 | public boolean isEmpty() { 36 | return classAndMethods.length == 0; 37 | } 38 | 39 | public boolean matchClass(String className) { 40 | for (ClassAndMethod classAndMethod : classAndMethods) { 41 | if (className.startsWith(classAndMethod.getClassName())) { 42 | return true; 43 | } 44 | } 45 | return false; 46 | } 47 | 48 | public boolean matchMethod(String className, String methodName) { 49 | for (ClassAndMethod classAndMethod : classAndMethods) { 50 | if (className.startsWith(classAndMethod.getClassName())) { 51 | if (METHOD_NAME_WILDCARD.equals(classAndMethod.getMethodName()) 52 | || methodName.equals(classAndMethod.getMethodName())) { 53 | return true; 54 | } 55 | } 56 | } 57 | return false; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassAndMethodLongMetricBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.Map; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | 22 | /** 23 | * ClassAndMethodLongMetricBuffer is a buffer to store metrics. It is thread safe for appendValue. 24 | * The reset method will create a new empty internal buffer and return the old one. 25 | */ 26 | public class ClassAndMethodLongMetricBuffer { 27 | private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); 28 | 29 | public void appendValue(String className, String methodName, String metricName, long value) { 30 | ClassAndMethodMetricKey methodMetricKey = new ClassAndMethodMetricKey(className, methodName, metricName); 31 | Histogram histogram = metrics.computeIfAbsent(methodMetricKey, key -> new Histogram()); 32 | histogram.appendValue(value); 33 | } 34 | 35 | public Map reset() { 36 | ConcurrentHashMap oldCopy = metrics; 37 | metrics = new ConcurrentHashMap<>(); 38 | return oldCopy; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassAndMethodMetricKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | public class ClassAndMethodMetricKey { 20 | private final String className; 21 | private final String methodName; 22 | private final String metricName; 23 | 24 | public ClassAndMethodMetricKey(String className, String methodName, String metricName) { 25 | this.className = className; 26 | this.methodName = methodName; 27 | this.metricName = metricName; 28 | } 29 | 30 | public String getClassName() { 31 | return className; 32 | } 33 | 34 | public String getMethodName() { 35 | return methodName; 36 | } 37 | 38 | public String getMetricName() { 39 | return metricName; 40 | } 41 | 42 | @Override 43 | public boolean equals(Object o) { 44 | if (this == o) return true; 45 | if (o == null || getClass() != o.getClass()) return false; 46 | 47 | ClassAndMethodMetricKey that = (ClassAndMethodMetricKey) o; 48 | 49 | if (className != null ? !className.equals(that.className) : that.className != null) return false; 50 | if (methodName != null ? !methodName.equals(that.methodName) : that.methodName != null) return false; 51 | return metricName != null ? metricName.equals(that.metricName) : that.metricName == null; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | int result = className != null ? className.hashCode() : 0; 57 | result = 31 * result + (methodName != null ? methodName.hashCode() : 0); 58 | result = 31 * result + (metricName != null ? metricName.hashCode() : 0); 59 | return result; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return className + '.' + methodName + " " + metricName; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassMethodArgument.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | public class ClassMethodArgument { 20 | private final String className; 21 | private final String methodName; 22 | private final int argumentIndex; 23 | 24 | public ClassMethodArgument(String className, String methodName, int argumentIndex) { 25 | if (className == null) { 26 | throw new NullPointerException("className"); 27 | } 28 | 29 | if (methodName == null) { 30 | throw new NullPointerException("methodName"); 31 | } 32 | 33 | if (argumentIndex < 0) { 34 | throw new IllegalArgumentException("argumentIndex (must equal or greater than 0: 0 means not collecting argument value, 1 means collecting first argument value)"); 35 | } 36 | 37 | this.className = className; 38 | this.methodName = methodName; 39 | this.argumentIndex = argumentIndex; 40 | } 41 | 42 | public String getClassName() { 43 | return className; 44 | } 45 | 46 | public String getMethodName() { 47 | return methodName; 48 | } 49 | 50 | public int getArgumentIndex() { 51 | return argumentIndex; 52 | } 53 | 54 | @Override 55 | public boolean equals(Object o) { 56 | if (this == o) return true; 57 | if (o == null || getClass() != o.getClass()) return false; 58 | 59 | ClassMethodArgument that = (ClassMethodArgument) o; 60 | 61 | if (argumentIndex != that.argumentIndex) return false; 62 | if (className != null ? !className.equals(that.className) : that.className != null) return false; 63 | return methodName != null ? methodName.equals(that.methodName) : that.methodName == null; 64 | } 65 | 66 | @Override 67 | public int hashCode() { 68 | int result = className != null ? className.hashCode() : 0; 69 | result = 31 * result + (methodName != null ? methodName.hashCode() : 0); 70 | result = 31 * result + argumentIndex; 71 | return result; 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | return "{" + 77 | "className='" + className + '\'' + 78 | ", methodName='" + methodName + '\'' + 79 | ", argumentIndex='" + argumentIndex + '\'' + 80 | '}'; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassMethodArgumentFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class ClassMethodArgumentFilter { 23 | private static final String METHOD_NAME_WILDCARD = "*"; 24 | 25 | private ClassMethodArgument[] classMethodArguments = new ClassMethodArgument[0]; 26 | 27 | public ClassMethodArgumentFilter(List classMethodArgumentToFilter) { 28 | if (classMethodArgumentToFilter != null) { 29 | this.classMethodArguments = new ClassMethodArgument[classMethodArgumentToFilter.size()]; 30 | for (int i = 0; i < classMethodArgumentToFilter.size(); i++) { 31 | this.classMethodArguments[i] = classMethodArgumentToFilter.get(i); 32 | } 33 | } 34 | } 35 | 36 | public boolean isEmpty() { 37 | return classMethodArguments.length == 0; 38 | } 39 | 40 | public boolean matchClass(String className) { 41 | for (ClassMethodArgument classAndMethod : classMethodArguments) { 42 | if (className.startsWith(classAndMethod.getClassName())) { 43 | return true; 44 | } 45 | } 46 | return false; 47 | } 48 | 49 | public List matchMethod(String className, String methodName) { 50 | List result = new ArrayList<>(); 51 | 52 | for (ClassMethodArgument classMethodArgument : classMethodArguments) { 53 | if (className.startsWith(classMethodArgument.getClassName())) { 54 | if (METHOD_NAME_WILDCARD.equals(classMethodArgument.getMethodName()) 55 | || methodName.equals(classMethodArgument.getMethodName())) { 56 | result.add(classMethodArgument.getArgumentIndex()); 57 | } 58 | } 59 | } 60 | 61 | return result; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ClassMethodArgumentMetricBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.Map; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | /** 24 | * This class is a buffer to store argument counters. It is thread safe for appendValue. 25 | * The reset method will create a new empty internal buffer and return the old one. 26 | */ 27 | public class ClassMethodArgumentMetricBuffer { 28 | private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); 29 | 30 | public void appendValue(String className, String methodName, String argument) { 31 | ClassAndMethodMetricKey methodMetricKey = new ClassAndMethodMetricKey(className, methodName, argument); 32 | AtomicLong counter = metrics.computeIfAbsent(methodMetricKey, key -> new AtomicLong(0)); 33 | counter.incrementAndGet(); 34 | } 35 | 36 | public Map reset() { 37 | ConcurrentHashMap oldCopy = metrics; 38 | metrics = new ConcurrentHashMap<>(); 39 | return oldCopy; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/DummyConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import com.uber.profiling.ConfigProvider; 20 | 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | public class DummyConfigProvider implements ConfigProvider { 26 | @Override 27 | public Map>> getConfig() { 28 | return new HashMap<>(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ErrorLogReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | public interface ErrorLogReporter { 20 | void report(String message, Throwable exception); 21 | 22 | void close(); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ExponentialBackoffRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.Random; 20 | import java.util.concurrent.Callable; 21 | 22 | public class ExponentialBackoffRetryPolicy { 23 | private static final AgentLogger logger = AgentLogger.getLogger(ExponentialBackoffRetryPolicy.class.getName()); 24 | 25 | private final int maxAttemptCount; 26 | private final long minSleepMillis; 27 | private final float scaleFactor; 28 | 29 | private Random random = new Random(); 30 | 31 | public ExponentialBackoffRetryPolicy(int maxAttemptCount, long minSleepMillis) { 32 | this(maxAttemptCount, minSleepMillis, 2.0f); 33 | } 34 | 35 | public ExponentialBackoffRetryPolicy(int maxAttemptCount, long minSleepMillis, float scaleFactor) { 36 | this.maxAttemptCount = maxAttemptCount; 37 | this.minSleepMillis = minSleepMillis; 38 | this.scaleFactor = scaleFactor; 39 | } 40 | 41 | public T attempt(Callable operation) { 42 | int remainingAttempts = maxAttemptCount - 1; 43 | long minSleepTime = minSleepMillis; 44 | long maxSleepTime = (long)(minSleepMillis * scaleFactor); 45 | 46 | Throwable previousException; 47 | 48 | try { 49 | return operation.call(); 50 | } catch (Throwable ex) { 51 | if (remainingAttempts <= 0) { 52 | throw new RuntimeException("Failed with first try and no remaining retry", ex); 53 | } 54 | previousException = ex; 55 | } 56 | 57 | while (remainingAttempts > 0) { 58 | long sleepTime = minSleepTime + random.nextInt((int)(maxSleepTime - minSleepTime)); 59 | logger.info(String.format("Retrying (after sleeping %s milliseconds) on exception: %s", sleepTime, previousException)); 60 | try { 61 | Thread.sleep(sleepTime); 62 | } catch (InterruptedException ex) { 63 | logger.warn("Sleep interrupted", ex); 64 | } 65 | try { 66 | return operation.call(); 67 | } catch (Throwable ex) { 68 | previousException = ex; 69 | } 70 | 71 | remainingAttempts--; 72 | minSleepTime *= scaleFactor; 73 | maxSleepTime *= scaleFactor; 74 | } 75 | 76 | String msg = String.format("Failed after trying %s times", maxAttemptCount); 77 | throw new RuntimeException(msg, previousException); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/Histogram.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.concurrent.atomic.AtomicLong; 20 | 21 | public class Histogram { 22 | 23 | private AtomicLong count = new AtomicLong(0); 24 | private AtomicLong sum = new AtomicLong(0); 25 | private AtomicLong min = new AtomicLong(Long.MAX_VALUE); 26 | private AtomicLong max = new AtomicLong(Long.MIN_VALUE); 27 | 28 | public void appendValue(long value) { 29 | count.incrementAndGet(); 30 | sum.addAndGet(value); 31 | 32 | min.updateAndGet(x -> Math.min(value, x)); 33 | max.updateAndGet(x -> Math.max(value, x)); 34 | } 35 | 36 | public long getCount() { 37 | return count.get(); 38 | } 39 | 40 | public long getSum() { 41 | return sum.get(); 42 | } 43 | 44 | public long getMin() { 45 | return min.get(); 46 | } 47 | 48 | public long getMax() { 49 | return max.get(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/IOUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.io.ByteArrayOutputStream; 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | 23 | public class IOUtils { 24 | public static byte[] toByteArray(InputStream input) { 25 | try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { 26 | int readByteCount; 27 | byte[] data = new byte[16 * 1024]; 28 | while ((readByteCount = input.read(data, 0, data.length)) != -1) { 29 | byteArrayOutputStream.write(data, 0, readByteCount); 30 | } 31 | byteArrayOutputStream.flush(); 32 | return byteArrayOutputStream.toByteArray(); 33 | } catch (IOException e) { 34 | throw new RuntimeException(e); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/JsonUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import com.fasterxml.jackson.core.JsonParser; 20 | import com.fasterxml.jackson.core.JsonProcessingException; 21 | import com.fasterxml.jackson.core.type.TypeReference; 22 | import com.fasterxml.jackson.databind.DeserializationFeature; 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | 25 | import java.io.IOException; 26 | 27 | public class JsonUtils { 28 | protected static ObjectMapper mapper = new ObjectMapper(); 29 | 30 | static { 31 | mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); 32 | mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); 33 | mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); 34 | mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 35 | } 36 | 37 | public static ObjectMapper getMapper() { 38 | return mapper; 39 | } 40 | 41 | public static String serialize(Object obj) { 42 | if (obj == null) { 43 | return ""; 44 | } 45 | try { 46 | return mapper.writeValueAsString(obj); 47 | } catch (JsonProcessingException e) { 48 | throw new RuntimeException(String.format("Failed to serialize %s (%s)", obj, obj.getClass()), e); 49 | } 50 | } 51 | 52 | 53 | public static T deserialize(String content, Class valueType) { 54 | try { 55 | return mapper.readValue(content, valueType); 56 | } catch (IOException e) { 57 | throw new RuntimeException( 58 | String.format("Failed to deserialize %s from json %s", valueType, content), e); 59 | } 60 | } 61 | 62 | // For example: JsonUtils.deserialize(responseBody, new TypeReference>() {}) 63 | public static T deserialize(String content, TypeReference valueTypeRef) { 64 | try { 65 | return mapper.readValue(content, valueTypeRef); 66 | } catch (IOException ex) { 67 | throw new RuntimeException( 68 | String.format("Failed to deserialize %s from json %s", valueTypeRef, content), ex); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/NetworkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.net.InetAddress; 20 | import java.util.Map; 21 | 22 | public class NetworkUtils { 23 | public static String getLocalHostName() { 24 | try { 25 | Map env = System.getenv(); 26 | if (env.containsKey("COMPUTERNAME")) 27 | return env.get("COMPUTERNAME"); 28 | else if (env.containsKey("HOSTNAME")) 29 | return env.get("HOSTNAME"); 30 | else 31 | return InetAddress.getLocalHost().getHostName(); 32 | } catch (Throwable e) { 33 | return "unknown_localhost_name"; 34 | } 35 | } 36 | 37 | public static void main(String[] args) { 38 | System.out.println(getLocalHostName()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/NoopConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import com.uber.profiling.ConfigProvider; 20 | 21 | import java.util.Arrays; 22 | import java.util.HashMap; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | public class NoopConfigProvider implements ConfigProvider { 27 | @Override 28 | public Map>> getConfig() { 29 | Map>> configMap = new HashMap<>(); 30 | 31 | Map> values = new HashMap<>(); 32 | values.put("noop", Arrays.asList("true")); 33 | 34 | configMap.put("", values); 35 | 36 | return configMap; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ProcessUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.lang.management.ManagementFactory; 20 | import java.lang.management.RuntimeMXBean; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | import java.util.regex.Matcher; 24 | import java.util.regex.Pattern; 25 | 26 | public class ProcessUtils { 27 | private static final String SPARK_PROCESS_KEYWORD = "spark.yarn.app.container.log.dir"; 28 | private static final String SPARK_CMDLINE_KEYWORD = "spark."; 29 | private static final String SPARK_EXECUTOR_CLASS_NAME = "spark.executor.CoarseGrainedExecutorBackend"; 30 | private static final String SPARK_EXECUTOR_KEYWORD = "spark.driver.port"; 31 | 32 | private static final Pattern XMX_REGEX = Pattern.compile("-[xX][mM][xX]([a-zA-Z0-9]+)"); 33 | 34 | public static String getCurrentProcessName() { 35 | try { 36 | return ManagementFactory.getRuntimeMXBean().getName(); 37 | } catch (Throwable ex) { 38 | return ex.getMessage(); 39 | } 40 | } 41 | 42 | public static String getJvmClassPath() { 43 | RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); 44 | return runtimeMXBean.getClassPath(); 45 | } 46 | 47 | public static List getJvmInputArguments() { 48 | RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); 49 | List jvmArgs = runtimeMXBean.getInputArguments(); 50 | return jvmArgs == null ? new ArrayList<>() : jvmArgs; 51 | } 52 | 53 | public static Long getJvmXmxBytes() { 54 | Long result = null; 55 | 56 | RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); 57 | List jvmArgs = runtimeMXBean.getInputArguments(); 58 | if (jvmArgs == null) { 59 | return null; 60 | } 61 | 62 | for (String entry : jvmArgs) { 63 | Matcher matcher = XMX_REGEX.matcher(entry); 64 | if (matcher.matches()) { 65 | String str = matcher.group(1); 66 | result = StringUtils.getBytesValueOrNull(str); 67 | } 68 | } 69 | 70 | return result; 71 | } 72 | 73 | public static boolean isSparkProcess(String cmdline) { 74 | if (cmdline != null && !cmdline.isEmpty()) { 75 | if (cmdline.contains(SPARK_CMDLINE_KEYWORD)) { 76 | return true; 77 | } 78 | } 79 | 80 | List strList = ProcessUtils.getJvmInputArguments(); 81 | for (String str : strList) { 82 | if (str.toLowerCase().contains(SPARK_PROCESS_KEYWORD.toLowerCase())) { 83 | return true; 84 | } 85 | } 86 | return false; 87 | } 88 | 89 | public static boolean isSparkExecutor(String cmdline) { 90 | if (cmdline != null && !cmdline.isEmpty()) { 91 | if (cmdline.contains(SPARK_EXECUTOR_CLASS_NAME)) { 92 | return true; 93 | } 94 | } 95 | 96 | List strList = ProcessUtils.getJvmInputArguments(); 97 | for (String str : strList) { 98 | if (str.toLowerCase().contains(SPARK_EXECUTOR_KEYWORD.toLowerCase())) { 99 | return true; 100 | } 101 | } 102 | return false; 103 | } 104 | 105 | public static boolean isSparkDriver(String cmdline) { 106 | return isSparkProcess(cmdline) && !isSparkExecutor(cmdline); 107 | } 108 | 109 | public static void main(String[] args) { 110 | System.out.println(getCurrentProcessName()); 111 | System.out.println(isSparkProcess(null)); 112 | System.out.println(isSparkExecutor(null)); 113 | System.out.println(isSparkDriver(null)); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/ReflectionUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.lang.reflect.Constructor; 20 | import java.lang.reflect.InvocationTargetException; 21 | import java.lang.reflect.Method; 22 | 23 | public class ReflectionUtils { 24 | private static final AgentLogger logger = AgentLogger.getLogger(ReflectionUtils.class.getName()); 25 | 26 | public static Constructor getConstructor(String implementaionClass, Class interfaceClass) { 27 | Class clazz = null; 28 | 29 | try { 30 | clazz = Class.forName(implementaionClass); 31 | } catch (Throwable e) { 32 | throw new RuntimeException(String.format("Failed to get class for %s", implementaionClass), e); 33 | } 34 | 35 | if (!interfaceClass.isAssignableFrom(clazz)) { 36 | throw new RuntimeException(String.format("Invalid class %s, please make sure it is an implementation of %s", clazz, interfaceClass.getName())); 37 | } 38 | 39 | try { 40 | Class concretelass = (Class) clazz; 41 | Constructor constructor = concretelass.getConstructor(); 42 | return constructor; 43 | } catch (Throwable e) { 44 | throw new RuntimeException(String.format("Failed to get constructor for %s", clazz.getName()), e); 45 | } 46 | } 47 | 48 | public static T createInstance(String implementaionClass, Class interfaceClass) { 49 | try { 50 | Constructor constructor = getConstructor(implementaionClass, interfaceClass); 51 | T result = constructor.newInstance(); 52 | logger.info(String.format("Created %s instance (%s) for interface %s", implementaionClass, result, interfaceClass)); 53 | return result; 54 | } catch (Throwable e) { 55 | throw new RuntimeException(String.format("Failed to create %s instance for interface %s", implementaionClass, interfaceClass), e); 56 | } 57 | } 58 | 59 | public static Object executeStaticMethods(String className, String methods) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { 60 | String[] methodArray = methods.split("\\."); 61 | Class clazz = Class.forName(className); 62 | Object clazzObject = null; 63 | Object result = null; 64 | for (String entry : methodArray) { 65 | Method method = clazz.getMethod(entry); 66 | if (method == null) { 67 | return null; 68 | } 69 | result = method.invoke(clazzObject); 70 | if (result == null) { 71 | return null; 72 | } 73 | 74 | clazz = result.getClass(); 75 | clazzObject = result; 76 | } 77 | return result; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/SparkAppCmdInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.Arrays; 20 | 21 | public class SparkAppCmdInfo { 22 | private String appClass; 23 | private String appJar; 24 | private String[] args = new String[0]; 25 | 26 | public String getAppClass() { 27 | return appClass; 28 | } 29 | 30 | public void setAppClass(String appClass) { 31 | this.appClass = appClass; 32 | } 33 | 34 | public String getAppJar() { 35 | return appJar; 36 | } 37 | 38 | public void setAppJar(String appJar) { 39 | this.appJar = appJar; 40 | } 41 | 42 | public String[] getArgs() { 43 | return args; 44 | } 45 | 46 | public void setArgs(String[] args) { 47 | if (args == null) { 48 | this.args = new String[0]; 49 | } else { 50 | this.args = Arrays.copyOf(args, args.length); 51 | } 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "SparkAppCmdInfo{" + 57 | "appClass='" + appClass + '\'' + 58 | ", appJar='" + appJar + '\'' + 59 | ", args=" + Arrays.toString(args) + 60 | '}'; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/SparkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import com.uber.profiling.profilers.Constants; 20 | 21 | import java.util.List; 22 | 23 | public class SparkUtils { 24 | // Try to get application ID by match regex in class path or system property 25 | public static String probeAppId(String appIdRegex) { 26 | String appId = System.getProperty("spark.app.id"); 27 | 28 | if (appId == null || appId.isEmpty()) { 29 | String classPath = ProcessUtils.getJvmClassPath(); 30 | List appIdCandidates = StringUtils.extractByRegex(classPath, appIdRegex); 31 | if (!appIdCandidates.isEmpty()) { 32 | appId = appIdCandidates.get(0); 33 | } 34 | } 35 | 36 | if (appId == null || appId.isEmpty()) { 37 | for (String entry : ProcessUtils.getJvmInputArguments()) { 38 | List appIdCandidates = StringUtils.extractByRegex(entry, appIdRegex); 39 | if (!appIdCandidates.isEmpty()) { 40 | appId = appIdCandidates.get(0); 41 | break; 42 | } 43 | } 44 | } 45 | 46 | return appId; 47 | } 48 | 49 | // Get application ID by invoking SparkEnv 50 | public static String getSparkEnvAppId() { 51 | // Do not use "org.apache.spark.SparkEnv" directly because the maven shade plugin will convert 52 | // the class name to ja_shaded.org.apache.spark.SparkEnv due to relocation. 53 | String className = org.apache.commons.lang3.StringUtils.joinWith( 54 | ".", 55 | "org", 56 | "apache", 57 | "spark", 58 | "SparkEnv"); 59 | try { 60 | Object result = ReflectionUtils.executeStaticMethods( 61 | className, 62 | "get.conf.getAppId"); 63 | if (result == null) { 64 | return null; 65 | } 66 | return result.toString(); 67 | } catch (Throwable e) { 68 | return null; 69 | } 70 | } 71 | 72 | public static String probeRole(String cmdline) { 73 | if (ProcessUtils.isSparkExecutor(cmdline)) { 74 | return Constants.EXECUTOR_ROLE; 75 | } else if (ProcessUtils.isSparkDriver(cmdline)) { 76 | return Constants.DRIVER_ROLE; 77 | } else { 78 | return null; 79 | } 80 | } 81 | 82 | public static SparkAppCmdInfo probeCmdInfo() { 83 | // TODO use /proc file system to get command when the system property is not available 84 | String cmd = System.getProperty("sun.java.command"); 85 | if (cmd == null || cmd.isEmpty()) { 86 | return null; 87 | } 88 | 89 | SparkAppCmdInfo result = new SparkAppCmdInfo(); 90 | 91 | result.setAppJar(StringUtils.getArgumentValue(cmd, "--jar")); 92 | result.setAppClass(StringUtils.getArgumentValue(cmd, "--class")); 93 | 94 | return result; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/Stacktrace.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.Arrays; 20 | 21 | public class Stacktrace { 22 | private String threadName; 23 | private String threadState; 24 | private ClassAndMethod[] stack = new ClassAndMethod[0]; 25 | 26 | public String getThreadName() { 27 | return threadName; 28 | } 29 | 30 | public void setThreadName(String threadName) { 31 | this.threadName = threadName; 32 | } 33 | 34 | public String getThreadState() { 35 | return threadState; 36 | } 37 | 38 | public void setThreadState(String threadState) { 39 | this.threadState = threadState; 40 | } 41 | 42 | public ClassAndMethod[] getStack() { 43 | return stack; 44 | } 45 | 46 | public void setStack(ClassAndMethod[] stack) { 47 | if (stack == null) { 48 | this.stack = new ClassAndMethod[0]; 49 | } else { 50 | this.stack = stack; 51 | } 52 | } 53 | 54 | @Override 55 | public boolean equals(Object o) { 56 | if (this == o) return true; 57 | if (o == null || getClass() != o.getClass()) return false; 58 | 59 | Stacktrace that = (Stacktrace) o; 60 | 61 | if (threadName != null ? !threadName.equals(that.threadName) : that.threadName != null) return false; 62 | if (threadState != null ? !threadState.equals(that.threadState) : that.threadState != null) return false; 63 | 64 | return Arrays.equals(stack, that.stack); 65 | } 66 | 67 | @Override 68 | public int hashCode() { 69 | int result = threadName != null ? threadName.hashCode() : 0; 70 | result = 31 * result + (threadState != null ? threadState.hashCode() : 0); 71 | result = 31 * result + Arrays.hashCode(stack); 72 | return result; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/uber/profiling/util/StacktraceMetricBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import java.util.Map; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | /** 24 | * StacktraceMetricBuffer is a buffer to store metrics. It is thread safe for appendValue. 25 | * The reset method will create a new empty internal buffer and return the old one. 26 | */ 27 | public class StacktraceMetricBuffer { 28 | private AtomicLong lastResetMillis = new AtomicLong(System.currentTimeMillis()); 29 | 30 | private volatile ConcurrentHashMap metrics = new ConcurrentHashMap<>(); 31 | 32 | public void appendValue(Stacktrace stacktrace) { 33 | AtomicLong counter = metrics.computeIfAbsent(stacktrace, key -> new AtomicLong(0)); 34 | counter.incrementAndGet(); 35 | } 36 | 37 | public long getLastResetMillis() { 38 | return lastResetMillis.get(); 39 | } 40 | 41 | public Map reset() { 42 | ConcurrentHashMap oldCopy = metrics; 43 | metrics = new ConcurrentHashMap<>(); 44 | 45 | lastResetMillis.set(System.currentTimeMillis()); 46 | 47 | return oldCopy; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java_datadog/com/uber/profiling/reporters/DatadogOutputReporter.java: -------------------------------------------------------------------------------- 1 | package com.uber.profiling.reporters; 2 | 3 | import java.util.AbstractMap; 4 | import java.util.ArrayList; 5 | import java.util.Deque; 6 | import java.util.LinkedList; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.apache.commons.lang3.StringUtils; 11 | 12 | import com.timgroup.statsd.NonBlockingStatsDClient; 13 | import com.timgroup.statsd.StatsDClient; 14 | import com.uber.profiling.Reporter; 15 | import com.uber.profiling.util.AgentLogger; 16 | 17 | public class DatadogOutputReporter implements Reporter { 18 | private static final AgentLogger logger = AgentLogger.getLogger(DatadogOutputReporter.class.getName()); 19 | 20 | private StatsDClient statsdClient = null; 21 | 22 | private String prefix = ""; 23 | private String hostname = "localhost"; 24 | private int port = 8125; 25 | 26 | @Override 27 | public void report(String profilerName, Map metrics) { 28 | ensureStatsdConn(); 29 | 30 | List> individualMetrics = new ArrayList<>(); 31 | Deque> queue = new LinkedList<>(metrics.entrySet()); 32 | while (!queue.isEmpty()) { 33 | Map.Entry entry = queue.pollLast(); 34 | String key = entry.getKey(); 35 | Object value = entry.getValue(); 36 | 37 | if (value instanceof Number) { 38 | // If it's a number, we can record the metric as is. 39 | individualMetrics.add(new AbstractMap.SimpleEntry<>(key, (Number)value)); 40 | } else if (value instanceof Map) { 41 | // If it's a map, we'll want to get individual metrics out of the map. 42 | ((Map) value).forEach((nestedKey, nestedValue) -> { 43 | String newMetricName = key + "." + nestedKey; 44 | 45 | if (nestedValue instanceof Number) { 46 | individualMetrics.add(new AbstractMap.SimpleEntry<>(newMetricName, (Number)nestedValue)); 47 | } else { 48 | queue.push(new AbstractMap.SimpleEntry<>(newMetricName, nestedValue)); 49 | } 50 | }); 51 | } 52 | } 53 | 54 | for (Map.Entry entry : individualMetrics) { 55 | this.statsdClient.gauge(entry.getKey(), entry.getValue().doubleValue()); 56 | } 57 | } 58 | 59 | @Override 60 | public void close() { 61 | synchronized (this) { 62 | this.statsdClient.close(); 63 | this.statsdClient = null; 64 | } 65 | } 66 | 67 | private void ensureStatsdConn() { 68 | synchronized (this) { 69 | if (statsdClient == null) { 70 | statsdClient = new NonBlockingStatsDClient(prefix, hostname, port); 71 | } 72 | } 73 | } 74 | 75 | // properties from yaml file 76 | @Override 77 | public void updateArguments(Map> connectionProperties) { 78 | for (Map.Entry> entry : connectionProperties.entrySet()) { 79 | String key = entry.getKey(); 80 | List value = entry.getValue(); 81 | if (StringUtils.isNotEmpty(key) && value != null && !value.isEmpty()) { 82 | String stringValue = value.get(0); 83 | if (StringUtils.isBlank(stringValue)) { 84 | break; 85 | } 86 | 87 | switch (key) { 88 | case "datadog.statsd.prefix": 89 | logger.info("Got value for prefix = " + stringValue); 90 | this.prefix = stringValue; 91 | break; 92 | case "datadog.statsd.hostname": 93 | logger.info("Got value for hostname = " + stringValue); 94 | this.hostname = stringValue; 95 | break; 96 | case "datadog.statsd.port": 97 | logger.info("Got value for port = " + stringValue); 98 | this.port = Integer.parseInt(stringValue); 99 | break; 100 | } 101 | } 102 | } 103 | } 104 | } 105 | 106 | -------------------------------------------------------------------------------- /src/main/java_graphite/graphite.yaml: -------------------------------------------------------------------------------- 1 | graphite: 2 | host: 127.0.0.1 3 | port: 2003 4 | prefix: jvm -------------------------------------------------------------------------------- /src/main/java_jdbc/com/uber/profiling/reporters/CpuAndMemoryProfilerMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | public class CpuAndMemoryProfilerMetric { 20 | private Long epochMillis; 21 | private String name; 22 | private String host; 23 | private String processUuid; 24 | private String appId; 25 | 26 | private String tag; 27 | private String role; 28 | private Double processCpuLoad; 29 | private Double systemCpuLoad; 30 | private Long processCpuTime; 31 | private Long heapMemoryTotalUsed; 32 | private Long heapMemoryCommitted; 33 | private Long heapMemoryMax; 34 | private Long nonHeapMemoryTotalUsed; 35 | private Long nonHeapMemoryCommitted; 36 | private Long nonHeapMemoryMax; 37 | private Long vmRSS; 38 | private Long vmHWM; 39 | private Long vmSize; 40 | private Long vmPeak; 41 | 42 | public Long getEpochMillis() { 43 | return epochMillis; 44 | } 45 | 46 | public void setEpochMillis(Long epochMillis) { 47 | this.epochMillis = epochMillis; 48 | } 49 | 50 | public String getName() { 51 | return name; 52 | } 53 | 54 | public void setName(String name) { 55 | this.name = name; 56 | } 57 | 58 | public String getHost() { 59 | return host; 60 | } 61 | 62 | public void setHost(String host) { 63 | this.host = host; 64 | } 65 | 66 | public String getProcessUuid() { 67 | return processUuid; 68 | } 69 | 70 | public void setProcessUuid(String processUuid) { 71 | this.processUuid = processUuid; 72 | } 73 | 74 | public String getAppId() { 75 | return appId; 76 | } 77 | 78 | public void setAppId(String appId) { 79 | this.appId = appId; 80 | } 81 | 82 | public String getTag() { 83 | return tag; 84 | } 85 | 86 | public void setTag(String tag) { 87 | this.tag = tag; 88 | } 89 | 90 | public String getRole() { 91 | return role; 92 | } 93 | 94 | public void setRole(String role) { 95 | this.role = role; 96 | } 97 | 98 | public Double getProcessCpuLoad() { 99 | return processCpuLoad; 100 | } 101 | 102 | public void setProcessCpuLoad(Double processCpuLoad) { 103 | this.processCpuLoad = processCpuLoad; 104 | } 105 | 106 | public Double getSystemCpuLoad() { 107 | return systemCpuLoad; 108 | } 109 | 110 | public void setSystemCpuLoad(Double systemCpuLoad) { 111 | this.systemCpuLoad = systemCpuLoad; 112 | } 113 | 114 | public Long getProcessCpuTime() { 115 | return processCpuTime; 116 | } 117 | 118 | public void setProcessCpuTime(Long processCpuTime) { 119 | this.processCpuTime = processCpuTime; 120 | } 121 | 122 | public Long getHeapMemoryTotalUsed() { 123 | return heapMemoryTotalUsed; 124 | } 125 | 126 | public void setHeapMemoryTotalUsed(Long heapMemoryTotalUsed) { 127 | this.heapMemoryTotalUsed = heapMemoryTotalUsed; 128 | } 129 | 130 | public Long getHeapMemoryCommitted() { 131 | return heapMemoryCommitted; 132 | } 133 | 134 | public void setHeapMemoryCommitted(Long heapMemoryCommitted) { 135 | this.heapMemoryCommitted = heapMemoryCommitted; 136 | } 137 | 138 | public Long getHeapMemoryMax() { 139 | return heapMemoryMax; 140 | } 141 | 142 | public void setHeapMemoryMax(Long heapMemoryMax) { 143 | this.heapMemoryMax = heapMemoryMax; 144 | } 145 | 146 | public Long getNonHeapMemoryTotalUsed() { 147 | return nonHeapMemoryTotalUsed; 148 | } 149 | 150 | public void setNonHeapMemoryTotalUsed(Long nonHeapMemoryTotalUsed) { 151 | this.nonHeapMemoryTotalUsed = nonHeapMemoryTotalUsed; 152 | } 153 | 154 | public Long getNonHeapMemoryCommitted() { 155 | return nonHeapMemoryCommitted; 156 | } 157 | 158 | public void setNonHeapMemoryCommitted(Long nonHeapMemoryCommitted) { 159 | this.nonHeapMemoryCommitted = nonHeapMemoryCommitted; 160 | } 161 | 162 | public Long getNonHeapMemoryMax() { 163 | return nonHeapMemoryMax; 164 | } 165 | 166 | public void setNonHeapMemoryMax(Long nonHeapMemoryMax) { 167 | this.nonHeapMemoryMax = nonHeapMemoryMax; 168 | } 169 | 170 | public Long getVmRSS() { 171 | return vmRSS; 172 | } 173 | 174 | public void setVmRSS(Long vmRSS) { 175 | this.vmRSS = vmRSS; 176 | } 177 | 178 | public Long getVmHWM() { 179 | return vmHWM; 180 | } 181 | 182 | public void setVmHWM(Long vmHWM) { 183 | this.vmHWM = vmHWM; 184 | } 185 | 186 | public Long getVmSize() { 187 | return vmSize; 188 | } 189 | 190 | public void setVmSize(Long vmSize) { 191 | this.vmSize = vmSize; 192 | } 193 | 194 | public Long getVmPeak() { 195 | return vmPeak; 196 | } 197 | 198 | public void setVmPeak(Long vmPeak) { 199 | this.vmPeak = vmPeak; 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /src/main/java_jdbc/com/uber/profiling/reporters/CpuAndMemoryProfilerMetricDao.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import com.uber.profiling.reporters.util.BaseJdbcDao; 20 | 21 | import java.util.Arrays; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | public class CpuAndMemoryProfilerMetricDao extends BaseJdbcDao { 26 | 27 | public static final String PARTITION_KEY = "DAYOFMONTH(epochMillis)"; 28 | 29 | public static final String[] PRIMARY_KEYS = new String[] {"epochMillis", "name", "host", "processUuid", "appId"}; 30 | 31 | public static final String[] INDEX_COLUMNS = new String[] {"epochMillis", "name", "host", "processUuid", "appId"}; 32 | 33 | public static final String[] DATETIME_COLUMNS = new String[] {"epochMillis"}; 34 | 35 | public static final String[] TEXT_COLUMNS = new String[] {}; 36 | 37 | public CpuAndMemoryProfilerMetricDao(String jdbcDriverClass, String connectionString, String tableName) { 38 | super( 39 | jdbcDriverClass, 40 | connectionString, 41 | CpuAndMemoryProfilerMetric.class, 42 | tableName, 43 | PARTITION_KEY, 44 | Arrays.asList(PRIMARY_KEYS), 45 | Arrays.asList(INDEX_COLUMNS), 46 | Arrays.asList(DATETIME_COLUMNS), 47 | Arrays.asList(TEXT_COLUMNS)); 48 | } 49 | 50 | public void insertOrUpdate(CpuAndMemoryProfilerMetric entity) { 51 | Map map = new HashMap(); 52 | map.put("epochMillis", entity.getEpochMillis()); 53 | map.put("name", entity.getName()); 54 | map.put("host", entity.getHost()); 55 | map.put("processUuid", entity.getProcessUuid()); 56 | map.put("appId", entity.getAppId()); 57 | map.put("tag", entity.getTag()); 58 | map.put("role", entity.getRole()); 59 | map.put("processCpuLoad", entity.getProcessCpuLoad()); 60 | map.put("systemCpuLoad", entity.getSystemCpuLoad()); 61 | map.put("processCpuTime", entity.getProcessCpuTime()); 62 | map.put("heapMemoryTotalUsed", entity.getHeapMemoryTotalUsed()); 63 | map.put("heapMemoryCommitted", entity.getHeapMemoryCommitted()); 64 | map.put("heapMemoryMax", entity.getHeapMemoryMax()); 65 | map.put("nonHeapMemoryCommitted", entity.getNonHeapMemoryCommitted()); 66 | map.put("nonHeapMemoryTotalUsed", entity.getNonHeapMemoryTotalUsed()); 67 | map.put("nonHeapMemoryMax", entity.getNonHeapMemoryMax()); 68 | map.put("vmRSS", entity.getVmRSS()); 69 | map.put("vmHWM", entity.getVmHWM()); 70 | map.put("vmSize", entity.getVmSize()); 71 | map.put("vmPeak", entity.getVmPeak()); 72 | insertOrUpdate(map); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java_jdbc/com/uber/profiling/reporters/util/BeanUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters.util; 18 | 19 | import java.beans.BeanInfo; 20 | import java.beans.IntrospectionException; 21 | import java.beans.Introspector; 22 | import java.beans.PropertyDescriptor; 23 | import java.lang.reflect.InvocationTargetException; 24 | import java.util.HashMap; 25 | import java.util.Map; 26 | 27 | public class BeanUtils { 28 | public static T merge(T oldBean, T newBean) { 29 | T result = null; 30 | try { 31 | result = (T) oldBean.getClass().getConstructor().newInstance(); 32 | 33 | BeanInfo beanInfo = Introspector.getBeanInfo(oldBean.getClass()); 34 | 35 | for (PropertyDescriptor descriptor : beanInfo.getPropertyDescriptors()) { 36 | 37 | // Only copy writable attributes 38 | if (descriptor.getWriteMethod() != null) { 39 | Object value1 = descriptor.getReadMethod().invoke(oldBean); 40 | Object value2 = descriptor.getReadMethod().invoke(newBean); 41 | 42 | if (value1 == null && value2 != null) { 43 | descriptor.getWriteMethod().invoke(result, value2); 44 | } else if (value1 != null && value2 == null) { 45 | descriptor.getWriteMethod().invoke(result, value1); 46 | } else if (value1 != null && value2 != null) { 47 | descriptor.getWriteMethod().invoke(result, value2); 48 | } 49 | } 50 | } 51 | 52 | return result; 53 | } catch (InstantiationException 54 | | IllegalAccessException 55 | | InvocationTargetException 56 | | NoSuchMethodException 57 | | IntrospectionException e) { 58 | throw new RuntimeException( 59 | String.format("Failed to merge two beans, old: %s, new: %s", oldBean, newBean), e); 60 | } 61 | } 62 | 63 | public static Map getPropertyDescriptorMapWithLowerCaseName( 64 | Class clazz) { 65 | try { 66 | Map result = new HashMap<>(); 67 | 68 | BeanInfo beanInfo = Introspector.getBeanInfo(clazz); 69 | for (PropertyDescriptor descriptor : beanInfo.getPropertyDescriptors()) { 70 | if (descriptor.getWriteMethod() == null || descriptor.getReadMethod() == null) { 71 | continue; 72 | } 73 | result.put(descriptor.getName().toLowerCase(), descriptor); 74 | } 75 | 76 | return result; 77 | } catch (IntrospectionException e) { 78 | throw new RuntimeException( 79 | String.format("Failed to get properties for bean class: %s", clazz), e); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java_jdbc/com/uber/profiling/reporters/util/DateTimeUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters.util; 18 | 19 | import com.uber.profiling.util.AgentLogger; 20 | import org.apache.commons.lang3.math.NumberUtils; 21 | 22 | import java.text.DateFormat; 23 | import java.text.ParseException; 24 | import java.text.SimpleDateFormat; 25 | import java.time.LocalDateTime; 26 | import java.time.ZoneOffset; 27 | import java.time.format.DateTimeFormatter; 28 | import java.util.Calendar; 29 | import java.util.Date; 30 | import java.util.TimeZone; 31 | 32 | public class DateTimeUtils { 33 | private static final AgentLogger logger = AgentLogger.getLogger(DateTimeUtils.class.getName()); 34 | 35 | public static Calendar getUtcCalendar() { 36 | TimeZone timeZone = TimeZone.getTimeZone("UTC"); 37 | Calendar calendar = Calendar.getInstance(timeZone); 38 | return calendar; 39 | } 40 | 41 | public static String formatAsIso(long millis) { 42 | return formatAsIso(new Date(millis)); 43 | } 44 | 45 | public static String formatAsIso(Date date) { 46 | TimeZone tz = TimeZone.getTimeZone("UTC"); 47 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSSS'Z'"); 48 | df.setTimeZone(tz); 49 | return df.format(date); 50 | } 51 | 52 | public static String formatAsIsoWithoutMillis(long millis) { 53 | return formatAsIsoWithoutMillis(new Date(millis)); 54 | } 55 | 56 | public static String formatAsIsoWithoutMillis(Date date) { 57 | TimeZone tz = TimeZone.getTimeZone("UTC"); 58 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); 59 | df.setTimeZone(tz); 60 | return df.format(date); 61 | } 62 | 63 | public static Date parseDateTimeSmart(String str) { 64 | if (NumberUtils.isCreatable(str)) { 65 | double doubleValue = Double.parseDouble(str); 66 | Date dt = new Date((long) doubleValue); 67 | Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 68 | cal.setTime(dt); 69 | int year = cal.get(Calendar.YEAR); 70 | if (year >= 2000 && year < 3000) { 71 | return dt; 72 | } 73 | return new Date((long) doubleValue * 1000); 74 | } else { 75 | return parseIsoDateTime(str); 76 | } 77 | } 78 | 79 | public static long getMillisSmart(double value) { 80 | Date dt = new Date((long) value); 81 | Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 82 | cal.setTime(dt); 83 | int year = cal.get(Calendar.YEAR); 84 | if (year >= 2000 && year < 3000) { 85 | return (long) value; 86 | } 87 | return (long) (value * 1000); 88 | } 89 | 90 | public static long getMillisSmart(long value) { 91 | Date dt = new Date(value); 92 | Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 93 | cal.setTime(dt); 94 | int year = cal.get(Calendar.YEAR); 95 | if (year >= 2000 && year < 3000) { 96 | return value; 97 | } 98 | return value * 1000; 99 | } 100 | 101 | public static Date parseIsoDateTime(String str) { 102 | try { 103 | DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME; 104 | LocalDateTime localDateTime = LocalDateTime.parse(str, dtf); 105 | return Date.from(localDateTime.atOffset(ZoneOffset.UTC).toInstant()); 106 | } catch (IllegalArgumentException e) { 107 | logger.debug(String.format("Failed to parse date time %s with exception %s, will try another format", str, e)); 108 | } 109 | 110 | try { 111 | DateTimeFormatter dtf = DateTimeFormatter.ISO_LOCAL_DATE_TIME; 112 | LocalDateTime localDateTime = LocalDateTime.parse(str, dtf); 113 | return Date.from(localDateTime.atOffset(ZoneOffset.UTC).toInstant()); 114 | } catch (IllegalArgumentException e) { 115 | logger.debug(String.format("Failed to parse date time %s with exception %s, will try another format", str, e)); 116 | } 117 | 118 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); 119 | TimeZone tz = TimeZone.getTimeZone("UTC"); 120 | df.setTimeZone(tz); 121 | try { 122 | return df.parse(str); 123 | } catch (ParseException e) { 124 | logger.debug(String.format("Failed to parse date time %s with exception %s, will try another format", str, e)); 125 | } 126 | 127 | df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 128 | df.setTimeZone(tz); 129 | try { 130 | return df.parse(str); 131 | } catch (ParseException e) { 132 | throw new RuntimeException("Failed to parse date time: " + str, e); 133 | } 134 | } 135 | 136 | public static Date addDays(Date date, int days) { 137 | Calendar c = Calendar.getInstance(); 138 | c.setTime(date); 139 | c.add(Calendar.DATE, days); 140 | return c.getTime(); 141 | } 142 | 143 | public static long truncateToDay(long millis) { 144 | long millisPerDay = 1000 * 60 * 60 * 24; 145 | return (millis / millisPerDay) * millisPerDay; 146 | } 147 | 148 | public static Date truncateToDay(Date date) { 149 | long millisPerDay = 1000 * 60 * 60 * 24; 150 | return new Date((date.getTime() / millisPerDay) * millisPerDay); 151 | } 152 | 153 | public static int getDayOfMonth(Date date) { 154 | Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 155 | cal.setTime(date); 156 | int day = cal.get(Calendar.DAY_OF_MONTH); 157 | return day; 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java_jdbc/com/uber/profiling/reporters/util/DbConnectionProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters.util; 18 | 19 | import com.uber.profiling.util.AgentLogger; 20 | 21 | import java.sql.Connection; 22 | import java.sql.DriverManager; 23 | import java.sql.SQLException; 24 | 25 | public class DbConnectionProvider { 26 | private static final AgentLogger logger = AgentLogger.getLogger(DbConnectionProvider.class.getName()); 27 | 28 | private static final String MYSQL_JDBC_DRIVER_CLASS = "com.mysql.jdbc.Driver"; 29 | 30 | private final String jdbcDriverClass; 31 | private final String connectionString; 32 | 33 | private volatile static Connection connection; 34 | 35 | public DbConnectionProvider(String jdbcDriverClass, String connectionString) { 36 | this.jdbcDriverClass = jdbcDriverClass; 37 | this.connectionString = connectionString; 38 | } 39 | 40 | public String getConnectionString() { 41 | return connectionString; 42 | } 43 | 44 | public Connection getConnection() { 45 | if (connection != null) { 46 | try { 47 | if (!connection.isClosed()) { 48 | return connection; 49 | } 50 | } catch (SQLException e) { 51 | logger.warn("Failed to check whether db connection is closed", e); 52 | } 53 | } 54 | try { 55 | if (jdbcDriverClass != null && !jdbcDriverClass.isEmpty()) { 56 | try { 57 | Class.forName(jdbcDriverClass); 58 | } catch (ClassNotFoundException e) { 59 | throw new RuntimeException("Failed to find JDBC class " + jdbcDriverClass, e); 60 | } 61 | } 62 | connection = DriverManager.getConnection(connectionString); 63 | return connection; 64 | } catch (SQLException e) { 65 | throw new RuntimeException("Failed to create JDBC connection: " + connectionString, e); 66 | } 67 | } 68 | 69 | public void close() { 70 | if (connection != null) { 71 | try { 72 | connection.close(); 73 | connection = null; 74 | } catch (Throwable e) { 75 | connection = null; 76 | logger.warn("Failed to close JDBC connection", e); 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java_jdbc/com/uber/profiling/reporters/util/SqlUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters.util; 18 | 19 | import com.uber.profiling.util.AgentLogger; 20 | import org.apache.commons.lang3.StringUtils; 21 | 22 | import java.sql.ResultSet; 23 | import java.sql.SQLException; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | public class SqlUtils { 28 | private static final AgentLogger logger = AgentLogger.getLogger(SqlUtils.class.getName()); 29 | 30 | public static void printResultSet(ResultSet resultSet) { 31 | printResultSetColumns(resultSet); 32 | printResultSetRows(resultSet); 33 | } 34 | 35 | public static void printResultSetColumns(ResultSet resultSet) { 36 | try { 37 | List columnNames = new ArrayList<>(); 38 | List columnTypes = new ArrayList<>(); 39 | for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { 40 | columnNames.add(resultSet.getMetaData().getColumnName(i + 1)); 41 | columnTypes.add(resultSet.getMetaData().getColumnTypeName(i + 1)); 42 | } 43 | logger.info("Column names: " + StringUtils.join(columnNames, ", ")); 44 | logger.info("Column types: " + StringUtils.join(columnTypes, ", ")); 45 | } catch (SQLException e) { 46 | logger.warn("Failed to print sql result", e); 47 | } 48 | } 49 | 50 | public static void printResultSetCurrentRow(ResultSet resultSet) { 51 | try { 52 | List rowValues = new ArrayList<>(); 53 | for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { 54 | rowValues.add(String.valueOf(resultSet.getObject(i + 1))); 55 | } 56 | logger.info("Row: " + StringUtils.join(rowValues, ", ")); 57 | } catch (SQLException e) { 58 | logger.warn("Failed to print sql result", e); 59 | } 60 | } 61 | 62 | public static void printResultSetRows(ResultSet resultSet) { 63 | try { 64 | while (resultSet.next()) { 65 | List rowValues = new ArrayList<>(); 66 | for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { 67 | rowValues.add(String.valueOf(resultSet.getObject(i + 1))); 68 | } 69 | logger.info("Row: " + StringUtils.join(rowValues, ", ")); 70 | } 71 | } catch (SQLException e) { 72 | logger.warn("Failed to print sql result", e); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java_redis/com/uber/profiling/RedisOutputReporter.java: -------------------------------------------------------------------------------- 1 | package com.uber.profiling.reporters; 2 | 3 | import com.uber.profiling.Reporter; 4 | import com.uber.profiling.util.AgentLogger; 5 | import com.uber.profiling.util.JsonUtils; 6 | 7 | import java.net.InetAddress; 8 | import java.net.UnknownHostException; 9 | import java.util.Date; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import redis.clients.jedis.Jedis; 14 | import redis.clients.jedis.JedisPool; 15 | 16 | public class RedisOutputReporter implements Reporter { 17 | 18 | private static final AgentLogger logger = AgentLogger.getLogger(RedisOutputReporter.class.getName()); 19 | private JedisPool redisConn = null; 20 | 21 | @Override 22 | public void updateArguments(Map> parsedArgs) { 23 | } 24 | 25 | //JedisPool should always be used as it is thread safe. 26 | @Override 27 | public void report(String profilerName, Map metrics) { 28 | ensureJedisConn(); 29 | try { 30 | Jedis jedisClient = redisConn.getResource(); 31 | jedisClient.set(createOriginStamp(profilerName), JsonUtils.serialize(metrics)); 32 | redisConn.returnResource(jedisClient); 33 | } catch (Exception err) { 34 | logger.warn(err.toString()); 35 | } 36 | } 37 | 38 | public String createOriginStamp(String profilerName) { 39 | try { 40 | return (profilerName + "-" + InetAddress.getLocalHost().getHostAddress() + "-" + System.currentTimeMillis()); 41 | } catch (UnknownHostException err) { 42 | logger.warn("Address could not be determined and will be omitted!"); 43 | return (profilerName + "-" + System.currentTimeMillis()); 44 | } 45 | } 46 | 47 | @Override 48 | public void close() { 49 | synchronized (this) { 50 | redisConn.close(); 51 | redisConn = null; 52 | } 53 | } 54 | 55 | private void ensureJedisConn() { 56 | synchronized (this) { 57 | if (redisConn == null || redisConn.isClosed()) { 58 | redisConn = new JedisPool(System.getenv("JEDIS_PROFILER_CONNECTION")); 59 | return; 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/AgentThreadFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | 24 | public class AgentThreadFactoryTest { 25 | @Test 26 | public void newThread() throws InterruptedException { 27 | final AtomicInteger i = new AtomicInteger(10); 28 | 29 | AgentThreadFactory threadFactory = new AgentThreadFactory(); 30 | Thread thread = threadFactory.newThread(new Runnable() { 31 | @Override 32 | public void run() { 33 | i.incrementAndGet(); 34 | } 35 | }); 36 | 37 | thread.start(); 38 | thread.join(); 39 | 40 | Assert.assertEquals(11, i.get()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/ProfilerRunnableTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | 24 | public class ProfilerRunnableTest { 25 | @Test 26 | public void invokeRunnable() { 27 | final AtomicInteger i = new AtomicInteger(10); 28 | 29 | ProfilerRunner profilerRunnable = new ProfilerRunner(new Profiler() { 30 | @Override 31 | public long getIntervalMillis() { 32 | return 0; 33 | } 34 | 35 | @Override 36 | public void setReporter(Reporter reporter) { 37 | } 38 | 39 | @Override 40 | public void profile() { 41 | i.incrementAndGet(); 42 | } 43 | }); 44 | 45 | profilerRunnable.run(); 46 | 47 | Assert.assertEquals(11, i.get()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/profilers/CpuAndMemoryProfilerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Reporter; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class CpuAndMemoryProfilerTest { 28 | @Test 29 | public void profile() { 30 | final List nameList = new ArrayList<>(); 31 | final List> metricList = new ArrayList<>(); 32 | 33 | CpuAndMemoryProfiler profiler = new CpuAndMemoryProfiler(new Reporter() { 34 | @Override 35 | public void report(String profilerName, Map metrics) { 36 | nameList.add(profilerName); 37 | metricList.add(metrics); 38 | } 39 | 40 | @Override 41 | public void close() { 42 | } 43 | }); 44 | 45 | profiler.setIntervalMillis(123); 46 | Assert.assertEquals(123L, profiler.getIntervalMillis()); 47 | 48 | profiler.profile(); 49 | profiler.profile(); 50 | 51 | Assert.assertEquals(2, nameList.size()); 52 | Assert.assertEquals(CpuAndMemoryProfiler.PROFILER_NAME, nameList.get(0)); 53 | 54 | Assert.assertEquals(2, metricList.size()); 55 | Assert.assertTrue(metricList.get(0).containsKey("processUuid")); 56 | Assert.assertTrue(metricList.get(0).containsKey("processCpuLoad")); 57 | Assert.assertTrue(metricList.get(0).containsKey("heapMemoryTotalUsed")); 58 | Assert.assertTrue(metricList.get(0).containsKey("gc")); 59 | 60 | Object obj = metricList.get(0).get("gc"); 61 | Assert.assertTrue(obj instanceof List); 62 | 63 | List> gcMetrics = (List>) obj; 64 | Assert.assertTrue(gcMetrics.size() >= 1); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/profilers/MethodArgumentProfilerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Reporter; 20 | import com.uber.profiling.util.ClassMethodArgumentMetricBuffer; 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.Map; 27 | import java.util.stream.Collectors; 28 | 29 | public class MethodArgumentProfilerTest { 30 | @Test 31 | public void profile() { 32 | final List nameList = new ArrayList<>(); 33 | final List> metricList = new ArrayList<>(); 34 | 35 | ClassMethodArgumentMetricBuffer buffer = new ClassMethodArgumentMetricBuffer(); 36 | 37 | MethodArgumentCollector collector = new MethodArgumentCollector(buffer); 38 | 39 | Reporter reporter = new Reporter() { 40 | @Override 41 | public void report(String profilerName, Map metrics) { 42 | nameList.add(profilerName); 43 | metricList.add(metrics); 44 | } 45 | 46 | @Override 47 | public void close() { 48 | } 49 | }; 50 | 51 | MethodArgumentProfiler profiler = new MethodArgumentProfiler(buffer, reporter); 52 | 53 | profiler.setIntervalMillis(123); 54 | Assert.assertEquals(123L, profiler.getIntervalMillis()); 55 | 56 | collector.collectMetric("class1", "method1", "arg1"); 57 | collector.collectMetric("class1", "method1", "arg1"); 58 | collector.collectMetric("class2", "method2", "arg2"); 59 | 60 | profiler.profile(); 61 | 62 | Assert.assertEquals(2, nameList.size()); 63 | Assert.assertEquals(MethodArgumentProfiler.PROFILER_NAME, nameList.get(0)); 64 | 65 | Assert.assertEquals(2, metricList.size()); 66 | 67 | List> metricsToCheck = metricList.stream().filter(t -> 68 | t.get("className").equals("class1") 69 | && t.get("methodName").equals("method1") 70 | && t.get("metricName").equals("arg1")) 71 | .collect(Collectors.toList()); 72 | Assert.assertEquals(1, metricsToCheck.size()); 73 | Assert.assertEquals(2.0, (Double) metricsToCheck.get(0).get("metricValue"), 0.01); 74 | 75 | metricsToCheck = metricList.stream().filter(t -> 76 | t.get("className").equals("class2") 77 | && t.get("methodName").equals("method2") 78 | && t.get("metricName").equals("arg2")) 79 | .collect(Collectors.toList()); 80 | Assert.assertEquals(1, metricsToCheck.size()); 81 | Assert.assertEquals(1.0, (Double) metricsToCheck.get(0).get("metricValue"), 0.01); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/profilers/MethodDurationProfilerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Reporter; 20 | import com.uber.profiling.util.ClassAndMethodLongMetricBuffer; 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.Map; 27 | import java.util.stream.Collectors; 28 | 29 | public class MethodDurationProfilerTest { 30 | @Test 31 | public void profile() { 32 | final List nameList = new ArrayList<>(); 33 | final List> metricList = new ArrayList<>(); 34 | 35 | ClassAndMethodLongMetricBuffer buffer = new ClassAndMethodLongMetricBuffer(); 36 | 37 | MethodDurationCollector collector = new MethodDurationCollector(buffer); 38 | 39 | Reporter reporter = new Reporter() { 40 | @Override 41 | public void report(String profilerName, Map metrics) { 42 | nameList.add(profilerName); 43 | metricList.add(metrics); 44 | } 45 | 46 | @Override 47 | public void close() { 48 | } 49 | }; 50 | 51 | MethodDurationProfiler profiler = new MethodDurationProfiler(buffer, reporter); 52 | 53 | profiler.setIntervalMillis(123); 54 | Assert.assertEquals(123L, profiler.getIntervalMillis()); 55 | 56 | collector.collectLongMetric("class1", "method1", "metric1", 111); 57 | collector.collectLongMetric("class1", "method1", "metric1", 333); 58 | collector.collectLongMetric("class2", "method2", "metric2", 222); 59 | 60 | profiler.profile(); 61 | 62 | int metricCountForHistogram = 4; 63 | Assert.assertEquals(2 * metricCountForHistogram, nameList.size()); 64 | Assert.assertEquals(MethodDurationProfiler.PROFILER_NAME, nameList.get(0)); 65 | 66 | Assert.assertEquals(2 * metricCountForHistogram, metricList.size()); 67 | 68 | List> metricsToCheck = metricList.stream().filter(t -> 69 | t.get("className").equals("class1") 70 | && t.get("methodName").equals("method1") 71 | && t.get("metricName").equals("metric1.count")) 72 | .collect(Collectors.toList()); 73 | Assert.assertEquals(1, metricsToCheck.size()); 74 | Assert.assertEquals(2.0, (Double) metricsToCheck.get(0).get("metricValue"), 0.01); 75 | 76 | metricsToCheck = metricList.stream().filter(t -> 77 | t.get("className").equals("class1") 78 | && t.get("methodName").equals("method1") 79 | && t.get("metricName").equals("metric1.sum")) 80 | .collect(Collectors.toList()); 81 | Assert.assertEquals(1, metricsToCheck.size()); 82 | Assert.assertEquals(444.0, (Double) metricsToCheck.get(0).get("metricValue"), 0.01); 83 | 84 | metricsToCheck = metricList.stream().filter(t -> 85 | t.get("className").equals("class2") 86 | && t.get("methodName").equals("method2") 87 | && t.get("metricName").equals("metric2.count")) 88 | .collect(Collectors.toList()); 89 | Assert.assertEquals(1, metricsToCheck.size()); 90 | Assert.assertEquals(1.0, (Double) metricsToCheck.get(0).get("metricValue"), 0.01); 91 | 92 | metricsToCheck = metricList.stream().filter(t -> 93 | t.get("className").equals("class2") 94 | && t.get("methodName").equals("method2") 95 | && t.get("metricName").equals("metric2.sum")) 96 | .collect(Collectors.toList()); 97 | Assert.assertEquals(1, metricsToCheck.size()); 98 | Assert.assertEquals(222.0, (Double) metricsToCheck.get(0).get("metricValue"), 0.01); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/profilers/ProcessInfoProfilerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Reporter; 20 | import com.uber.profiling.util.ProcFileUtils; 21 | import com.uber.profiling.util.ProcessUtils; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | public class ProcessInfoProfilerTest { 30 | @Test 31 | public void profile() { 32 | final List nameList = new ArrayList<>(); 33 | final List> metricList = new ArrayList<>(); 34 | 35 | ProcessInfoProfiler profiler = new ProcessInfoProfiler(new Reporter() { 36 | @Override 37 | public void report(String profilerName, Map metrics) { 38 | nameList.add(profilerName); 39 | metricList.add(metrics); 40 | } 41 | 42 | @Override 43 | public void close() { 44 | } 45 | }); 46 | 47 | Assert.assertEquals(0L, profiler.getIntervalMillis()); 48 | 49 | profiler.profile(); 50 | profiler.profile(); 51 | 52 | System.out.println("Metric list:"); 53 | System.out.println(metricList); 54 | 55 | Assert.assertTrue(nameList.size() >= 2); 56 | Assert.assertEquals(ProcessInfoProfiler.PROFILER_NAME, nameList.get(0)); 57 | 58 | Assert.assertTrue(metricList.size() >= 2); 59 | 60 | Assert.assertTrue(metricList.get(0).containsKey("processUuid")); 61 | Assert.assertTrue(metricList.get(0).containsKey("jvmInputArguments")); 62 | Assert.assertTrue(metricList.get(0).containsKey("jvmClassPath")); 63 | Assert.assertTrue(metricList.get(0).containsKey("cmdline")); 64 | 65 | Assert.assertTrue(metricList.get(metricList.size() - 1).containsKey("processUuid")); 66 | Assert.assertTrue(metricList.get(metricList.size() - 1).containsKey("jvmInputArguments")); 67 | Assert.assertTrue(metricList.get(metricList.size() - 1).containsKey("jvmClassPath")); 68 | Assert.assertTrue(metricList.get(metricList.size() - 1).containsKey("cmdline")); 69 | 70 | // Verify: if cmdline is empty, there should be jvmClassPath/jvmInputArguments, 71 | // otherwise there should be no jvmClassPath/jvmInputArguments 72 | if (ProcFileUtils.getCmdline() == null || ProcFileUtils.getCmdline().isEmpty()) { 73 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("cmdline").equals("")).count() == 0); 74 | 75 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("jvmClassPath").equals("")).count() > 0); 76 | 77 | if (ProcessUtils.getJvmInputArguments().isEmpty()) { 78 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("jvmInputArguments").equals("")).count() == 0); 79 | } else { 80 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("jvmInputArguments").equals("")).count() > 0); 81 | } 82 | } else { 83 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("cmdline").equals("")).count() > 0); 84 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("jvmClassPath").equals("")).count() == 0); 85 | Assert.assertTrue(metricList.stream().filter(map -> !map.get("jvmInputArguments").equals("")).count() == 0); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/profilers/StacktraceReporterProfilerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.profilers; 18 | 19 | import com.uber.profiling.Reporter; 20 | import com.uber.profiling.util.ClassAndMethod; 21 | import com.uber.profiling.util.Stacktrace; 22 | import com.uber.profiling.util.StacktraceMetricBuffer; 23 | import org.junit.Assert; 24 | import org.junit.Test; 25 | 26 | import java.util.ArrayList; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class StacktraceReporterProfilerTest { 31 | @Test 32 | public void profile() { 33 | final List nameList = new ArrayList<>(); 34 | final List> metricList = new ArrayList<>(); 35 | 36 | Reporter reporter = new Reporter() { 37 | @Override 38 | public void report(String profilerName, Map metrics) { 39 | nameList.add(profilerName); 40 | metricList.add(metrics); 41 | } 42 | 43 | @Override 44 | public void close() { 45 | } 46 | }; 47 | 48 | long epochMillis1 = System.currentTimeMillis(); 49 | 50 | StacktraceMetricBuffer buffer = new StacktraceMetricBuffer(); 51 | StacktraceReporterProfiler profiler = new StacktraceReporterProfiler(buffer, reporter); 52 | 53 | profiler.setIntervalMillis(123); 54 | Assert.assertEquals(123L, profiler.getIntervalMillis()); 55 | 56 | profiler.profile(); 57 | Assert.assertEquals(0, nameList.size()); 58 | Assert.assertEquals(0, metricList.size()); 59 | 60 | Stacktrace stacktrace = new Stacktrace(); 61 | buffer.appendValue(stacktrace); 62 | 63 | profiler.profile(); 64 | 65 | long epochMillis2 = System.currentTimeMillis(); 66 | 67 | Assert.assertEquals(1, nameList.size()); 68 | Assert.assertEquals("Stacktrace", nameList.get(0)); 69 | 70 | Assert.assertEquals(1, metricList.size()); 71 | 72 | Map map = metricList.get(0); 73 | 74 | Assert.assertTrue((long)map.get("startEpoch") >= epochMillis1); 75 | Assert.assertTrue((long)map.get("startEpoch") <= epochMillis2); 76 | Assert.assertTrue((long)map.get("endEpoch") >= epochMillis1); 77 | Assert.assertTrue((long)map.get("endEpoch") <= epochMillis2); 78 | Assert.assertTrue((long)map.get("endEpoch") >= (long)map.get("startEpoch")); 79 | Assert.assertEquals(1L, (long)map.get("count")); 80 | Assert.assertNull(map.get("threadName")); 81 | Assert.assertNull(map.get("threadState")); 82 | Assert.assertArrayEquals(new String[0], ((ArrayList)map.get("stacktrace")).toArray(new String[0])); 83 | 84 | stacktrace = new Stacktrace(); 85 | stacktrace.setThreadName("thread1"); 86 | stacktrace.setThreadState("RUNNING"); 87 | stacktrace.setStack(new ClassAndMethod[] {new ClassAndMethod("class1", "method1"), new ClassAndMethod("class2", "method2")}); 88 | 89 | buffer.appendValue(stacktrace); 90 | buffer.appendValue(stacktrace); 91 | 92 | profiler.profile(); 93 | 94 | Assert.assertEquals(2, nameList.size()); 95 | Assert.assertEquals("Stacktrace", nameList.get(0)); 96 | Assert.assertEquals("Stacktrace", nameList.get(1)); 97 | 98 | Assert.assertEquals(2, metricList.size()); 99 | 100 | map = metricList.get(1); 101 | 102 | Assert.assertEquals(2L, (long)map.get("count")); 103 | Assert.assertEquals("thread1", map.get("threadName")); 104 | Assert.assertEquals("RUNNING", map.get("threadState")); 105 | Assert.assertArrayEquals(new String[]{"class1.method1", "class2.method2"}, ((ArrayList) map.get("stacktrace")).toArray(new String[0])); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/profilers/ThreadInfoProfilerTest.java: -------------------------------------------------------------------------------- 1 | package com.uber.profiling.profilers; 2 | 3 | import com.uber.profiling.Reporter; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | public class ThreadInfoProfilerTest { 12 | @Test 13 | public void profile() { 14 | final List nameList = new ArrayList<>(); 15 | final List> metricList = new ArrayList<>(); 16 | 17 | // create a Profile Instance. 18 | ThreadInfoProfiler profiler = new ThreadInfoProfiler(new Reporter() { 19 | @Override 20 | public void report(String profilerName, Map metrics) { 21 | nameList.add(profilerName); 22 | metricList.add(metrics); 23 | } 24 | 25 | @Override 26 | public void close() { 27 | 28 | } 29 | }); 30 | // Set interval 31 | profiler.setIntervalMillis(150); 32 | Assert.assertEquals(150L, profiler.getIntervalMillis()); 33 | 34 | // run 2 cycles on the profile. 35 | profiler.profile(); 36 | profiler.profile(); 37 | 38 | //start assertion. 39 | Assert.assertEquals(2, nameList.size()); 40 | Assert.assertEquals(ThreadInfoProfiler.PROFILER_NAME, nameList.get(0)); 41 | 42 | Assert.assertEquals(2, metricList.size()); 43 | Assert.assertTrue(metricList.get(0).containsKey("totalStartedThreadCount")); 44 | Assert.assertTrue(metricList.get(0).containsKey("newThreadCount")); 45 | Assert.assertTrue(metricList.get(0).containsKey("liveThreadCount")); 46 | Assert.assertTrue(metricList.get(0).containsKey("peakThreadCount")); 47 | } 48 | } -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/reporters/ConsoleOutputReporterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import org.junit.Test; 20 | 21 | import java.util.HashMap; 22 | 23 | public class ConsoleOutputReporterTest { 24 | @Test 25 | public void report() { 26 | ConsoleOutputReporter reporter = new ConsoleOutputReporter(); 27 | reporter.report("Test", new HashMap()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/transformers/MethodProfilerStaticProxyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.transformers; 18 | 19 | import com.uber.profiling.profilers.Constants; 20 | import com.uber.profiling.profilers.MethodArgumentCollector; 21 | import com.uber.profiling.util.ClassAndMethodMetricKey; 22 | import com.uber.profiling.util.ClassMethodArgumentMetricBuffer; 23 | import junit.framework.Assert; 24 | import org.junit.After; 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | 28 | import java.util.Map; 29 | import java.util.concurrent.atomic.AtomicLong; 30 | 31 | public class MethodProfilerStaticProxyTest { 32 | private ClassMethodArgumentMetricBuffer buffer; 33 | 34 | @Before 35 | public void before() { 36 | buffer = new ClassMethodArgumentMetricBuffer(); 37 | MethodArgumentCollector collector = new MethodArgumentCollector(buffer); 38 | MethodProfilerStaticProxy.setArgumentCollector(collector); 39 | } 40 | 41 | @After 42 | public void after() { 43 | MethodProfilerStaticProxy.setCollector(null); 44 | } 45 | 46 | @Test 47 | public void collectMethodArgument_nullValue() { 48 | MethodProfilerStaticProxy.collectMethodArgument("class1", "method1", 1, null); 49 | MethodProfilerStaticProxy.collectMethodArgument("class1", "method1", 1, null); 50 | 51 | Map metrics = buffer.reset(); 52 | Assert.assertEquals(1, metrics.size()); 53 | ClassAndMethodMetricKey key = metrics.keySet().iterator().next(); 54 | Assert.assertEquals("class1", key.getClassName()); 55 | Assert.assertEquals("method1", key.getMethodName()); 56 | Assert.assertEquals("arg.1.null", key.getMetricName()); 57 | Assert.assertEquals(2, metrics.get(key).intValue()); 58 | } 59 | 60 | @Test 61 | public void collectMethodArgument_veryLongValue() { 62 | StringBuilder sb = new StringBuilder(); 63 | for (int i = 0; i < Constants.MAX_STRING_LENGTH; i++) { 64 | sb.append('a'); 65 | } 66 | sb.append('b'); 67 | String veryLongValue = sb.toString(); 68 | 69 | MethodProfilerStaticProxy.collectMethodArgument("class1", "method1", 1, veryLongValue); 70 | MethodProfilerStaticProxy.collectMethodArgument("class1", "method1", 1, veryLongValue); 71 | 72 | Map metrics = buffer.reset(); 73 | Assert.assertEquals(1, metrics.size()); 74 | ClassAndMethodMetricKey key = metrics.keySet().iterator().next(); 75 | Assert.assertEquals("class1", key.getClassName()); 76 | Assert.assertEquals("method1", key.getMethodName()); 77 | Assert.assertEquals(Constants.MAX_STRING_LENGTH, key.getMetricName().length()); 78 | Assert.assertTrue(key.getMetricName().startsWith("arg.1.")); 79 | Assert.assertEquals(2, metrics.get(key).intValue()); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/ClassAndMethodFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.Arrays; 23 | 24 | public class ClassAndMethodFilterTest { 25 | @Test 26 | public void matchClass() { 27 | ClassAndMethodFilter filter = new ClassAndMethodFilter(null); 28 | Assert.assertFalse(filter.matchMethod("class1", "method1")); 29 | 30 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("class1", ""))); 31 | Assert.assertTrue(filter.matchClass("class1")); 32 | 33 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("", "method1"))); 34 | Assert.assertTrue(filter.matchClass("class1")); 35 | 36 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("class2", "method1"))); 37 | Assert.assertFalse(filter.matchClass("class1")); 38 | Assert.assertTrue(filter.matchClass("class2")); 39 | 40 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("class2", "method1"), 41 | new ClassAndMethod("class1", "method1"))); 42 | Assert.assertTrue(filter.matchClass("class1")); 43 | Assert.assertTrue(filter.matchClass("class2")); 44 | 45 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("class2", "method1"), 46 | new ClassAndMethod("class1", "method1"), 47 | new ClassAndMethod("class3", "*"))); 48 | Assert.assertTrue(filter.matchClass("class1xx")); 49 | Assert.assertTrue(filter.matchClass("class2xx")); 50 | Assert.assertTrue(filter.matchClass("class3xx")); 51 | } 52 | 53 | @Test 54 | public void matchMethod() { 55 | ClassAndMethodFilter filter = new ClassAndMethodFilter(null); 56 | Assert.assertFalse(filter.matchMethod("class1", "method1")); 57 | 58 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("class1", ""))); 59 | Assert.assertFalse(filter.matchMethod("class1", "method1")); 60 | 61 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("", "method1"))); 62 | Assert.assertTrue(filter.matchMethod("class1", "method1")); 63 | 64 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("class2", "method1"))); 65 | Assert.assertFalse(filter.matchMethod("class1", "method1")); 66 | 67 | filter = new ClassAndMethodFilter(Arrays.asList(new ClassAndMethod("", "method1"), 68 | new ClassAndMethod("class1", "method1"))); 69 | Assert.assertTrue(filter.matchMethod("class1", "method1")); 70 | } 71 | 72 | @Test 73 | public void matchMethod_wildcard() { 74 | ClassAndMethodFilter filter = new ClassAndMethodFilter( 75 | Arrays.asList(new ClassAndMethod("class1", ""))); 76 | Assert.assertFalse(filter.matchMethod("class1", "method1")); 77 | 78 | filter = new ClassAndMethodFilter( 79 | Arrays.asList(new ClassAndMethod("class1", ""), 80 | new ClassAndMethod("class1", "*"))); 81 | Assert.assertTrue(filter.matchMethod("class1", "method1")); 82 | } 83 | 84 | @Test 85 | public void matchMethod_prefix() { 86 | ClassAndMethodFilter filter = new ClassAndMethodFilter( 87 | Arrays.asList(new ClassAndMethod("package11.class1", "method1"), 88 | new ClassAndMethod("package22", "method2"))); 89 | Assert.assertTrue(filter.matchMethod("package22.class2", "method2")); 90 | Assert.assertFalse(filter.matchMethod("package2", "method2")); 91 | 92 | filter = new ClassAndMethodFilter( 93 | Arrays.asList(new ClassAndMethod("package11.class1", "method1"), 94 | new ClassAndMethod("package22", "method2"), 95 | new ClassAndMethod("package33", "*"))); 96 | Assert.assertTrue(filter.matchMethod("package22.class2", "method2")); 97 | Assert.assertFalse(filter.matchMethod("package2", "method2")); 98 | Assert.assertTrue(filter.matchMethod("package33.xx.yy", "method3")); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/ClassAndMethodMetricBufferTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.Map; 23 | 24 | public class ClassAndMethodMetricBufferTest { 25 | @Test 26 | public void appendValue() { 27 | ClassAndMethodLongMetricBuffer buffer = new ClassAndMethodLongMetricBuffer(); 28 | buffer.appendValue("class1", "method1", "metric1", 11); 29 | buffer.appendValue("class1", "method2", "metric1", 22); 30 | buffer.appendValue("class1", "method2", "metric1", 55); 31 | buffer.appendValue("class2", "method2", "metric1", 1001); 32 | 33 | Map map = buffer.reset(); 34 | Assert.assertEquals(3, map.size()); 35 | 36 | Histogram histogram = map.get(new ClassAndMethodMetricKey("class1", "method1", "metric1")); 37 | Assert.assertEquals(1, histogram.getCount()); 38 | Assert.assertEquals(11, histogram.getSum()); 39 | Assert.assertEquals(11, histogram.getMin()); 40 | Assert.assertEquals(11, histogram.getMax()); 41 | 42 | histogram = map.get(new ClassAndMethodMetricKey("class1", "method2", "metric1")); 43 | Assert.assertEquals(2, histogram.getCount()); 44 | Assert.assertEquals(77, histogram.getSum()); 45 | Assert.assertEquals(22, histogram.getMin()); 46 | Assert.assertEquals(55, histogram.getMax()); 47 | 48 | histogram = map.get(new ClassAndMethodMetricKey("class2", "method2", "metric1")); 49 | Assert.assertEquals(1, histogram.getCount()); 50 | Assert.assertEquals(1001, histogram.getSum()); 51 | Assert.assertEquals(1001, histogram.getMin()); 52 | Assert.assertEquals(1001, histogram.getMax()); 53 | 54 | map = buffer.reset(); 55 | Assert.assertEquals(0, map.size()); 56 | 57 | map = buffer.reset(); 58 | Assert.assertEquals(0, map.size()); 59 | } 60 | 61 | @Test 62 | public void appendValue_concurrent() throws InterruptedException { 63 | ClassAndMethodLongMetricBuffer buffer = new ClassAndMethodLongMetricBuffer(); 64 | 65 | String[] classNames = new String[]{"class1", "class2", "class1", "class2", "class101"}; 66 | String[] methodNames = new String[]{"method1", "method2", "method1", "method3", "method101"}; 67 | int[] values = new int[]{1, 2, 10, 20, 101}; 68 | 69 | Thread[] threads = new Thread[classNames.length]; 70 | 71 | int repeatTimes = 1000000; 72 | 73 | for (int i = 0; i < threads.length; i++) { 74 | final int index = i; 75 | Thread thread = new Thread(() -> { 76 | for (int repeat = 0; repeat < repeatTimes; repeat++) { 77 | buffer.appendValue(classNames[index], methodNames[index], "duration", values[index]); 78 | } 79 | }); 80 | threads[i] = thread; 81 | } 82 | 83 | for (int i = 0; i < threads.length; i++) { 84 | threads[i].start(); 85 | } 86 | 87 | for (int i = 0; i < threads.length; i++) { 88 | threads[i].join(); 89 | } 90 | 91 | Map result = buffer.reset(); 92 | Assert.assertEquals(4, result.size()); 93 | 94 | Assert.assertEquals(2 * repeatTimes, 95 | result.get(new ClassAndMethodMetricKey("class1", "method1", "duration")).getCount()); 96 | Assert.assertEquals(11 * repeatTimes, 97 | result.get(new ClassAndMethodMetricKey("class1", "method1", "duration")).getSum()); 98 | Assert.assertEquals(1, 99 | result.get(new ClassAndMethodMetricKey("class1", "method1", "duration")).getMin()); 100 | Assert.assertEquals(10, 101 | result.get(new ClassAndMethodMetricKey("class1", "method1", "duration")).getMax()); 102 | 103 | Assert.assertEquals(repeatTimes, 104 | result.get(new ClassAndMethodMetricKey("class2", "method2", "duration")).getCount()); 105 | Assert.assertEquals(2 * repeatTimes, 106 | result.get(new ClassAndMethodMetricKey("class2", "method2", "duration")).getSum()); 107 | Assert.assertEquals(2, 108 | result.get(new ClassAndMethodMetricKey("class2", "method2", "duration")).getMin()); 109 | Assert.assertEquals(2, 110 | result.get(new ClassAndMethodMetricKey("class2", "method2", "duration")).getMax()); 111 | 112 | Assert.assertEquals(repeatTimes, 113 | result.get(new ClassAndMethodMetricKey("class2", "method3", "duration")).getCount()); 114 | Assert.assertEquals(20 * repeatTimes, 115 | result.get(new ClassAndMethodMetricKey("class2", "method3", "duration")).getSum()); 116 | Assert.assertEquals(20, 117 | result.get(new ClassAndMethodMetricKey("class2", "method3", "duration")).getMin()); 118 | Assert.assertEquals(20, 119 | result.get(new ClassAndMethodMetricKey("class2", "method3", "duration")).getMax()); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/ClassMethodArgmentMetricBufferTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.Map; 23 | import java.util.concurrent.atomic.AtomicLong; 24 | 25 | public class ClassMethodArgmentMetricBufferTest { 26 | @Test 27 | public void appendValue() { 28 | ClassMethodArgumentMetricBuffer buffer = new ClassMethodArgumentMetricBuffer(); 29 | buffer.appendValue("class1", "method1", "arg1"); 30 | buffer.appendValue("class1", "method2", "arg1"); 31 | buffer.appendValue("class1", "method2", "arg1"); 32 | buffer.appendValue("class2", "method2", "arg1"); 33 | 34 | Map map = buffer.reset(); 35 | Assert.assertEquals(3, map.size()); 36 | 37 | AtomicLong count = map.get(new ClassAndMethodMetricKey("class1", "method1", "arg1")); 38 | Assert.assertEquals(1, count.get()); 39 | 40 | count = map.get(new ClassAndMethodMetricKey("class1", "method2", "arg1")); 41 | Assert.assertEquals(2, count.get()); 42 | 43 | count = map.get(new ClassAndMethodMetricKey("class2", "method2", "arg1")); 44 | Assert.assertEquals(1, count.get()); 45 | 46 | map = buffer.reset(); 47 | Assert.assertEquals(0, map.size()); 48 | 49 | map = buffer.reset(); 50 | Assert.assertEquals(0, map.size()); 51 | } 52 | 53 | @Test 54 | public void appendValue_concurrent() throws InterruptedException { 55 | ClassMethodArgumentMetricBuffer buffer = new ClassMethodArgumentMetricBuffer(); 56 | 57 | String[] classNames = new String[]{"class1", "class2", "class1", "class2", "class101"}; 58 | String[] methodNames = new String[]{"method1", "method2", "method1", "method3", "method101"}; 59 | 60 | Thread[] threads = new Thread[classNames.length]; 61 | 62 | int repeatTimes = 1000000; 63 | 64 | for (int i = 0; i < threads.length; i++) { 65 | final int index = i; 66 | Thread thread = new Thread(() -> { 67 | for (int repeat = 0; repeat < repeatTimes; repeat++) { 68 | buffer.appendValue(classNames[index], methodNames[index], "arg1"); 69 | } 70 | }); 71 | threads[i] = thread; 72 | } 73 | 74 | for (int i = 0; i < threads.length; i++) { 75 | threads[i].start(); 76 | } 77 | 78 | for (int i = 0; i < threads.length; i++) { 79 | threads[i].join(); 80 | } 81 | 82 | Map result = buffer.reset(); 83 | Assert.assertEquals(4, result.size()); 84 | 85 | Assert.assertEquals(2 * repeatTimes, 86 | result.get(new ClassAndMethodMetricKey("class1", "method1", "arg1")).get()); 87 | 88 | Assert.assertEquals(repeatTimes, 89 | result.get(new ClassAndMethodMetricKey("class2", "method2", "arg1")).get()); 90 | 91 | Assert.assertEquals(repeatTimes, 92 | result.get(new ClassAndMethodMetricKey("class2", "method3", "arg1")).get()); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/ClassMethodArgumentFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.Arrays; 23 | 24 | public class ClassMethodArgumentFilterTest { 25 | @Test 26 | public void matchClass() { 27 | ClassMethodArgumentFilter filter = new ClassMethodArgumentFilter(null); 28 | Assert.assertEquals(0, filter.matchMethod("class1", "method1").size()); 29 | 30 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("class1", "", 1))); 31 | Assert.assertTrue(filter.matchClass("class1")); 32 | 33 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("", "method1", 1))); 34 | Assert.assertTrue(filter.matchClass("class1")); 35 | 36 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("class2", "method1", 1))); 37 | Assert.assertFalse(filter.matchClass("class1")); 38 | Assert.assertTrue(filter.matchClass("class2")); 39 | 40 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("class2", "method1", 1), 41 | new ClassMethodArgument("class1", "method1", 1))); 42 | Assert.assertTrue(filter.matchClass("class1")); 43 | Assert.assertTrue(filter.matchClass("class2")); 44 | 45 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("class2", "method1", 1), 46 | new ClassMethodArgument("class1", "method1", 1), 47 | new ClassMethodArgument("class3", "*", 1))); 48 | Assert.assertTrue(filter.matchClass("class1xx")); 49 | Assert.assertTrue(filter.matchClass("class2xx")); 50 | Assert.assertTrue(filter.matchClass("class3xx")); 51 | } 52 | 53 | @Test 54 | public void matchMethod() { 55 | ClassMethodArgumentFilter filter = new ClassMethodArgumentFilter(null); 56 | Assert.assertEquals(0, filter.matchMethod("class1", "method1").size()); 57 | 58 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("class1", "", 1))); 59 | Assert.assertEquals(0, filter.matchMethod("class1", "method1").size()); 60 | 61 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("", "method1", 10))); 62 | Assert.assertEquals(1, filter.matchMethod("class1", "method1").size()); 63 | Assert.assertEquals(10, filter.matchMethod("class1", "method1").get(0).intValue()); 64 | 65 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("class2", "method1", 1))); 66 | Assert.assertEquals(0, filter.matchMethod("class1", "method1").size()); 67 | 68 | filter = new ClassMethodArgumentFilter(Arrays.asList(new ClassMethodArgument("", "method1", 10), 69 | new ClassMethodArgument("class1", "method1", 2))); 70 | Assert.assertEquals(2, filter.matchMethod("class1", "method1").size()); 71 | Assert.assertEquals(10, filter.matchMethod("class1", "method1").get(0).intValue()); 72 | Assert.assertEquals(2, filter.matchMethod("class1", "method1").get(1).intValue()); 73 | } 74 | 75 | @Test 76 | public void matchMethod_wildcard() { 77 | ClassMethodArgumentFilter filter = new ClassMethodArgumentFilter( 78 | Arrays.asList(new ClassMethodArgument("class1", "", 10))); 79 | Assert.assertEquals(0, filter.matchMethod("class1", "method1").size()); 80 | 81 | filter = new ClassMethodArgumentFilter( 82 | Arrays.asList(new ClassMethodArgument("class1", "", 10), 83 | new ClassMethodArgument("class1", "*", 20))); 84 | Assert.assertEquals(1, filter.matchMethod("class1", "method1").size()); 85 | Assert.assertEquals(20, filter.matchMethod("class1", "method1").get(0).intValue()); 86 | } 87 | 88 | @Test 89 | public void matchMethod_prefix() { 90 | ClassMethodArgumentFilter filter = new ClassMethodArgumentFilter( 91 | Arrays.asList(new ClassMethodArgument("package11.class1", "method1", 10), 92 | new ClassMethodArgument("package22", "method2", 20))); 93 | Assert.assertEquals(1, filter.matchMethod("package22.class2", "method2").size()); 94 | Assert.assertEquals(0, filter.matchMethod("package2", "method2").size()); 95 | 96 | filter = new ClassMethodArgumentFilter( 97 | Arrays.asList(new ClassMethodArgument("package11.class1", "method1", 10), 98 | new ClassMethodArgument("package22", "method2", 20), 99 | new ClassMethodArgument("package33", "*", 30))); 100 | Assert.assertEquals(1, filter.matchMethod("package22.class2", "method2").size()); 101 | Assert.assertEquals(20, filter.matchMethod("package22.class2", "method2").get(0).intValue()); 102 | Assert.assertEquals(0, filter.matchMethod("package2", "method2").size()); 103 | Assert.assertEquals(1, filter.matchMethod("package33.xx.yy", "method3").size()); 104 | Assert.assertEquals(30, filter.matchMethod("package33.xx.yy", "method3").get(0).intValue()); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/IOUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.io.ByteArrayInputStream; 23 | 24 | public class IOUtilsTest { 25 | @Test 26 | public void toByteArray() { 27 | byte[] bytes = new byte[]{1, 2, 3}; 28 | ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); 29 | byte[] result = IOUtils.toByteArray(byteArrayInputStream); 30 | Assert.assertEquals(3, result.length); 31 | Assert.assertEquals(1, result[0]); 32 | Assert.assertEquals(3, result[result.length - 1]); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/ReflectionUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.lang.reflect.InvocationTargetException; 23 | 24 | public class ReflectionUtilsTest { 25 | static class ClassA { 26 | public String method1() { 27 | return "hello"; 28 | } 29 | } 30 | 31 | static class ClassB { 32 | public static ClassA getClassA() { 33 | return new ClassA(); 34 | } 35 | } 36 | 37 | @Test 38 | public void executeStaticMethods() throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { 39 | Object result = ReflectionUtils.executeStaticMethods("com.uber.profiling.util.ReflectionUtilsTest$ClassB", "getClassA.method1"); 40 | Assert.assertEquals("hello", result); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/com/uber/profiling/util/SparkUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.util; 18 | 19 | import com.uber.profiling.Arguments; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.lang.reflect.InvocationTargetException; 24 | 25 | public class SparkUtilsTest { 26 | @Test 27 | public void probeAppId() { 28 | Assert.assertNull(SparkUtils.probeAppId(Arguments.ARG_APP_ID_REGEX)); 29 | Assert.assertEquals("jar", SparkUtils.probeAppId("jar")); 30 | } 31 | 32 | @Test 33 | public void getAppId() throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { 34 | Assert.assertNull(SparkUtils.getSparkEnvAppId()); 35 | } 36 | 37 | @Test 38 | public void probeRole() { 39 | Assert.assertEquals("executor", SparkUtils.probeRole("java org.apache.spark.executor.CoarseGrainedExecutorBackend")); 40 | Assert.assertEquals("driver", SparkUtils.probeRole("java org.apache.spark.MockDriver")); 41 | Assert.assertEquals(null, SparkUtils.probeRole("java foo")); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java_jdbc/com/uber/profiling/reporters/CpuAndMemoryProfilerMetricDaoTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import java.io.File; 20 | import java.io.IOException; 21 | import java.util.Arrays; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | import org.junit.Assert; 26 | import org.junit.Test; 27 | 28 | public class CpuAndMemoryProfilerMetricDaoTest { 29 | @Test 30 | public void test() throws IOException { 31 | File file = File.createTempFile("h2dbfile", ".db"); 32 | file.deleteOnExit(); 33 | 34 | String connectionString = 35 | String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath()); 36 | 37 | CpuAndMemoryProfilerMetricDao dao = new CpuAndMemoryProfilerMetricDao("org.h2.Driver", connectionString, "test_CpuAndMemoryProfilerMetricDao_table"); 38 | dao.createTable(); 39 | dao.queryColumns(1000, "*"); 40 | 41 | CpuAndMemoryProfilerMetric entity1 = new CpuAndMemoryProfilerMetric(); 42 | entity1.setEpochMillis(1535651091000L); 43 | entity1.setName("name01"); 44 | entity1.setHost("host01"); 45 | entity1.setProcessUuid("process01"); 46 | entity1.setAppId("app01"); 47 | entity1.setTag("tag01"); 48 | entity1.setProcessCpuLoad(0.111); 49 | entity1.setHeapMemoryCommitted(1000L); 50 | 51 | dao.insertOrUpdate(entity1); 52 | 53 | CpuAndMemoryProfilerMetric entity2 = new CpuAndMemoryProfilerMetric(); 54 | entity2.setEpochMillis(1535651091000L); 55 | entity2.setName("name02"); 56 | entity2.setHost("host02"); 57 | entity2.setProcessUuid("process02"); 58 | entity2.setAppId("app02"); 59 | entity2.setTag("tag02"); 60 | entity2.setProcessCpuLoad(0.222); 61 | entity2.setHeapMemoryCommitted(2000L); 62 | 63 | dao.insertOrUpdate(entity2); 64 | 65 | Assert.assertEquals(2, dao.getTotalCount()); 66 | 67 | CpuAndMemoryProfilerMetric readback1 = dao.getByPrimaryKeys( 68 | Arrays.asList( 69 | entity1.getEpochMillis(), 70 | entity1.getName(), 71 | entity1.getHost(), 72 | entity1.getProcessUuid(), 73 | entity1.getAppId()), 74 | CpuAndMemoryProfilerMetric.class); 75 | 76 | Assert.assertEquals(entity1.getEpochMillis(), readback1.getEpochMillis()); 77 | Assert.assertEquals(entity1.getName(), readback1.getName()); 78 | Assert.assertEquals(entity1.getHost(), readback1.getHost()); 79 | Assert.assertEquals(entity1.getProcessUuid(), readback1.getProcessUuid()); 80 | Assert.assertEquals(entity1.getAppId(), readback1.getAppId()); 81 | Assert.assertEquals(entity1.getTag(), readback1.getTag()); 82 | Assert.assertEquals(entity1.getProcessCpuLoad(), readback1.getProcessCpuLoad()); 83 | Assert.assertEquals(entity1.getHeapMemoryCommitted(), readback1.getHeapMemoryCommitted()); 84 | 85 | CpuAndMemoryProfilerMetric readback2 = dao.getByPrimaryKeys( 86 | Arrays.asList( 87 | entity2.getEpochMillis(), 88 | entity2.getName(), 89 | entity2.getHost(), 90 | entity2.getProcessUuid(), 91 | entity2.getAppId()), 92 | CpuAndMemoryProfilerMetric.class); 93 | 94 | Assert.assertEquals(entity2.getEpochMillis(), readback2.getEpochMillis()); 95 | Assert.assertEquals(entity2.getName(), readback2.getName()); 96 | Assert.assertEquals(entity2.getHost(), readback2.getHost()); 97 | Assert.assertEquals(entity2.getProcessUuid(), readback2.getProcessUuid()); 98 | Assert.assertEquals(entity2.getAppId(), readback2.getAppId()); 99 | Assert.assertEquals(entity2.getTag(), readback2.getTag()); 100 | Assert.assertEquals(entity2.getProcessCpuLoad(), readback2.getProcessCpuLoad()); 101 | Assert.assertEquals(entity2.getHeapMemoryCommitted(), readback2.getHeapMemoryCommitted()); 102 | 103 | dao.close(); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/test/java_jdbc/com/uber/profiling/reporters/JdbcOutputReporterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.profiling.reporters; 18 | 19 | import com.uber.profiling.profilers.CpuAndMemoryProfiler; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | import java.util.Arrays; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class JdbcOutputReporterTest { 31 | @Test 32 | public void test() throws IOException { 33 | File file = File.createTempFile("h2dbfile", ".db"); 34 | file.deleteOnExit(); 35 | 36 | String connectionString = 37 | String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath()); 38 | String tableName = "test_cpuAndMemoryProfilerTable"; 39 | 40 | JdbcOutputReporter reporter = new JdbcOutputReporter(); 41 | 42 | Map> arguments = new HashMap<>(); 43 | arguments.put("driverClass", Arrays.asList("org.h2.Driver")); 44 | arguments.put("connectionString", Arrays.asList(connectionString)); 45 | arguments.put("cpuAndMemoryProfilerTableName", Arrays.asList(tableName)); 46 | arguments.put("tablePartition", Arrays.asList("false")); 47 | 48 | reporter.updateArguments(arguments); 49 | 50 | Map map = new HashMap<>(); 51 | 52 | map.put("epochMillis", 1603833802000L); 53 | map.put("name", "process1"); 54 | map.put("host", "host1"); 55 | map.put("processUuid", "uuid1"); 56 | map.put("appId", "app1"); 57 | map.put("tag", null); 58 | map.put("role", "role1"); 59 | map.put("processCpuLoad", 0.2); 60 | map.put("systemCpuLoad", 0.3); 61 | map.put("processCpuTime", 1001L); 62 | map.put("heapMemoryTotalUsed", 2002L); 63 | map.put("heapMemoryCommitted", 3003L); 64 | map.put("heapMemoryMax", 4004L); 65 | map.put("nonHeapMemoryCommitted", 5005L); 66 | map.put("nonHeapMemoryTotalUsed", 6006L); 67 | map.put("nonHeapMemoryMax", 7007L); 68 | map.put("vmRSS", 8001L); 69 | map.put("vmHWM", 8002L); 70 | map.put("vmSize", 8003L); 71 | map.put("vmPeak", 8004L); 72 | 73 | reporter.report(CpuAndMemoryProfiler.PROFILER_NAME, map); 74 | reporter.report(CpuAndMemoryProfiler.PROFILER_NAME, map); 75 | 76 | map.put("epochMillis", 1603833802001L); 77 | reporter.report(CpuAndMemoryProfiler.PROFILER_NAME, map); 78 | 79 | reporter.close(); 80 | 81 | try (CpuAndMemoryProfilerMetricDao dao = new CpuAndMemoryProfilerMetricDao("org.h2.Driver", connectionString, tableName)) { 82 | Assert.assertEquals(2, dao.getTotalCount()); 83 | 84 | CpuAndMemoryProfilerMetric entity = dao.getByPrimaryKeys( 85 | Arrays.asList(1603833802001L, "process1", "host1", "uuid1", "app1"), 86 | CpuAndMemoryProfilerMetric.class); 87 | Assert.assertNotNull(entity); 88 | Assert.assertEquals(null, entity.getTag()); 89 | Assert.assertEquals((Double)0.2, entity.getProcessCpuLoad()); 90 | Assert.assertEquals((Double)0.3, entity.getSystemCpuLoad()); 91 | Assert.assertEquals((Long)1001L, entity.getProcessCpuTime()); 92 | Assert.assertEquals((Long)2002L, entity.getHeapMemoryTotalUsed()); 93 | Assert.assertEquals((Long)3003L, entity.getHeapMemoryCommitted()); 94 | Assert.assertEquals((Long)4004L, entity.getHeapMemoryMax()); 95 | Assert.assertEquals((Long)5005L, entity.getNonHeapMemoryCommitted()); 96 | Assert.assertEquals((Long)6006L, entity.getNonHeapMemoryTotalUsed()); 97 | Assert.assertEquals((Long)7007L, entity.getNonHeapMemoryMax()); 98 | Assert.assertEquals((Long)8001L, entity.getVmRSS()); 99 | Assert.assertEquals((Long)8002L, entity.getVmHWM()); 100 | Assert.assertEquals((Long)8003L, entity.getVmSize()); 101 | Assert.assertEquals((Long)8004L, entity.getVmPeak()); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /stackcollapse.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import json 4 | 5 | parser = argparse.ArgumentParser(description = 'Collapse stack trace logs to FlameGraph input file.') 6 | parser.add_argument("--input", "-i", type=str, required=True, help="Input stack trace file") 7 | 8 | args = parser.parse_args() 9 | inputFile = args.input 10 | 11 | with open(inputFile) as f: 12 | stacktraceDict = {} 13 | 14 | for line in f.readlines(): 15 | stacktraceLog = json.loads(line) 16 | 17 | assert 'stacktrace' in stacktraceLog, "Malformated json. 'stacktrace' key doesn't exist." 18 | stacktrace = stacktraceLog['stacktrace'] 19 | 20 | if len(stacktrace) == 0: 21 | continue 22 | 23 | assert 'count' in stacktraceLog, "Malformated json. 'count' key doesn't exist." 24 | count = stacktraceLog['count'] 25 | 26 | key = ';'.join(list(reversed(stacktrace))) 27 | if key in stacktraceDict: 28 | stacktraceDict[key] += count 29 | else: 30 | stacktraceDict[key] = count 31 | 32 | for stacktraceItem in stacktraceDict: 33 | print("%s %i" % (stacktraceItem, stacktraceDict[stacktraceItem])) 34 | --------------------------------------------------------------------------------