├── .gitignore
├── src
└── main
│ ├── resources
│ ├── index.jelly
│ └── blackboard
│ │ └── test
│ │ └── jenkins
│ │ └── jmhbenchmark
│ │ ├── BenchmarkProjectAction
│ │ └── index.jelly
│ │ ├── BenchmarkPublisher
│ │ └── config.jelly
│ │ └── BenchmarkBuildActionDisplay
│ │ └── index.jelly
│ ├── java
│ └── blackboard
│ │ └── test
│ │ └── jenkins
│ │ └── jmhbenchmark
│ │ ├── ReportParser.java
│ │ ├── BenchmarkBuildActionDisplay.java
│ │ ├── BenchmarkTrend.java
│ │ ├── BenchmarkReport.java
│ │ ├── BenchmarkBuildAction.java
│ │ ├── BenchmarkResult.java
│ │ ├── CsvParser.java
│ │ ├── BenchmarkProjectAction.java
│ │ └── BenchmarkPublisher.java
│ └── webapp
│ └── css
│ └── style.css
├── pom.xml
└── readme.md
/.gitignore:
--------------------------------------------------------------------------------
1 | /.settings
2 | /target
3 | /work
4 | /.classpath
5 | /.project
6 | /bin
7 |
--------------------------------------------------------------------------------
/src/main/resources/index.jelly:
--------------------------------------------------------------------------------
1 |
4 |
5 | This plugin publishes JMH benchmark result to each build and provides a visualization for trending benchmark result across build. It also compares performance degradation/gain of a benchmark between successive builds and mark a build unstable if the degradation is below the specified threshold.
6 |
7 |
--------------------------------------------------------------------------------
/src/main/java/blackboard/test/jenkins/jmhbenchmark/ReportParser.java:
--------------------------------------------------------------------------------
1 | package blackboard.test.jenkins.jmhbenchmark;
2 |
3 | import hudson.model.AbstractBuild;
4 | import hudson.model.TaskListener;
5 |
6 | import java.io.File;
7 | import java.io.IOException;
8 |
9 | public abstract class ReportParser
10 | {
11 | public abstract BenchmarkReport parse( AbstractBuild build, File report, TaskListener listener )
12 | throws IOException;
13 |
14 | }
15 |
--------------------------------------------------------------------------------
/src/main/resources/blackboard/test/jenkins/jmhbenchmark/BenchmarkProjectAction/index.jelly:
--------------------------------------------------------------------------------
1 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 | 
11 |
12 |
13 |
14 |
11 |
12 | | No. |
13 |
14 | ${column} |
15 |
16 |
17 | ${columnParams} |
18 |
19 |
20 |
21 |
22 | |
23 | ${entry.value.shortBenchmarkName} |
24 | ${entry.value.mode} |
25 | ${entry.value.threads} |
26 | ${entry.value.samples} |
27 | ${it.getFormattedNumber(entry.value.mean)} |
28 | ${it.getFormattedNumber(entry.value.meanError)} |
29 | ${entry.value.unit} |
30 |
31 |
32 | ( ${entry.value.meanChangeFromPrev} / ${entry.value.meanChangeFromBaseline} ) |
33 |
34 |
35 | ( ${entry.value.meanChangeFromPrev} / ${entry.value.meanChangeFromBaseline} ) |
36 |
37 |
38 | ( ${entry.value.meanChangeFromPrev} / ${entry.value.meanChangeFromBaseline} ) |
39 |
40 |
41 |
42 | ${entry.value.params[columnParams]} |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/src/main/java/blackboard/test/jenkins/jmhbenchmark/BenchmarkBuildAction.java:
--------------------------------------------------------------------------------
1 | package blackboard.test.jenkins.jmhbenchmark;
2 |
3 | import hudson.model.Action;
4 | import hudson.model.AbstractBuild;
5 | import hudson.util.StreamTaskListener;
6 |
7 | import java.lang.ref.WeakReference;
8 |
9 | import org.kohsuke.stapler.StaplerProxy;
10 |
11 | /**
12 | * The {@link Action} that is executed at the build level. It creates a link (i.e. JMH Benchmark Report) at the
13 | * left menu of the build page. When this link is clicked, the benchmark report ({@link BenchmarkReport}) for
14 | * the selected build is displayed.
15 | *
16 | */
17 | public class BenchmarkBuildAction implements Action, StaplerProxy
18 | {
19 | private final AbstractBuild _build;
20 | private final BenchmarkReport _performanceReport;
21 | private final int _decimalPlaces;
22 | private transient WeakReference _buildActionDisplay;
23 |
24 | public BenchmarkBuildAction( AbstractBuild pBuild, BenchmarkReport performanceReport, int decimalPlaces )
25 | {
26 | _build = pBuild;
27 | _performanceReport = performanceReport;
28 | _decimalPlaces = decimalPlaces;
29 | }
30 |
31 | public String getIconFileName()
32 | {
33 | return "graph.gif";
34 | }
35 |
36 | public String getDisplayName()
37 | {
38 | return "JMH Benchmark Report";
39 | }
40 |
41 | public String getUrlName()
42 | {
43 | return "jmhbenchmark";
44 | }
45 |
46 | public AbstractBuild getBuild()
47 | {
48 | return _build;
49 | }
50 |
51 | public BenchmarkReport getPerformanceReport()
52 | {
53 | return _performanceReport;
54 | }
55 |
56 | public BenchmarkBuildActionDisplay getBuildActionDisplay()
57 | {
58 | BenchmarkBuildActionDisplay buildDisplay = null;
59 | WeakReference wr = _buildActionDisplay;
60 |
61 | if ( wr != null )
62 | {
63 | buildDisplay = wr.get();
64 | if ( buildDisplay != null )
65 | return buildDisplay;
66 | }
67 |
68 | buildDisplay = new BenchmarkBuildActionDisplay( this, StreamTaskListener.fromStdout(), _decimalPlaces );
69 |
70 | _buildActionDisplay = new WeakReference( buildDisplay );
71 | return buildDisplay;
72 | }
73 |
74 | public void setBuildActionDisplay( WeakReference buildActionDisplay )
75 | {
76 | _buildActionDisplay = buildActionDisplay;
77 | }
78 |
79 | public BenchmarkBuildActionDisplay getTarget()
80 | {
81 | return getBuildActionDisplay();
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | ### JMH Benchmark Jenkins Plugin ###
2 |
3 | The JMH Benchmark plugin allows you to integrate a JMH benchmark result with jenkins by publishing the result to each build and provides a visualization for trending build data. It also compares performance degradation/gain between a given build and the previous and baseline builds.
4 |
5 | JMH is a Java harness for building, running, and analyzing nano/micro/milli/macro benchmarks written in Java and other languages targetting the JVM. For more details about JMH, see http://openjdk.java.net/projects/code-tools/jmh/.
6 |
7 | ### Continuous Integration with the JMH Benchmark Jenkins Plugin ###
8 |
9 | 1. As a build step, the JMH benchmark tests are run using a build automation tool such as Gradle. The test results are saved as a CSV format into a local file where the file location is specified relative to the `WORKSPACE` of the Jenkins project. The raw benchmark result in the CSV format is also copied to the master. As an example, if you use the JMH Gradle plugin, available in https://github.com/blackboard/jmh-gradle-plugin, here is how you may configure the build step.
10 | * *Switches*: `-P-rf=csv -P-rff="${WORKSPACE}/learn-apis-platform_mainline-jmh-benchmark.csv"`
11 | * *Tasks*: `benchmarkJmh`
12 | * *Build File*: `mainline/projects/build.gradle`
13 | 2. As a post-build action, the JMH Benchmark plugin will post the benchmark results to each build. Currently, the configuration accepts four input parameters:
14 | * *Baseline Build Number* - the build number that will be used as a baseline. `0` is the default value if no baseline exists..
15 | * *Performance Degradation Threshold (in %)* - this threshold applies between the current and previous successful build as well the current and baseline build if the latter is specified. The default threshold is -20%.
16 | * *Performance Increase Threshold (in %)* - this threshold is an indicator for a performance improvement in the current build compared to the previous successful build and the baseline build if baseline is defined. The default threshold is +20%
17 | * *Decimal Places in Benchmark Report* - the number of decimal places used in the benchmark report.
18 |
19 | The plugin provides the following two links to view the build and trend data:
20 |
21 | * *JMH Benchmark Report* - this is accessed for each build and the benchmark output is available in a tabular form for a given build. In addition to the benchmark report, data on the the percentage gain/loss of each benchmark is given in comparison to the previous and baseline builds.
22 | * *JMH Report Trend* - this is accessed from the project page. This report trends data in a visual form for each benchmark over a specified number of past builds.
23 |
24 | *Note:* currently, the plugin can mark a build as unstable if at least one benchmark has a performance less than the degradation threshold. But, the plugin doesn't fail a build based on the benchmark test result.
25 |
26 |
27 | ### Testing the plugin in Jenkins ###
28 |
29 | * Download the source and build it: `$ mvn clean install`. *jmhbenchmark.hpi* is created under the target folder.
30 | * Install `jmhbenchmark.hpi` in Jenkins.
31 |
32 | ### Development ###
33 |
34 | To import the project in Eclipse and develop:
35 |
36 | * Run `$ mvn -DdownloadSources=true -DdownloadJavadocs=true -DoutputDirectory=target/eclipse-classes eclipse:eclipse`
37 | * Use "Import..." (under the File menu in Eclipse) and select "General" > "Existing Projects into Workspace".
38 | * Install Maven Integration for Eclipse plugin.
--------------------------------------------------------------------------------
/src/main/java/blackboard/test/jenkins/jmhbenchmark/BenchmarkResult.java:
--------------------------------------------------------------------------------
1 | package blackboard.test.jenkins.jmhbenchmark;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | /**
7 | * BenchmarkResult contains properties associated with a JMH benchmark such as name of the benchmark, mode the benchmark is run,
8 | * number of threads, samples, mean, mean error, unit, list of parameters if any. Since the benchmark is also expected to run in
9 | * a Continuous Integration, BenchmarkResult also contains data on the percentage change of the benchmark score from previous
10 | * builds and from a baseline build.
11 | *
12 | */
13 | public class BenchmarkResult
14 | {
15 | private String _benchmarkName;
16 | private String _shortBenchmarkName;
17 | private String _mode;
18 | private int _threads;
19 | private int _samples;
20 | private double _mean;
21 | private double _meanError;
22 | private String _unit;
23 | private Map _params;
24 | private double _meanChangeFromPrev;
25 | private double _meanChangeFromBaseline;
26 | private String _changeIndicator;
27 |
28 | public BenchmarkResult()
29 | {
30 | _params = new HashMap();
31 | _changeIndicator = "";
32 | }
33 |
34 | public String getBenchmarkName()
35 | {
36 | return _benchmarkName;
37 | }
38 |
39 | public void setBenchmarkName( String benchmarkName )
40 | {
41 | _benchmarkName = benchmarkName;
42 | }
43 |
44 | public String getShortBenchmarkName()
45 | {
46 | return _shortBenchmarkName;
47 | }
48 |
49 | public void setShortBenchmarkName( String shortBenchmarkName )
50 | {
51 | _shortBenchmarkName = shortBenchmarkName;
52 | }
53 |
54 | public String getMode()
55 | {
56 | return _mode;
57 | }
58 |
59 | public void setMode( String mode )
60 | {
61 | _mode = mode;
62 | }
63 |
64 | public int getThreads()
65 | {
66 | return _threads;
67 | }
68 |
69 | public void setThreads( int threads )
70 | {
71 | _threads = threads;
72 | }
73 |
74 | public int getSamples()
75 | {
76 | return _samples;
77 | }
78 |
79 | public void setSamples( int samples )
80 | {
81 | _samples = samples;
82 | }
83 |
84 | public double getMean()
85 | {
86 | return _mean;
87 | }
88 |
89 | public void setMean( double mean )
90 | {
91 | _mean = mean;
92 | }
93 |
94 | public double getMeanError()
95 | {
96 | return _meanError;
97 | }
98 |
99 | public void setMeanError( double meanError )
100 | {
101 | _meanError = meanError;
102 | }
103 |
104 | public String getUnit()
105 | {
106 | return _unit;
107 | }
108 |
109 | public void setUnit( String unit )
110 | {
111 | _unit = unit;
112 | }
113 |
114 | public Map getParams()
115 | {
116 | return _params;
117 | }
118 |
119 | public void addParams( String paramName, String paramValue )
120 | {
121 | _params.put( paramName, paramValue );
122 | }
123 |
124 | public double getMeanChangeFromPrev()
125 | {
126 | return _meanChangeFromPrev;
127 | }
128 |
129 | public void setMeanChangeFromPrev( double meanChangeFromPrev )
130 | {
131 | _meanChangeFromPrev = meanChangeFromPrev;
132 | }
133 |
134 | public double getMeanChangeFromBaseline()
135 | {
136 | return _meanChangeFromBaseline;
137 | }
138 |
139 | public void setMeanChangeFromBaseline( double meanChangeFromBaseline )
140 | {
141 | _meanChangeFromBaseline = meanChangeFromBaseline;
142 | }
143 |
144 | public String getChangeIndicator()
145 | {
146 | return _changeIndicator;
147 | }
148 |
149 | public void setChangeIndicator( String changeIndicator )
150 | {
151 | _changeIndicator = changeIndicator;
152 | }
153 |
154 | public String getKey()
155 | {
156 | StringBuilder sb = new StringBuilder( 100 );
157 | sb.append( _shortBenchmarkName );
158 | for ( Map.Entry entry : _params.entrySet() )
159 | {
160 | if ( !entry.getValue().equals( "" ) )
161 | {
162 | sb.append( ":" );
163 | sb.append( entry.getValue() );
164 | }
165 | }
166 | return sb.toString();
167 | }
168 | }
169 |
--------------------------------------------------------------------------------
/src/main/java/blackboard/test/jenkins/jmhbenchmark/CsvParser.java:
--------------------------------------------------------------------------------
1 | package blackboard.test.jenkins.jmhbenchmark;
2 |
3 | import hudson.model.TaskListener;
4 | import hudson.model.AbstractBuild;
5 |
6 | import java.io.BufferedReader;
7 | import java.io.File;
8 | import java.io.FileReader;
9 | import java.io.IOException;
10 | import java.io.PrintStream;
11 | import java.util.ArrayList;
12 | import java.util.List;
13 |
14 | public class CsvParser extends ReportParser
15 | {
16 | private static final String IMPROVEMENT_IN_MEAN_HEADER_NAME = "Score Improvement % (previous/baseline)";
17 | private static final int BENCHMARK_NAME = 0;
18 | private static final int BENCHMARK_MODE = 1;
19 | private static final int THREADS = 2;
20 | private static final int SAMPLES = 3;
21 | private static final int MEAN = 4;
22 | private static final int MEAN_ERROR_99_9 = 5;
23 | private static final int UNIT = 6;
24 |
25 | public CsvParser()
26 | {
27 | }
28 |
29 | public BenchmarkReport parse( AbstractBuild build, File reportFile, TaskListener listener ) throws IOException
30 | {
31 | PrintStream logger = null;
32 | if ( listener != null )
33 | {
34 | logger = listener.getLogger();
35 | }
36 |
37 | final BenchmarkReport report = new BenchmarkReport();
38 |
39 | BufferedReader reader = null;
40 | try
41 | {
42 | List headerColumns = null;
43 | logger.println("Parsing the file : " + reportFile.getName());
44 | reader = new BufferedReader( new FileReader( reportFile ) );
45 | String line = reader.readLine();
46 | if ( line != null )
47 | {
48 | // This is the first line. The CSV file has header at the first line.
49 | headerColumns = getHeadersList( line );
50 | report.setHeader( headerColumns );
51 | report.addHeaderColumn( IMPROVEMENT_IN_MEAN_HEADER_NAME );
52 | line = reader.readLine();
53 | }
54 | while ( line != null )
55 | {
56 | BenchmarkResult sample = getIndividualBenchmark( line, headerColumns );
57 | report.addBenchmarkResult( sample.getKey(), sample );
58 | line = reader.readLine();
59 | }
60 | }
61 | finally
62 | {
63 | if ( reader != null )
64 | {
65 | reader.close();
66 | }
67 | }
68 | return report;
69 |
70 | }
71 |
72 | /**
73 | * If a benchmark hasn't used parameters, the output is in the order:
74 | * "Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit"
75 | * If a benchmark used parameters, the output is in the order:
76 | * "Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit", "Param: paramName1","Param: paramName2"
77 | * @param line - a line of data from the csv file to which the benchmark result is written
78 | * @param header - the header of the csv file
79 | * @return
80 | */
81 | private BenchmarkResult getIndividualBenchmark( String line, List header )
82 | {
83 | BenchmarkResult sample = new BenchmarkResult();
84 |
85 | String[] values = line.split( "," );
86 | String benchmarkName = stripQuotes( values[ BENCHMARK_NAME ] );
87 | sample.setBenchmarkName( benchmarkName );
88 | sample.setShortBenchmarkName( getShortName(benchmarkName) );
89 | sample.setMode( stripQuotes( values[ BENCHMARK_MODE ] ) );
90 | sample.setThreads( Integer.valueOf( values[ THREADS ] ) );
91 | sample.setSamples( Integer.valueOf( values[ SAMPLES ] ) );
92 | sample.setMean( Double.valueOf( values[ MEAN ] ) );
93 | if ( !values[ MEAN_ERROR_99_9 ].equals( "NaN" ) )
94 | {
95 | sample.setMeanError( Double.valueOf( values[ MEAN_ERROR_99_9 ] ) );
96 | }
97 | sample.setUnit( stripQuotes( values[ UNIT ] ) );
98 |
99 | for ( int i = 7; i < values.length; i++ )
100 | {
101 | sample.addParams( header.get( i ), stripQuotes( values[ i ] ) );
102 | }
103 |
104 | return sample;
105 | }
106 |
107 | private String stripQuotes( String quotedStr )
108 | {
109 | return quotedStr.replace( "\"", "" );
110 | }
111 |
112 | private String getShortName( String name )
113 | {
114 | String[] splitNames = name.split( "\\." );
115 | StringBuilder sb = new StringBuilder( 100 );
116 | for ( int i = 0; i < splitNames.length - 2; i++ )
117 | {
118 | sb.append( splitNames[i].charAt( 0 ) );
119 | sb.append( "." );
120 | }
121 | sb.append( splitNames[splitNames.length - 2] );
122 | sb.append( "." );
123 | sb.append( splitNames[splitNames.length - 1] );
124 |
125 | return sb.toString();
126 | }
127 |
128 | private List getHeadersList( String line )
129 | {
130 | List headers = new ArrayList();
131 | String[] columns = line.split( "," );
132 | for ( int i = 0; i < columns.length; i++ )
133 | {
134 | headers.add( stripQuotes( columns[ i ] ) );
135 | }
136 | return headers;
137 | }
138 | }
139 |
--------------------------------------------------------------------------------
/src/main/java/blackboard/test/jenkins/jmhbenchmark/BenchmarkProjectAction.java:
--------------------------------------------------------------------------------
1 | package blackboard.test.jenkins.jmhbenchmark;
2 |
3 | import java.awt.BasicStroke;
4 | import java.awt.Color;
5 | import java.io.IOException;
6 | import java.util.List;
7 | import java.util.Map;
8 | import java.util.TreeMap;
9 |
10 | import org.jfree.chart.ChartFactory;
11 | import org.jfree.chart.JFreeChart;
12 | import org.jfree.chart.axis.CategoryAxis;
13 | import org.jfree.chart.axis.CategoryLabelPositions;
14 | import org.jfree.chart.plot.CategoryPlot;
15 | import org.jfree.chart.plot.PlotOrientation;
16 | import org.jfree.chart.renderer.category.LineAndShapeRenderer;
17 | import org.jfree.data.category.CategoryDataset;
18 | import org.kohsuke.stapler.StaplerRequest;
19 | import org.kohsuke.stapler.StaplerResponse;
20 |
21 | import hudson.model.AbstractBuild;
22 | import hudson.model.AbstractProject;
23 | import hudson.model.Action;
24 | import hudson.util.ChartUtil;
25 | import hudson.util.ColorPalette;
26 | import hudson.util.Graph;
27 | import hudson.util.ChartUtil.NumberOnlyBuildLabel;
28 | import hudson.util.DataSetBuilder;
29 | import hudson.util.ShiftedCategoryAxis;
30 |
31 | /**
32 | * The {@link Action} that is executed at the project level. It constructs a benchmark trend ({@link BenchmarkTrend})
33 | * on score and score error for each benchmark over the previous builds and creates a graph that shows this trend.
34 | *
35 | */
36 | public class BenchmarkProjectAction implements Action
37 | {
38 | private static final String PLUGIN_NAME = "jmhbenchmark";
39 | private static final String DISPLAY_NAME = "JMH Report Trend";
40 |
41 | public final AbstractProject _project;
42 |
43 | private Map _benchmarkTrend = new TreeMap();
44 |
45 | public BenchmarkProjectAction( @SuppressWarnings( "rawtypes" ) AbstractProject project )
46 | {
47 | _project = project;
48 | }
49 |
50 | public String getIconFileName()
51 | {
52 | return "graph.gif";
53 | }
54 |
55 | public String getDisplayName()
56 | {
57 | return DISPLAY_NAME;
58 | }
59 |
60 | public String getUrlName()
61 | {
62 | return PLUGIN_NAME;
63 | }
64 |
65 | public AbstractProject getProject()
66 | {
67 | return _project;
68 | }
69 |
70 | public Map getBenchmarkTrend()
71 | {
72 | Map buildBenchmarkData = null;
73 | BenchmarkTrend trendBenchmarkData = null;
74 | List> builds = getProject().getBuilds();
75 |
76 | _benchmarkTrend.clear();
77 |
78 | for ( AbstractBuild currentBuild : builds )
79 | {
80 | NumberOnlyBuildLabel buildNoLabel = new NumberOnlyBuildLabel( currentBuild );
81 |
82 | BenchmarkBuildAction jmhBenchmarkBuildAction = currentBuild.getAction( BenchmarkBuildAction.class );
83 | if ( jmhBenchmarkBuildAction == null )
84 | {
85 | continue;
86 | }
87 |
88 | BenchmarkReport perfReport = jmhBenchmarkBuildAction.getBuildActionDisplay().getJmhPerfReport();
89 |
90 | buildBenchmarkData = perfReport.getReport();
91 |
92 | if(buildBenchmarkData != null)
93 | {
94 | for ( Map.Entry entry : buildBenchmarkData.entrySet() )
95 | {
96 | String key = entry.getKey();
97 | BenchmarkResult val = entry.getValue();
98 | if ( _benchmarkTrend.containsKey( key ) )
99 | {
100 | trendBenchmarkData = _benchmarkTrend.get( key );
101 | if ( isBenchmarkConfigSame( val, trendBenchmarkData ) )
102 | {
103 | trendBenchmarkData.addMeanTrend( buildNoLabel, val.getMean() );
104 | trendBenchmarkData.addMeanErrorTrend( buildNoLabel, val.getMeanError() );
105 | }
106 | }
107 | else
108 | {
109 | // create a benchmark trend if it hasn't yet been created
110 | trendBenchmarkData = new BenchmarkTrend( val.getShortBenchmarkName(), val.getMode(), val.getThreads(),
111 | val.getSamples(), val.getUnit() );
112 | trendBenchmarkData.addMeanTrend( buildNoLabel, val.getMean() );
113 | trendBenchmarkData.addMeanErrorTrend( buildNoLabel, val.getMeanError() );
114 | _benchmarkTrend.put( key, trendBenchmarkData );
115 | }
116 | }
117 | }
118 | }
119 |
120 | return _benchmarkTrend;
121 | }
122 |
123 | private boolean isBenchmarkConfigSame( BenchmarkResult build, BenchmarkTrend topBuild )
124 | {
125 | if ( build.getMode().equals( topBuild.getMode() ) && ( build.getThreads() == topBuild.getThreads() )
126 | && ( build.getSamples() == topBuild.getSamples() ) && build.getUnit().equals( topBuild.getUnit() ) )
127 | {
128 | return true;
129 | }
130 | return false;
131 | }
132 |
133 | public void doSummarizerGraphForMetric( final StaplerRequest request, final StaplerResponse response )
134 | throws IOException
135 | {
136 |
137 | final String benchmarkKey = request.getParameter( "benchmarkKey" );
138 | final String benchmarkUnit = request.getParameter( "benchmarkUnit" );
139 | final String benchmarkThreads = request.getParameter( "benchmarkThreads" );
140 | final String benchmarkSamples = request.getParameter( "benchmarkSamples" );
141 | final String benchmarkMode = request.getParameter( "benchmarkMode" );
142 |
143 | BenchmarkTrend val = _benchmarkTrend.get( benchmarkKey );
144 |
145 | final Map meanTrend = val.getMeanTrend();
146 | final Map meanErrorTrend = val.getMeanErrorTrend();
147 |
148 | String graphTitle = benchmarkKey + ", threads=" + benchmarkThreads + ", samples=" + benchmarkSamples + ", mode="
149 | + benchmarkMode;
150 |
151 | final Graph graph = new GraphImpl( graphTitle, benchmarkUnit )
152 | {
153 |
154 | protected DataSetBuilder createDataSet()
155 | {
156 | DataSetBuilder dataSetBuilder = new DataSetBuilder();
157 |
158 | for ( ChartUtil.NumberOnlyBuildLabel label : meanTrend.keySet() )
159 | {
160 | dataSetBuilder.add( meanTrend.get( label ), "Score", label );
161 | }
162 |
163 | for ( ChartUtil.NumberOnlyBuildLabel label : meanErrorTrend.keySet() )
164 | {
165 | dataSetBuilder.add( meanErrorTrend.get( label ), "Score Error (99.9%)", label );
166 | }
167 |
168 | return dataSetBuilder;
169 | }
170 | };
171 |
172 | graph.doPng( request, response );
173 | }
174 |
175 | private abstract class GraphImpl extends Graph
176 | {
177 | private final String _graphTitle;
178 | private final String _unit;
179 |
180 | protected GraphImpl( final String graphTitle, final String unit )
181 | {
182 | super( -1, 300, 200 );
183 | _graphTitle = graphTitle;
184 | _unit = unit;
185 | }
186 |
187 | protected abstract DataSetBuilder createDataSet();
188 |
189 | protected JFreeChart createGraph()
190 | {
191 | final CategoryDataset dataset = createDataSet().build();
192 |
193 | final JFreeChart chart = ChartFactory.createLineChart( _graphTitle, // title
194 | "Build Number #", // category axis label
195 | _unit, // value axis label
196 | dataset, // data
197 | PlotOrientation.VERTICAL, // orientation
198 | true, // include legend
199 | true, // tooltips
200 | false // urls
201 | );
202 |
203 | chart.setBackgroundPaint( Color.white );
204 |
205 | final CategoryPlot plot = chart.getCategoryPlot();
206 |
207 | CategoryAxis domainAxis = new ShiftedCategoryAxis( null );
208 | plot.setDomainAxis( domainAxis );
209 | domainAxis.setCategoryLabelPositions( CategoryLabelPositions.UP_90 );
210 | domainAxis.setLowerMargin( 0.0 );
211 | domainAxis.setUpperMargin( 0.0 );
212 | domainAxis.setCategoryMargin( 0.0 );
213 |
214 | final LineAndShapeRenderer renderer = (LineAndShapeRenderer) plot.getRenderer();
215 | renderer.setBaseStroke( new BasicStroke( 3.0f ) );
216 | ColorPalette.apply( renderer );
217 |
218 | return chart;
219 | }
220 | }
221 | }
222 |
--------------------------------------------------------------------------------
/src/main/java/blackboard/test/jenkins/jmhbenchmark/BenchmarkPublisher.java:
--------------------------------------------------------------------------------
1 | package blackboard.test.jenkins.jmhbenchmark;
2 |
3 | import hudson.*;
4 | import hudson.model.*;
5 | import hudson.model.AbstractBuild;
6 | import hudson.model.AbstractProject;
7 | import hudson.model.ParametersAction;
8 | import hudson.model.StringParameterValue;
9 | import hudson.tasks.*;
10 | import hudson.util.FormValidation;
11 |
12 | import java.io.*;
13 | import java.util.*;
14 |
15 | import org.kohsuke.stapler.DataBoundConstructor;
16 | import org.kohsuke.stapler.QueryParameter;
17 |
18 | /**
19 | * BenchmarkPublisher is the main class for this Jenkin's plugin that will be run as a post-build action in a CI build
20 | * process. As a build step, the JMH benchmark tests are run using build tools such as Gradle. The post-build action
21 | * triggers the execution of {@link BenchmarkPublisher#perform(AbstractBuild, Launcher, BuildListener)}, which currently
22 | * consumes JMH benchmark results in a CSV format. The post-build action first copies the raw benchmark result to the
23 | * master. It then parses the benchmark result and post the benchmark report ({@link BenchmarkReport}) to each build.
24 | * Based on a configurable performance degradation threshold, a build is marked as unstable if the benchmark's score is
25 | * degraded below this threshold.
26 | *
27 | * Configuration of the post-build action is performed from the job's configuration page. The configurable parameters
28 | * are: performance degradation threshold, performance gain threshold, baseline build number, number of decimal places
29 | * to use for the benchmark result.
30 | */
31 | public class BenchmarkPublisher extends Recorder
32 | {
33 | private final int _performanceIncreaseThreshold;
34 | private final int _performanceDegradationThreshold;
35 | private final int _decimalPlaces;
36 | private final int _baselineBuildNumber;
37 | private ReportParser _parser;
38 | private static final String BENCHMARK_OUTPUT_FOLDER = "jmh_benchmark_result";
39 | private static final String BENCHMARK_MODE_THRPT = "thrpt";
40 | private static String BUILD_PROJECT_NAME;
41 | // two decimal places are used to set changes from previous or baseline build
42 | private static final int MULTIPLIER = 100;
43 |
44 | @DataBoundConstructor
45 | public BenchmarkPublisher( int performanceIncreaseThreshold, int performanceDegradationThreshold, int decimalPlaces,
46 | int baselineBuildNumber )
47 | {
48 | _performanceIncreaseThreshold = performanceIncreaseThreshold;
49 | _performanceDegradationThreshold = performanceDegradationThreshold;
50 | _decimalPlaces = decimalPlaces;
51 | _baselineBuildNumber = baselineBuildNumber;
52 | }
53 |
54 | public int getDecimalPlaces()
55 | {
56 | return _decimalPlaces;
57 | }
58 |
59 | public int getPerformanceIncreaseThreshold()
60 | {
61 | return _performanceIncreaseThreshold;
62 | }
63 |
64 | public int getPerformanceDegradationThreshold()
65 | {
66 | return _performanceDegradationThreshold;
67 | }
68 |
69 | public int getBaselineBuildNumber()
70 | {
71 | return _baselineBuildNumber;
72 | }
73 |
74 | @Override
75 | public boolean perform( AbstractBuild build, Launcher launcher, BuildListener listener )
76 | throws IOException, InterruptedException
77 | {
78 | BUILD_PROJECT_NAME = build.getProject().getName();
79 | boolean buildStable = true;
80 | Set failedBenchmarks = new HashSet();
81 |
82 | PrintStream logger = listener.getLogger();
83 | if ( build.getResult().isWorseThan( Result.UNSTABLE ) )
84 | {
85 | build.setResult( Result.FAILURE );
86 | return true;
87 | }
88 |
89 | FilePath[] files = build.getWorkspace().list( "*.csv" );
90 | if ( files.length <= 0 )
91 | {
92 | build.setResult( Result.FAILURE );
93 | logger.println( "JMH Benchmark: benchmark file could not be found." );
94 | return true;
95 | }
96 |
97 | File localReports = copyBenchmarkOutputToMaster( build, files[ 0 ], BUILD_PROJECT_NAME );
98 |
99 | // currently only a CSV parser is supported
100 | _parser = new CsvParser();
101 | BenchmarkReport parsedReport = _parser.parse( build, localReports, listener );
102 |
103 | // get previous successful build (i.e. not failed build) report to calculate the increase in mean value for each benchmark and set an indicator (i.e.
104 | // green or red for each benchmark) depending on threshold set in the configuration. If there is at least a red for
105 | // one benchmark, the build status will be unstable
106 |
107 | AbstractBuild prevSuccessfulBuild = build.getPreviousNotFailedBuild();
108 |
109 | // if there is no previous successful build, there is no baseline build
110 | if ( prevSuccessfulBuild != null )
111 | {
112 | Map currentApiTestReport = parsedReport.getReport();
113 | BenchmarkBuildAction prevBuildAction = prevSuccessfulBuild.getAction( BenchmarkBuildAction.class );
114 | BenchmarkReport prevPerfReport = null;
115 | Map prevApiTestReport = null;
116 |
117 | if ( prevBuildAction != null )
118 | {
119 | prevPerfReport = prevBuildAction.getBuildActionDisplay().getJmhPerfReport();
120 | prevApiTestReport = prevPerfReport.getReport();
121 | }
122 |
123 | AbstractBuild baselineBuild = getBaselineBuild( build );
124 | Map baselineApiTestReport = null;
125 | if ( baselineBuild != null )
126 | {
127 | BenchmarkBuildAction baselineBuildAction = baselineBuild.getAction( BenchmarkBuildAction.class );
128 | BenchmarkReport baselinePerfReport = null;
129 |
130 | if ( baselineBuildAction != null )
131 | {
132 | baselinePerfReport = baselineBuildAction.getBuildActionDisplay().getJmhPerfReport();
133 | baselineApiTestReport = baselinePerfReport.getReport();
134 | }
135 | }
136 |
137 | double decreaseInMeanFromPrev = 0;
138 | double decreaseInMeanFromBaseline = 0;
139 |
140 | if ( prevApiTestReport != null )
141 | {
142 | for ( Map.Entry entry : currentApiTestReport.entrySet() )
143 | {
144 | String key = entry.getKey();
145 | BenchmarkResult currVal = entry.getValue();
146 | BenchmarkResult prevVal = prevApiTestReport.get( key );
147 |
148 | if ( prevVal != null )
149 | {
150 | // decrease in mean from previous is calculated as ((prev - curr)/prev) * 100%
151 | decreaseInMeanFromPrev = ( 1 - currVal.getMean() / prevVal.getMean() ) * 100.0;
152 | decreaseInMeanFromPrev = (double) Math.round( decreaseInMeanFromPrev * MULTIPLIER ) / MULTIPLIER;
153 | if ( currVal.getMode().equalsIgnoreCase( BENCHMARK_MODE_THRPT ) )
154 | {
155 | decreaseInMeanFromPrev = -1 * decreaseInMeanFromPrev;
156 | }
157 | currVal.setMeanChangeFromPrev( decreaseInMeanFromPrev );
158 | }
159 |
160 | if ( baselineApiTestReport != null )
161 | {
162 | BenchmarkResult baselineVal = baselineApiTestReport.get( key );
163 |
164 | if ( baselineVal != null )
165 | {
166 | // decrease in mean from baseline is calculated as ((baseline - curr)/baseline) * 100%
167 | decreaseInMeanFromBaseline = ( 1 - currVal.getMean() / baselineVal.getMean() ) * 100.0;
168 | decreaseInMeanFromBaseline = (double) Math.round( decreaseInMeanFromBaseline * MULTIPLIER ) / MULTIPLIER;
169 | if ( currVal.getMode().equalsIgnoreCase( BENCHMARK_MODE_THRPT ) )
170 | {
171 | decreaseInMeanFromBaseline = -1 * decreaseInMeanFromBaseline;
172 | }
173 | currVal.setMeanChangeFromBaseline( decreaseInMeanFromBaseline );
174 | }
175 | }
176 |
177 | if ( decreaseInMeanFromBaseline >= _performanceIncreaseThreshold
178 | || decreaseInMeanFromPrev >= _performanceIncreaseThreshold )
179 | {
180 | currVal.setChangeIndicator( "green" );
181 | }
182 | else if ( decreaseInMeanFromBaseline <= _performanceDegradationThreshold
183 | || decreaseInMeanFromPrev <= _performanceDegradationThreshold )
184 | {
185 | currVal.setChangeIndicator( "red" );
186 | failedBenchmarks.add( currVal.getBenchmarkName() );
187 | buildStable = false;
188 | }
189 | }
190 | }
191 | }
192 |
193 | BenchmarkBuildAction buildAction = new BenchmarkBuildAction( build, parsedReport, _decimalPlaces );
194 | build.addAction( buildAction );
195 |
196 | if ( !buildStable )
197 | {
198 | StringBuilder sb = new StringBuilder();
199 | for ( String bm : failedBenchmarks )
200 | {
201 | sb.append( "|" ).append( bm );
202 | }
203 | build.addAction( new ParametersAction( new StringParameterValue( "JMH_FAILED_BENCHMARKS", sb.toString() ) ) );
204 | build.setResult( Result.UNSTABLE );
205 | }
206 |
207 | return true;
208 | }
209 |
210 | private AbstractBuild getBaselineBuild( AbstractBuild build )
211 | {
212 | if ( _baselineBuildNumber == 0 )
213 | return null;
214 |
215 | AbstractBuild nthBuild = build;
216 |
217 | int nextBuildNumber = build.number - _baselineBuildNumber;
218 |
219 | for ( int i = 1; i <= nextBuildNumber; i++ )
220 | {
221 | nthBuild = nthBuild.getPreviousBuild();
222 | if ( nthBuild == null )
223 | return null;
224 | // this is required since old builds can be cleaned or the baseline builds are old builds that have been kept forever.
225 | if ( nthBuild.number == _baselineBuildNumber )
226 | return nthBuild;
227 | }
228 | return nthBuild;
229 | }
230 |
231 | private File copyBenchmarkOutputToMaster( AbstractBuild build, FilePath output, String projectFolderName )
232 | throws IOException, InterruptedException
233 | {
234 | File localReport = getPerformanceReport( build, projectFolderName, output.getName() );
235 | output.copyTo( new FilePath( localReport ) );
236 | return localReport;
237 | }
238 |
239 | public static File getPerformanceReport( AbstractBuild build, String projectFolderName,
240 | String benchmarkOutputFileName )
241 | {
242 | return new File( build.getRootDir(), getRelativePath( projectFolderName, benchmarkOutputFileName ) );
243 | }
244 |
245 | private static String getRelativePath( String... suffixes )
246 | {
247 | StringBuilder sb = new StringBuilder( 150 );
248 | sb.append( BENCHMARK_OUTPUT_FOLDER );
249 | for ( String suffix : suffixes )
250 | {
251 | sb.append( File.separator ).append( suffix );
252 | }
253 | return sb.toString();
254 | }
255 |
256 | @Override
257 | public Action getProjectAction( AbstractProject project )
258 | {
259 | return new BenchmarkProjectAction( project );
260 | }
261 |
262 | @Override
263 | public DescriptorImpl getDescriptor()
264 | {
265 | return (DescriptorImpl) super.getDescriptor();
266 | }
267 |
268 | /**
269 | * The class is marked as public so that it can be accessed from views.
270 | *
271 | * See src/main/resources/org/jenkinsci/plugins/jmhbenchmark/BenchmarkPublisher/config.jelly for the actual
272 | * HTML fragment for the configuration screen.
273 | */
274 | @Extension
275 | public static final class DescriptorImpl extends BuildStepDescriptor
276 | {
277 | public boolean isApplicable( @SuppressWarnings( "rawtypes" ) Class jobType )
278 | {
279 | // Indicates that this builder can be used with all kinds of project types
280 | return true;
281 | }
282 |
283 | public String getDisplayName()
284 | {
285 | return "Publish JMH Test Result";
286 | }
287 |
288 | public FormValidation doCheckDecimalPlaces( @QueryParameter String decimalPlaces )
289 | {
290 | try
291 | {
292 | Integer.parseInt( decimalPlaces );
293 | }
294 | catch ( NumberFormatException ex )
295 | {
296 | return FormValidation.error( "Not a valid number" );
297 | }
298 | return FormValidation.ok();
299 | }
300 |
301 | public FormValidation doCheckPerformanceDegradationThreshold( @QueryParameter String performanceDegradationThreshold )
302 | {
303 | try
304 | {
305 | Integer.parseInt( performanceDegradationThreshold );
306 | }
307 | catch ( NumberFormatException ex )
308 | {
309 | return FormValidation.error( "Not a valid number" );
310 | }
311 | return FormValidation.ok();
312 | }
313 |
314 | public FormValidation doCheckPerformanceIncreaseThreshold( @QueryParameter String performanceIncreaseThreshold )
315 | {
316 | try
317 | {
318 | Integer.parseInt( performanceIncreaseThreshold );
319 | }
320 | catch ( NumberFormatException ex )
321 | {
322 | return FormValidation.error( "Not a valid number" );
323 | }
324 | return FormValidation.ok();
325 | }
326 |
327 | public FormValidation doCheckBaselineBuildNumber( @QueryParameter String baselineBuildNumber )
328 | {
329 | try
330 | {
331 | Integer.parseInt( baselineBuildNumber );
332 | }
333 | catch ( NumberFormatException ex )
334 | {
335 | return FormValidation.error( "Not a valid number" );
336 | }
337 | return FormValidation.ok();
338 | }
339 | }
340 |
341 | public BuildStepMonitor getRequiredMonitorService()
342 | {
343 | return BuildStepMonitor.BUILD;
344 | }
345 | }
346 |
--------------------------------------------------------------------------------