build() {
43 | assert name != null;
44 | assert metricsProvider != null;
45 | return new Experiment<>(name, context, false, metricsProvider, comparator,
46 | executorService);
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/IncompatibleTypesExperiment.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 | import com.github.rawls238.scientist4j.metrics.MetricsProvider;
5 |
6 | import java.util.Arrays;
7 | import java.util.HashMap;
8 | import java.util.Map;
9 | import java.util.Optional;
10 | import java.util.concurrent.*;
11 | import java.util.function.BiPredicate;
12 |
13 | /**
14 | * An Experiment that can handle a control and candidate function that return incompatible types.
15 | *
16 | * Note that this requires a comparator function to be passed in to the constructor because the existing default of
17 | * Objects::equals doesn't make any sense when the objects are of different types.
18 | *
19 | * @param The return type of the control function.
20 | * @param The return type of the candidate function.
21 | */
22 | public class IncompatibleTypesExperiment {
23 | private static final String CONTROL = "control";
24 | private static final String CANDIDATE = "candidate";
25 | private final ExecutorService executor;
26 | private static final String NAMESPACE_PREFIX = "scientist";
27 | private final MetricsProvider metricsProvider;
28 | private final String name;
29 | private final boolean raiseOnMismatch;
30 | private final Map context;
31 | private final MetricsProvider.Timer controlTimer;
32 | private final MetricsProvider.Timer candidateTimer;
33 | private final MetricsProvider.Counter mismatchCount;
34 | private final MetricsProvider.Counter candidateExceptionCount;
35 | private final MetricsProvider.Counter totalCount;
36 | private final BiPredicate comparator;
37 |
38 | public IncompatibleTypesExperiment(final MetricsProvider metricsProvider, final BiPredicate comparator) {
39 | this("Experiment", metricsProvider, comparator);
40 | }
41 |
42 | public IncompatibleTypesExperiment(final String name, final MetricsProvider metricsProvider,
43 | final BiPredicate comparator) {
44 | this(name, false, metricsProvider, comparator);
45 | }
46 |
47 | public IncompatibleTypesExperiment(final String name, final Map context,
48 | final MetricsProvider metricsProvider, final BiPredicate comparator) {
49 | this(name, context, false, metricsProvider, comparator);
50 | }
51 |
52 | public IncompatibleTypesExperiment(final String name, final boolean raiseOnMismatch,
53 | final MetricsProvider metricsProvider, final BiPredicate comparator) {
54 | this(name, new HashMap<>(), raiseOnMismatch, metricsProvider, comparator);
55 | }
56 |
57 | public IncompatibleTypesExperiment(final String name, final Map context,
58 | final boolean raiseOnMismatch, final MetricsProvider metricsProvider, final BiPredicate comparator) {
59 | this(name, context, raiseOnMismatch, metricsProvider, comparator, Executors.newFixedThreadPool(2));
60 | }
61 |
62 | public IncompatibleTypesExperiment(final String name, final Map context,
63 | final boolean raiseOnMismatch, final MetricsProvider metricsProvider, final BiPredicate comparator,
64 | final ExecutorService executorService) {
65 | this.name = name;
66 | this.context = context;
67 | this.raiseOnMismatch = raiseOnMismatch;
68 | this.comparator = comparator;
69 | this.metricsProvider = metricsProvider;
70 | controlTimer = getMetricsProvider().timer(NAMESPACE_PREFIX, this.name, CONTROL);
71 | candidateTimer = getMetricsProvider().timer(NAMESPACE_PREFIX, this.name, CANDIDATE);
72 | mismatchCount = getMetricsProvider().counter(NAMESPACE_PREFIX, this.name, "mismatch");
73 | candidateExceptionCount = getMetricsProvider().counter(NAMESPACE_PREFIX, this.name, "candidate.exception");
74 | totalCount = getMetricsProvider().counter(NAMESPACE_PREFIX, this.name, "total");
75 | executor = executorService;
76 | }
77 |
78 | /**
79 | * Allow override here if extending the class
80 | */
81 | public MetricsProvider getMetricsProvider() {
82 | return this.metricsProvider;
83 | }
84 |
85 | /**
86 | * Note that if {@code raiseOnMismatch} is true, {@link #runAsync(Callable, Callable)} will block waiting for
87 | * the candidate function to complete before it can raise any resulting errors. In situations where the candidate
88 | * function may be significantly slower than the control, it is not recommended to raise on mismatch.
89 | */
90 | public boolean getRaiseOnMismatch() {
91 | return raiseOnMismatch;
92 | }
93 |
94 | public String getName() {
95 | return name;
96 | }
97 |
98 | public T run(final Callable control, final Callable candidate) throws Exception {
99 | if (isAsync()) {
100 | return runAsync(control, candidate);
101 | } else {
102 | return runSync(control, candidate);
103 | }
104 | }
105 |
106 | private T runSync(final Callable control, final Callable candidate) throws Exception {
107 | Observation controlObservation;
108 | Optional> candidateObservation = Optional.empty();
109 | if (Math.random() < 0.5) {
110 | controlObservation = executeResult(CONTROL, controlTimer, control, true);
111 | if (runIf() && enabled()) {
112 | candidateObservation = Optional.of(executeResult(CANDIDATE, candidateTimer, candidate, false));
113 | }
114 | } else {
115 | if (runIf() && enabled()) {
116 | candidateObservation = Optional.of(executeResult(CANDIDATE, candidateTimer, candidate, false));
117 | }
118 | controlObservation = executeResult(CONTROL, controlTimer, control, true);
119 | }
120 |
121 | countExceptions(candidateObservation, candidateExceptionCount);
122 | IncompatibleTypesExperimentResult result =
123 | new IncompatibleTypesExperimentResult<>(this, controlObservation, candidateObservation, context);
124 | publish(result);
125 | return controlObservation.getValue();
126 | }
127 |
128 | public T runAsync(final Callable control, final Callable candidate) throws Exception {
129 | Future>> observationFutureCandidate;
130 | Future> observationFutureControl;
131 |
132 | if (runIf() && enabled()) {
133 | if (Math.random() < 0.5) {
134 | observationFutureControl =
135 | executor.submit(() -> executeResult(CONTROL, controlTimer, control, true));
136 | observationFutureCandidate = executor.submit(
137 | () -> Optional.of(executeResult(CANDIDATE, candidateTimer, candidate, false)));
138 | } else {
139 | observationFutureCandidate = executor.submit(
140 | () -> Optional.of(executeResult(CANDIDATE, candidateTimer, candidate, false)));
141 | observationFutureControl =
142 | executor.submit(() -> executeResult(CONTROL, controlTimer, control, true));
143 | }
144 | } else {
145 | observationFutureControl = executor.submit(() -> executeResult(CONTROL, controlTimer, control, true));
146 | observationFutureCandidate = null;
147 | }
148 |
149 | Observation controlObservation;
150 | try {
151 | controlObservation = observationFutureControl.get();
152 | } catch (InterruptedException e) {
153 | Thread.currentThread().interrupt();
154 | throw new RuntimeException(e);
155 | } catch (ExecutionException e) {
156 | throw new RuntimeException(e);
157 | }
158 |
159 | Future publishedResult =
160 | executor.submit(() -> publishAsync(controlObservation, observationFutureCandidate));
161 |
162 | if (raiseOnMismatch) {
163 | try {
164 | publishedResult.get();
165 | } catch (ExecutionException e) {
166 | throw (Exception) e.getCause();
167 | }
168 | }
169 |
170 | return controlObservation.getValue();
171 | }
172 |
173 | private Void publishAsync(final Observation controlObservation,
174 | final Future>> observationFutureCandidate) throws Exception {
175 | Optional> candidateObservation = Optional.empty();
176 | if (observationFutureCandidate != null) {
177 | candidateObservation = observationFutureCandidate.get();
178 | }
179 |
180 | countExceptions(candidateObservation, candidateExceptionCount);
181 | IncompatibleTypesExperimentResult result =
182 | new IncompatibleTypesExperimentResult<>(this, controlObservation, candidateObservation, context);
183 | publish(result);
184 | return null;
185 | }
186 |
187 | private void countExceptions(final Optional> observation, final MetricsProvider.Counter exceptions) {
188 | if (observation.isPresent() && observation.get().getException().isPresent()) {
189 | exceptions.increment();
190 | }
191 | }
192 |
193 | public Observation executeResult(final String name, final MetricsProvider.Timer timer,
194 | final Callable control, final boolean shouldThrow) throws Exception {
195 | Observation observation = new Observation<>(name, timer);
196 |
197 | observation.time(() ->
198 | {
199 | try {
200 | observation.setValue(control.call());
201 | } catch (Exception e) {
202 | observation.setException(e);
203 | }
204 | });
205 |
206 | Optional exception = observation.getException();
207 | if (shouldThrow && exception.isPresent()) {
208 | throw exception.get();
209 | }
210 |
211 | return observation;
212 | }
213 |
214 | protected boolean compareResults(final T controlVal, final U candidateVal) {
215 | return this.comparator.test(controlVal, candidateVal);
216 | }
217 |
218 | public boolean compare(final Observation controlVal, final Observation candidateVal)
219 | throws MismatchException {
220 | boolean resultsMatch = !candidateVal.getException().isPresent() &&
221 | compareResults(controlVal.getValue(), candidateVal.getValue());
222 | totalCount.increment();
223 | if (!resultsMatch) {
224 | mismatchCount.increment();
225 | handleComparisonMismatch(controlVal, candidateVal);
226 | }
227 | return true;
228 | }
229 |
230 | protected void publish(final IncompatibleTypesExperimentResult result) {
231 | }
232 |
233 | protected boolean runIf() {
234 | return true;
235 | }
236 |
237 | protected boolean enabled() {
238 | return true;
239 | }
240 |
241 | protected boolean isAsync() {
242 | return false;
243 | }
244 |
245 | private void handleComparisonMismatch(final Observation controlVal, final Observation candidateVal)
246 | throws MismatchException {
247 | String msg;
248 | Optional exception = candidateVal.getException();
249 | if (exception.isPresent()) {
250 | String stackTrace = Arrays.toString(exception.get().getStackTrace());
251 | String exceptionName = exception.get().getClass().getName();
252 | msg = candidateVal.getName() + " raised an exception: " + exceptionName + " " + stackTrace;
253 | } else {
254 | msg =
255 | candidateVal.getName() + " does not match control value (" + controlVal.getValue().toString() + " != " +
256 | candidateVal.getValue().toString() + ")";
257 | }
258 | throw new MismatchException(msg);
259 | }
260 | }
261 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/IncompatibleTypesExperimentResult.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 |
5 | import java.util.Map;
6 | import java.util.Optional;
7 |
8 | /**
9 | * @param The return type of the control function
10 | * @param The return type of the candidate function.
11 | */
12 | public class IncompatibleTypesExperimentResult {
13 | private final Observation control;
14 | private final Optional> candidate;
15 | private Optional match;
16 | private final Map context;
17 |
18 | public IncompatibleTypesExperimentResult(final IncompatibleTypesExperiment experiment, final Observation control,
19 | final Optional> candidate, final Map context) throws MismatchException {
20 | this.control = control;
21 | this.candidate = candidate;
22 | this.context = context;
23 | this.match = Optional.empty();
24 |
25 | if (candidate.isPresent()) {
26 | try {
27 | this.match = Optional.of(experiment.compare(control, candidate.get()));
28 | } catch (MismatchException e) {
29 | this.match = Optional.of(false);
30 | throw e;
31 | }
32 | }
33 | }
34 |
35 | public Optional getMatch() {
36 | return match;
37 | }
38 |
39 | public Observation getControl() {
40 | return control;
41 | }
42 |
43 | public Optional> getCandidate() {
44 | return candidate;
45 | }
46 |
47 | public Map getContext() {
48 | return context;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/Observation.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.metrics.MetricsProvider.Timer;
4 |
5 | import java.util.Optional;
6 |
7 | public class Observation {
8 |
9 | private String name;
10 | private Optional exception;
11 | private T value;
12 | private Timer timer;
13 |
14 | public Observation(String name, Timer timer) {
15 | this.name = name;
16 | this.timer = timer;
17 | this.exception = Optional.empty();
18 | }
19 |
20 | public String getName() {
21 | return name;
22 | }
23 |
24 | public void setValue(T o) {
25 | this.value = o;
26 | }
27 |
28 | public T getValue() {
29 | return value;
30 | }
31 |
32 | public void setException(Exception e) {
33 | this.exception = Optional.of(e);
34 | }
35 |
36 | public Optional getException() {
37 | return exception;
38 | }
39 |
40 | public long getDuration() {
41 | return timer.getDuration();
42 | }
43 |
44 | public void time(Runnable runnable) {
45 | timer.record(runnable);
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/Result.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 |
5 | import java.util.Map;
6 | import java.util.Optional;
7 |
8 | public class Result {
9 | private Experiment experiment;
10 | private Observation control;
11 | private Optional> candidate;
12 | private Optional match;
13 | private Map context;
14 |
15 | public Result(Experiment experiment, Observation control, Optional> candidate, Map context) throws MismatchException {
16 | this.experiment = experiment;
17 | this.control = control;
18 | this.candidate = candidate;
19 | this.context = context;
20 | this.match = Optional.empty();
21 |
22 | if (candidate.isPresent()) {
23 | Optional ex = Optional.empty();
24 | try {
25 | this.match = Optional.of(experiment.compare(control, candidate.get()));
26 | } catch (MismatchException e) {
27 | ex = Optional.of(e);
28 | this.match = Optional.of(false);
29 | } finally {
30 | if (experiment.getRaiseOnMismatch() && ex.isPresent()) {
31 | throw ex.get();
32 | }
33 | }
34 | }
35 | }
36 |
37 | public Optional getMatch() {
38 | return match;
39 | }
40 |
41 | public Observation getControl() {
42 | return control;
43 | }
44 |
45 | public Optional> getCandidate() {
46 | return candidate;
47 | }
48 |
49 | public Map getContext() {
50 | return context;
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/exceptions/LaboratoryException.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j.exceptions;
2 |
3 | public class LaboratoryException extends Exception {
4 | public LaboratoryException(String msg) {
5 | super(msg);
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/exceptions/MismatchException.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j.exceptions;
2 |
3 | public class MismatchException extends LaboratoryException {
4 | public MismatchException(String msg) {
5 | super(msg);
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/metrics/DropwizardMetricsProvider.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j.metrics;
2 |
3 | import io.dropwizard.metrics5.MetricRegistry;
4 | import io.dropwizard.metrics5.Timer.Context;
5 |
6 | import java.util.Arrays;
7 |
8 | public class DropwizardMetricsProvider implements MetricsProvider {
9 |
10 | private MetricRegistry registry;
11 |
12 | public DropwizardMetricsProvider() {
13 | this(new MetricRegistry());
14 | }
15 |
16 | public DropwizardMetricsProvider(MetricRegistry metricRegistry) {
17 | this.registry = metricRegistry;
18 | }
19 |
20 | @Override
21 | public Timer timer(String... nameComponents) {
22 | final io.dropwizard.metrics5.Timer timer = registry.timer(MetricRegistry.name(nameComponents[0], Arrays.copyOfRange(nameComponents, 1, nameComponents.length)));
23 |
24 | return new Timer() {
25 |
26 | long duration;
27 |
28 | @Override
29 | public void record(Runnable runnable) {
30 |
31 | final Context context = timer.time();
32 |
33 | try {
34 | runnable.run();
35 | } finally {
36 | duration = context.stop();
37 | }
38 | }
39 |
40 | @Override
41 | public long getDuration() {
42 | return duration;
43 | }
44 | };
45 | }
46 |
47 | @Override
48 | public Counter counter(String... nameComponents) {
49 |
50 | final io.dropwizard.metrics5.Counter counter = registry.counter(MetricRegistry.name(nameComponents[0], Arrays.copyOfRange(nameComponents, 1, nameComponents.length)));
51 |
52 | return new Counter() {
53 |
54 | @Override
55 | public void increment() {
56 | counter.inc();
57 | }
58 | };
59 | }
60 |
61 | @Override
62 | public MetricRegistry getRegistry() {
63 | return this.registry;
64 | }
65 |
66 | @Override
67 | public void setRegistry(MetricRegistry registry) {
68 | this.registry = registry;
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/metrics/MetricsProvider.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j.metrics;
2 |
3 | public interface MetricsProvider {
4 |
5 | Timer timer(String... nameComponents);
6 |
7 | Counter counter(String... nameComponents);
8 |
9 | interface Timer {
10 |
11 | void record(Runnable runnable);
12 |
13 | /**
14 | * The duration recorded by this timer
15 | *
16 | * @return timer duration in nanoseconds
17 | */
18 | long getDuration();
19 | }
20 |
21 | interface Counter {
22 |
23 | void increment();
24 | }
25 |
26 | T getRegistry();
27 |
28 | void setRegistry(T registry);
29 | }
30 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/main/java/com/github/rawls238/scientist4j/metrics/MicrometerMetricsProvider.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j.metrics;
2 |
3 | import io.micrometer.core.instrument.MeterRegistry;
4 | import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
5 |
6 | import java.util.concurrent.TimeUnit;
7 |
8 | public class MicrometerMetricsProvider implements MetricsProvider {
9 |
10 | private MeterRegistry registry;
11 |
12 | public MicrometerMetricsProvider() {
13 | this(new SimpleMeterRegistry());
14 | }
15 |
16 | public MicrometerMetricsProvider(MeterRegistry meterRegistry) {
17 | this.registry = meterRegistry;
18 | }
19 |
20 | @Override
21 | public Timer timer(String... nameComponents) {
22 |
23 | final io.micrometer.core.instrument.Timer timer = io.micrometer.core.instrument.Timer.builder(String.join(".", nameComponents)).register(this.registry);
24 |
25 | return new Timer() {
26 | @Override
27 | public void record(Runnable runnable) {
28 | timer.record(runnable);
29 | }
30 |
31 | @Override
32 | public long getDuration() {
33 | return (long)timer.totalTime(TimeUnit.NANOSECONDS);
34 | }
35 | };
36 | }
37 |
38 | @Override
39 | public Counter counter(String... nameComponents) {
40 |
41 | final io.micrometer.core.instrument.Counter counter = io.micrometer.core.instrument.Counter.builder(String.join(".", nameComponents)).register(this.registry);
42 |
43 | return new Counter() {
44 | @Override
45 | public void increment() {
46 | counter.increment();
47 | }
48 | };
49 | }
50 |
51 | @Override
52 | public MeterRegistry getRegistry() {
53 | return registry;
54 | }
55 |
56 | @Override
57 | public void setRegistry(MeterRegistry registry) {
58 | this.registry = registry;
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/ExperimentAsyncCandidateOnlyTest.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 | import com.github.rawls238.scientist4j.metrics.NoopMetricsProvider;
5 | import org.junit.Test;
6 |
7 | import java.util.Date;
8 | import java.util.concurrent.Callable;
9 | import java.util.concurrent.Executors;
10 | import java.util.concurrent.ThreadFactory;
11 |
12 | import static org.assertj.core.api.Assertions.assertThat;
13 |
14 | public class ExperimentAsyncCandidateOnlyTest {
15 |
16 | private Integer exceptionThrowingFunction() {
17 | throw new RuntimeException("throw an exception");
18 | }
19 |
20 | private Integer sleepFunction() {
21 | try {
22 | Thread.sleep(1001);
23 | } catch (InterruptedException e) {
24 | e.printStackTrace();
25 | Thread.currentThread().interrupt();
26 | }
27 | return 3;
28 | }
29 |
30 | private Integer shortSleepFunction() {
31 | try {
32 | Thread.sleep(101);
33 | } catch (InterruptedException e) {
34 | e.printStackTrace();
35 | Thread.currentThread().interrupt();
36 | }
37 | return 3;
38 | }
39 |
40 | private Integer safeFunction() {
41 | return 3;
42 | }
43 |
44 | private Integer safeFunctionWithDifferentResult() {
45 | return 4;
46 | }
47 |
48 | @Test
49 | public void itThrowsAnExceptionWhenControlFails() {
50 | Experiment experiment = new Experiment<>("test", new NoopMetricsProvider());
51 | boolean controlThrew = false;
52 | try {
53 | experiment.runAsyncCandidateOnly(this::exceptionThrowingFunction, this::exceptionThrowingFunction);
54 | } catch (RuntimeException e) {
55 | controlThrew = true;
56 | } catch (Exception e) {
57 |
58 | }
59 | assertThat(controlThrew).isEqualTo(true);
60 | }
61 |
62 | @Test
63 | public void itDoesntThrowAnExceptionWhenCandidateFails() {
64 | Experiment experiment = new Experiment<>("test", new NoopMetricsProvider());
65 | boolean candidateThrew = false;
66 | Integer val = 0;
67 | try {
68 | val = experiment.runAsyncCandidateOnly(this::safeFunction, this::exceptionThrowingFunction);
69 | } catch (Exception e) {
70 | candidateThrew = true;
71 | }
72 | assertThat(candidateThrew).isEqualTo(false);
73 | assertThat(val).isEqualTo(3);
74 | }
75 |
76 | @Test
77 | public void itThrowsOnMismatch() {
78 | Experiment experiment = new Experiment<>("test", true, new NoopMetricsProvider());
79 | boolean candidateThrew = false;
80 | try {
81 | experiment.runAsyncCandidateOnly(this::safeFunction, this::safeFunctionWithDifferentResult);
82 | } catch (MismatchException e) {
83 | candidateThrew = true;
84 | } catch (Exception e) {
85 |
86 | }
87 |
88 | assertThat(candidateThrew).isEqualTo(true);
89 | }
90 |
91 | @Test
92 | public void itDoesNotThrowOnMatch() {
93 | Experiment exp = new Experiment<>("test", true, new NoopMetricsProvider());
94 | boolean candidateThrew = false;
95 | Integer val = 0;
96 | try {
97 | val = exp.runAsync(this::safeFunction, this::safeFunction);
98 | } catch (Exception e) {
99 | candidateThrew = true;
100 | }
101 |
102 | assertThat(val).isEqualTo(3);
103 | assertThat(candidateThrew).isEqualTo(false);
104 | }
105 |
106 | @Test
107 | public void itWorksWithAnExtendedClass() {
108 | Experiment exp = new TestPublishExperiment<>("test", new NoopMetricsProvider());
109 | try {
110 | exp.run(this::safeFunction, this::safeFunction);
111 | } catch (Exception e) {
112 |
113 | }
114 | }
115 |
116 | @Test
117 | public void asyncRunsFaster() {
118 | Experiment exp = new Experiment<>("test", false, new NoopMetricsProvider());
119 | boolean candidateThrew = false;
120 | Integer val = 0;
121 | Date date1 = new Date();
122 |
123 | try {
124 | val = exp.runAsyncCandidateOnly(this::sleepFunction, this::sleepFunction);
125 | } catch (Exception e) {
126 | candidateThrew = true;
127 | }
128 | Date date2 = new Date();
129 | long difference = date2.getTime() - date1.getTime();
130 |
131 | assertThat(difference).isLessThan(2000);
132 | assertThat(difference).isGreaterThanOrEqualTo(1000);
133 | assertThat(val).isEqualTo(3);
134 | assertThat(candidateThrew).isEqualTo(false);
135 | }
136 |
137 | @Test
138 | public void controlRunsOnMainThreadCustomExecutorService() throws Exception {
139 | String threadName = "main";
140 | ThreadFactory threadFactory = runnable -> new Thread(runnable, threadName);
141 | Experiment exp = new ExperimentBuilder()
142 | .withName("test")
143 | .withMetricsProvider(new NoopMetricsProvider())
144 | .withExecutorService(Executors.newFixedThreadPool(4, threadFactory))
145 | .build();
146 | Callable getThreadName = () -> Thread.currentThread().getName();
147 |
148 | String val = exp.runAsyncCandidateOnly(getThreadName, getThreadName);
149 |
150 | assertThat(val).isEqualTo(threadName);
151 | }
152 |
153 | @Test
154 | public void raiseOnMismatchRunsSlower() throws Exception {
155 | Experiment raisesOnMismatch = new Experiment<>("raise", true, new NoopMetricsProvider());
156 | Experiment doesNotRaiseOnMismatch = new Experiment<>("does not raise", new NoopMetricsProvider());
157 | final long raisesExecutionTime = timeExperiment(raisesOnMismatch);
158 | final long doesNotRaiseExecutionTime = timeExperiment(doesNotRaiseOnMismatch);
159 |
160 | assertThat(raisesExecutionTime).isGreaterThan(doesNotRaiseExecutionTime);
161 | assertThat(raisesExecutionTime).isGreaterThan(1000);
162 | assertThat(doesNotRaiseExecutionTime).isLessThan(200);
163 | }
164 |
165 | private long timeExperiment(final Experiment exp) throws Exception {
166 | Date date1 = new Date();
167 | exp.runAsyncCandidateOnly(this::shortSleepFunction, this::sleepFunction);
168 | Date date2 = new Date();
169 | return date2.getTime() - date1.getTime();
170 | }
171 |
172 | }
173 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/ExperimentAsyncTest.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 | import com.github.rawls238.scientist4j.metrics.NoopMetricsProvider;
5 | import org.junit.Test;
6 |
7 | import java.util.Date;
8 | import java.util.concurrent.Callable;
9 | import java.util.concurrent.Executors;
10 | import java.util.concurrent.ThreadFactory;
11 |
12 | import static org.assertj.core.api.Assertions.assertThat;
13 |
14 | public class ExperimentAsyncTest {
15 |
16 | private Integer exceptionThrowingFunction() {
17 | throw new RuntimeException("throw an exception");
18 | }
19 |
20 | private Integer sleepFunction() {
21 | try {
22 | Thread.sleep(1001);
23 | } catch (InterruptedException e) {
24 | e.printStackTrace();
25 | }
26 | return 3;
27 | }
28 |
29 | private Integer shortSleepFunction() {
30 | try {
31 | Thread.sleep(101);
32 | } catch (InterruptedException e) {
33 | e.printStackTrace();
34 | }
35 | return 3;
36 | }
37 |
38 | private Integer safeFunction() {
39 | return 3;
40 | }
41 |
42 | private Integer safeFunctionWithDifferentResult() {
43 | return 4;
44 | }
45 |
46 | @Test
47 | public void itThrowsAnExceptionWhenControlFails() {
48 | Experiment experiment = new Experiment<>("test", new NoopMetricsProvider());
49 | boolean controlThrew = false;
50 | try {
51 | experiment.runAsync(this::exceptionThrowingFunction, this::exceptionThrowingFunction);
52 | } catch (RuntimeException e) {
53 | controlThrew = true;
54 | } catch (Exception e) {
55 |
56 | }
57 | assertThat(controlThrew).isEqualTo(true);
58 | }
59 |
60 | @Test
61 | public void itDoesntThrowAnExceptionWhenCandidateFails() {
62 | Experiment experiment = new Experiment<>("test", new NoopMetricsProvider());
63 | boolean candidateThrew = false;
64 | Integer val = 0;
65 | try {
66 | val = experiment.runAsync(this::safeFunction, this::exceptionThrowingFunction);
67 | } catch (Exception e) {
68 | candidateThrew = true;
69 | }
70 | assertThat(candidateThrew).isEqualTo(false);
71 | assertThat(val).isEqualTo(3);
72 | }
73 |
74 | @Test
75 | public void itThrowsOnMismatch() {
76 | Experiment experiment = new Experiment<>("test", true, new NoopMetricsProvider());
77 | boolean candidateThrew = false;
78 | try {
79 | experiment.runAsync(this::safeFunction, this::safeFunctionWithDifferentResult);
80 | } catch (MismatchException e) {
81 | candidateThrew = true;
82 | } catch (Exception e) {
83 |
84 | }
85 |
86 | assertThat(candidateThrew).isEqualTo(true);
87 | }
88 |
89 | @Test
90 | public void itDoesNotThrowOnMatch() {
91 | Experiment exp = new Experiment<>("test", true, new NoopMetricsProvider());
92 | boolean candidateThrew = false;
93 | Integer val = 0;
94 | try {
95 | val = exp.runAsync(this::safeFunction, this::safeFunction);
96 | } catch (Exception e) {
97 | candidateThrew = true;
98 | }
99 |
100 | assertThat(val).isEqualTo(3);
101 | assertThat(candidateThrew).isEqualTo(false);
102 | }
103 |
104 | @Test
105 | public void itWorksWithAnExtendedClass() {
106 | Experiment exp = new TestPublishExperiment<>("test", new NoopMetricsProvider());
107 | try {
108 | exp.run(this::safeFunction, this::safeFunction);
109 | } catch (Exception e) {
110 |
111 | }
112 | }
113 |
114 | @Test
115 | public void asyncRunsFaster() {
116 | Experiment exp = new Experiment<>("test", true, new NoopMetricsProvider());
117 | boolean candidateThrew = false;
118 | Integer val = 0;
119 | Date date1 = new Date();
120 |
121 | try {
122 | val = exp.runAsync(this::sleepFunction, this::sleepFunction);
123 | } catch (Exception e) {
124 | candidateThrew = true;
125 | }
126 | Date date2 = new Date();
127 | long difference = date2.getTime() - date1.getTime();
128 |
129 | assertThat(difference).isLessThan(2000);
130 | assertThat(difference).isGreaterThanOrEqualTo(1000);
131 | assertThat(val).isEqualTo(3);
132 | assertThat(candidateThrew).isEqualTo(false);
133 | }
134 |
135 | @Test
136 | public void allowsUsingCustomExecutorService() throws Exception {
137 | String threadName = "customThread";
138 | ThreadFactory threadFactory = runnable -> new Thread(runnable, threadName);
139 | Experiment exp = new ExperimentBuilder()
140 | .withName("test")
141 | .withMetricsProvider(new NoopMetricsProvider())
142 | .withExecutorService(Executors.newFixedThreadPool(4, threadFactory))
143 | .build();
144 | Callable getThreadName = () -> Thread.currentThread().getName();
145 |
146 | String val = exp.runAsync(getThreadName, getThreadName);
147 |
148 | assertThat(val).isEqualTo(threadName);
149 | }
150 |
151 | @Test
152 | public void raiseOnMismatchRunsSlower() throws Exception {
153 | Experiment raisesOnMismatch = new Experiment<>("raise", true, new NoopMetricsProvider());
154 | Experiment doesNotRaiseOnMismatch = new Experiment<>("does not raise", new NoopMetricsProvider());
155 | final long raisesExecutionTime = timeExperiment(raisesOnMismatch);
156 | final long doesNotRaiseExecutionTime = timeExperiment(doesNotRaiseOnMismatch);
157 |
158 | assertThat(raisesExecutionTime).isGreaterThan(doesNotRaiseExecutionTime);
159 | assertThat(raisesExecutionTime).isGreaterThan(1000);
160 | assertThat(doesNotRaiseExecutionTime).isLessThan(200);
161 | }
162 |
163 | private long timeExperiment(final Experiment exp) throws Exception {
164 | Date date1 = new Date();
165 | exp.runAsync(this::shortSleepFunction, this::sleepFunction);
166 | Date date2 = new Date();
167 | return date2.getTime() - date1.getTime();
168 | }
169 |
170 | }
171 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/ExperimentTest.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 | import com.github.rawls238.scientist4j.metrics.DropwizardMetricsProvider;
5 | import com.github.rawls238.scientist4j.metrics.MicrometerMetricsProvider;
6 | import com.github.rawls238.scientist4j.metrics.NoopMetricsProvider;
7 | import io.dropwizard.metrics5.Counter;
8 | import io.dropwizard.metrics5.MetricName;
9 | import org.junit.Test;
10 |
11 | import java.util.Date;
12 | import java.util.function.BiFunction;
13 |
14 | import static org.assertj.core.api.Assertions.assertThat;
15 | import static org.mockito.Mockito.*;
16 |
17 | public class ExperimentTest {
18 |
19 | private Integer exceptionThrowingFunction() {
20 | throw new ExpectingAnException("throw an exception");
21 | }
22 |
23 | private Integer safeFunction() {
24 | return 3;
25 | }
26 |
27 | private Integer sleepFunction() {
28 | try {
29 | Thread.sleep(1001);
30 | } catch (InterruptedException e) {
31 | e.printStackTrace();
32 | }
33 | return 3;
34 | }
35 |
36 | private Integer safeFunctionWithDifferentResult() {
37 | return 4;
38 | }
39 |
40 | @Test(expected = ExpectingAnException.class)
41 | public void itThrowsAnExceptionWhenControlFails() throws Exception {
42 | new Experiment("test", new NoopMetricsProvider())
43 | .run(this::exceptionThrowingFunction, this::exceptionThrowingFunction);
44 | }
45 |
46 | @Test
47 | public void itDoesntThrowAnExceptionWhenCandidateFails() throws Exception {
48 | Experiment experiment = new Experiment<>("test", new NoopMetricsProvider());
49 | Integer val = experiment.run(this::safeFunction, this::exceptionThrowingFunction);
50 | assertThat(val).isEqualTo(3);
51 | }
52 |
53 | @Test(expected = MismatchException.class)
54 | public void itThrowsOnMismatch() throws Exception {
55 | new Experiment("test", true, new NoopMetricsProvider())
56 | .run(this::safeFunction, this::safeFunctionWithDifferentResult);
57 | }
58 |
59 | @Test
60 | public void itDoesNotThrowOnMatch() throws Exception {
61 | Integer val = new Experiment("test", true, new NoopMetricsProvider())
62 | .run(this::safeFunction, this::safeFunction);
63 |
64 | assertThat(val).isEqualTo(3);
65 | }
66 |
67 | @Test
68 | public void itHandlesNullValues() throws Exception {
69 | Integer val = new Experiment("test", true, new NoopMetricsProvider())
70 | .run(() -> null, () -> null);
71 |
72 | assertThat(val).isNull();
73 | }
74 |
75 | @Test
76 | public void nonAsyncRunsLongTime() throws Exception {
77 | Experiment exp = new Experiment<>("test", true, new NoopMetricsProvider());
78 | Date date1 = new Date();
79 | Integer val = exp.run(this::sleepFunction, this::sleepFunction);
80 | Date date2 = new Date();
81 | long difference = date2.getTime() - date1.getTime();
82 |
83 | assertThat(difference).isGreaterThanOrEqualTo(2000);
84 | assertThat(val).isEqualTo(3);
85 | }
86 |
87 | @Test
88 | public void itWorksWithAnExtendedClass() throws Exception {
89 | Experiment exp = new TestPublishExperiment<>("test", new NoopMetricsProvider());
90 | exp.run(this::safeFunction, this::safeFunction);
91 | }
92 |
93 | @Test
94 | public void candidateExceptionsAreCounted_dropwizard() throws Exception {
95 | final DropwizardMetricsProvider provider = new DropwizardMetricsProvider();
96 | Experiment exp = new Experiment<>("test", provider);
97 |
98 | exp.run(() -> 1, this::exceptionThrowingFunction);
99 |
100 | Counter result = provider.getRegistry().getCounters().get(MetricName.build("scientist", "test", "candidate", "exception"));
101 | assertThat(result.getCount()).isEqualTo(1);
102 | }
103 |
104 | @Test
105 | public void candidateExceptionsAreCounted_micrometer() throws Exception {
106 | final MicrometerMetricsProvider provider = new MicrometerMetricsProvider();
107 | Experiment exp = new Experiment<>("test", provider);
108 |
109 | exp.run(() -> 1, this::exceptionThrowingFunction);
110 |
111 | io.micrometer.core.instrument.Counter result = provider.getRegistry().get("scientist.test.candidate.exception").counter();
112 | assertThat(result.count()).isEqualTo(1);
113 | }
114 |
115 | @Test
116 | public void shouldUseCustomComparator() throws Exception {
117 | @SuppressWarnings("unchecked") final BiFunction comparator = mock(BiFunction.class);
118 | when(comparator.apply(1, 2)).thenReturn(false);
119 | final Experiment e = new ExperimentBuilder()
120 | .withName("test")
121 | .withComparator(comparator)
122 | .withMetricsProvider(new NoopMetricsProvider())
123 | .build();
124 |
125 | e.run(() -> 1, () -> 2);
126 |
127 | verify(comparator).apply(1, 2);
128 | }
129 | }
130 |
131 | class ExpectingAnException extends RuntimeException {
132 | ExpectingAnException(final String message) {
133 | super(message);
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/IncompatibleTypesExperimentAsyncTest.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.exceptions.MismatchException;
4 | import com.github.rawls238.scientist4j.metrics.NoopMetricsProvider;
5 | import org.junit.Test;
6 |
7 | import java.util.Date;
8 |
9 | import static org.assertj.core.api.Assertions.assertThat;
10 |
11 | public class IncompatibleTypesExperimentAsyncTest {
12 |
13 | private Integer exceptionThrowingFunction() {
14 | throw new RuntimeException("throw an exception");
15 | }
16 |
17 | private String exceptionThrowingCandidateFunction() {
18 | throw new RuntimeException("throw an exception");
19 | }
20 |
21 | private Integer sleepFunction() {
22 | try {
23 | Thread.sleep(1001);
24 | } catch (InterruptedException e) {
25 | e.printStackTrace();
26 | Thread.currentThread().interrupt();
27 | }
28 | return 3;
29 | }
30 |
31 | private String sleepClandidateFunction() {
32 | try {
33 | Thread.sleep(1001);
34 | } catch (InterruptedException e) {
35 | e.printStackTrace();
36 | Thread.currentThread().interrupt();
37 | }
38 | return "3";
39 | }
40 |
41 | private Integer safeFunction() {
42 | return 3;
43 | }
44 |
45 | private String safeCandidateFunction() {
46 | return "3";
47 | }
48 |
49 | private String safeCandidateFunctionWithDifferentResult() {
50 | return "4";
51 | }
52 |
53 | @Test
54 | public void itThrowsAnExceptionWhenControlFails() throws Exception {
55 | IncompatibleTypesExperiment experiment = new IncompatibleTypesExperiment<>("test",
56 | new NoopMetricsProvider(), (integer, s) -> String.valueOf(integer).equals(s));
57 | boolean controlThrew = false;
58 | try {
59 | experiment.runAsync(this::exceptionThrowingFunction, this::exceptionThrowingCandidateFunction);
60 | } catch (RuntimeException e) {
61 | controlThrew = true;
62 | }
63 | assertThat(controlThrew).isTrue();
64 | }
65 |
66 | @Test
67 | public void itDoesntThrowAnExceptionWhenCandidateFails() {
68 | IncompatibleTypesExperiment experiment = new IncompatibleTypesExperiment<>("test",
69 | new NoopMetricsProvider(), (integer, s) -> String.valueOf(integer).equals(s));
70 | boolean candidateThrew = false;
71 | Integer val = 0;
72 | try {
73 | val = experiment.runAsync(this::safeFunction, this::exceptionThrowingCandidateFunction);
74 | } catch (Exception e) {
75 | candidateThrew = true;
76 | }
77 | assertThat(candidateThrew).isFalse();
78 | assertThat(val).isEqualTo(3);
79 | }
80 |
81 | @Test
82 | public void itThrowsOnMismatch() {
83 | IncompatibleTypesExperiment experiment = new IncompatibleTypesExperiment<>("test", true,
84 | new NoopMetricsProvider(), (integer, s) -> String.valueOf(integer).equals(s));
85 | boolean candidateThrew = false;
86 | try {
87 | experiment.runAsync(this::safeFunction, this::safeCandidateFunctionWithDifferentResult);
88 | } catch (MismatchException e) {
89 | candidateThrew = true;
90 | } catch (Exception e) {
91 |
92 | }
93 |
94 | assertThat(candidateThrew).isTrue();
95 | }
96 |
97 | @Test
98 | public void itDoesNotThrowOnMatch() {
99 | IncompatibleTypesExperiment experiment = new IncompatibleTypesExperiment<>("test",
100 | new NoopMetricsProvider(), (integer, s) -> String.valueOf(integer).equals(s));
101 | boolean candidateThrew = false;
102 | Integer val = 0;
103 | try {
104 | val = experiment.runAsync(this::safeFunction, this::safeCandidateFunction);
105 | } catch (Exception e) {
106 | candidateThrew = true;
107 | }
108 |
109 | assertThat(val).isEqualTo(3);
110 | assertThat(candidateThrew).isFalse();
111 | }
112 |
113 | @Test
114 | public void itWorksWithAnExtendedClass() {
115 | IncompatibleTypesExperiment exp = new TestPublishIncompatibleTypesExperiment("test",
116 | new NoopMetricsProvider(), (integer, s) -> String.valueOf(integer).equals(s));
117 | try {
118 | exp.run(this::safeFunction, this::safeCandidateFunction);
119 | } catch (Exception e) {
120 |
121 | }
122 | }
123 |
124 | @Test
125 | public void asyncRunsFaster() {
126 | IncompatibleTypesExperiment exp = new IncompatibleTypesExperiment<>("test",
127 | new NoopMetricsProvider(), (integer, s) -> String.valueOf(integer).equals(s));
128 | boolean candidateThrew = false;
129 | Integer val = 0;
130 | Date date1 = new Date();
131 |
132 | try {
133 | val = exp.runAsync(this::sleepFunction, this::sleepClandidateFunction);
134 | } catch (Exception e) {
135 | candidateThrew = true;
136 | }
137 | Date date2 = new Date();
138 | long difference = date2.getTime() - date1.getTime();
139 |
140 | assertThat(difference).isLessThan(2000);
141 | assertThat(difference).isGreaterThanOrEqualTo(1000);
142 | assertThat(val).isEqualTo(3);
143 | assertThat(candidateThrew).isFalse();
144 | }
145 | }
146 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/TestPublishExperiment.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.metrics.MetricsProvider;
4 |
5 | import static org.assertj.core.api.Assertions.assertThat;
6 |
7 | public class TestPublishExperiment extends Experiment {
8 | TestPublishExperiment(String name, MetricsProvider metricsProvider) {
9 | super(name, metricsProvider);
10 | }
11 |
12 | @Override
13 | protected void publish(Result r) {
14 | assertThat(r.getCandidate().get().getDuration()).isGreaterThan(0L);
15 | assertThat(r.getControl().getDuration()).isGreaterThan(0L);
16 | }
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/TestPublishIncompatibleTypesExperiment.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j;
2 |
3 | import com.github.rawls238.scientist4j.metrics.MetricsProvider;
4 |
5 | import java.util.function.BiPredicate;
6 |
7 | import static org.assertj.core.api.Assertions.assertThat;
8 |
9 | public class TestPublishIncompatibleTypesExperiment extends IncompatibleTypesExperiment {
10 | TestPublishIncompatibleTypesExperiment(String name, MetricsProvider metricsProvider, BiPredicate comparator) {
11 | super(name, metricsProvider, comparator);
12 | }
13 |
14 | @Override
15 | protected void publish(IncompatibleTypesExperimentResult result) {
16 | assertThat(result.getCandidate().get().getDuration()).isGreaterThan(0L);
17 | assertThat(result.getControl().getDuration()).isGreaterThan(0L);
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/Scientist4JCore/src/test/java/com/github/rawls238/scientist4j/metrics/NoopMetricsProvider.java:
--------------------------------------------------------------------------------
1 | package com.github.rawls238.scientist4j.metrics;
2 |
3 | import com.github.rawls238.scientist4j.metrics.MetricsProvider;
4 |
5 | /**
6 | * A minimal in-memory {@link MetricsProvider} implementation, suitable for test environments.
7 | */
8 | public class NoopMetricsProvider implements MetricsProvider