├── .travis ├── codesigning.asc.enc ├── deploy.sh └── settings.xml ├── .gitignore ├── LICENSE.txt ├── .travis.yml ├── src ├── main │ └── java │ │ └── io │ │ └── github │ │ └── artsok │ │ ├── internal │ │ ├── RepeatedIfException.java │ │ ├── RepeatedIfExceptionsDisplayNameFormatter.java │ │ ├── ParameterizedTestInvocationContext.java │ │ ├── ParameterizedTestParameterResolver.java │ │ ├── ParameterizedRepeatedIfExceptionsTestNameFormatter.java │ │ ├── RepeatedIfExceptionsInvocationContext.java │ │ └── ParameterizedRepeatedMethodContext.java │ │ ├── RepeatedIfExceptionsTest.java │ │ ├── ParameterizedRepeatedIfExceptionsTest.java │ │ └── extension │ │ ├── RepeatIfExceptionsExtension.java │ │ └── ParameterizedRepeatedExtension.java └── test │ └── java │ └── io │ └── github │ └── artsok │ ├── MockitoTest.java │ └── ReRunnerTest.java ├── pom.xml └── README.md /.travis/codesigning.asc.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/artsok/rerunner-jupiter/HEAD/.travis/codesigning.asc.enc -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Eclipse stuff # 2 | ################# 3 | .classpath 4 | .project 5 | .settings/ 6 | 7 | # Intellij stuff # 8 | ################## 9 | *.iml 10 | *.ipr 11 | *.iws 12 | .idea/ 13 | 14 | # Build and unit tests stuff # 15 | ############################# 16 | logs/ 17 | target/ 18 | pom.xml.tag 19 | pom.xml.releaseBackup 20 | pom.xml.next 21 | release.properties 22 | work/ 23 | amps-standalone/ 24 | */dependency-reduced-pom.xml 25 | *.userprefs 26 | bin/ 27 | obj/ 28 | 29 | # Mac stuff # 30 | ############# 31 | .DS_Store 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this file except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | sudo: false 3 | dist: trusty 4 | 5 | addons: 6 | sonarcloud: 7 | organization: "artsok-github" 8 | token: 9 | secure: $SONAR_TOKEN 10 | 11 | cache: 12 | directories: 13 | - '$HOME/.m2/repository' 14 | 15 | jdk: 16 | - oraclejdk12 17 | 18 | script: 19 | - mvn test sonar:sonar 20 | 21 | deploy: 22 | - provider: script 23 | script: ".travis/deploy.sh" 24 | on: 25 | repo: artsok/rerunner-jupiter 26 | branch: master 27 | 28 | - provider: script 29 | script: ".travis/deploy.sh" 30 | skip_cleanup: true 31 | on: 32 | repo: artsok/rerunner-jupiter 33 | tags: true -------------------------------------------------------------------------------- /.travis/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | openssl aes-256-cbc -K $encrypted_f6a37a5809a3_key -iv $encrypted_f6a37a5809a3_iv -in .travis/codesigning.asc.enc -out .travis/codesigning.asc -d 4 | gpg --fast-import .travis/codesigning.asc 5 | 6 | if [ ! -z "$TRAVIS_TAG" ] 7 | then 8 | echo "on a tag -> set pom.xml to $TRAVIS_TAG" 9 | mvn --settings .travis/settings.xml org.codehaus.mojo:versions-maven-plugin:2.1:set -DnewVersion=$TRAVIS_TAG 1>/dev/null 2>/dev/null 10 | #mvn versions:set -DnewVersion=$TRAVIS_TAG 11 | else 12 | echo "without tag -> keep snapshot version in pom.xml" 13 | fi 14 | 15 | #mvn clean deploy --settings .travis/settings.xml -DskipTests=true -B -U 16 | 17 | mvn clean deploy -P release --settings .travis/settings.xml -------------------------------------------------------------------------------- /.travis/settings.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | ossrh 6 | ${env.SONATYPE_USERNAME} 7 | ${env.SONATYPE_PASSWORD} 8 | 9 | 10 | 11 | 12 | 13 | ossrh 14 | 15 | true 16 | 17 | 18 | gpg 19 | ${env.GPG_KEY_NAME} 20 | ${env.GPG_PASSPHRASE} 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/RepeatedIfException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2017 Artem Sokovets (http://github.com/artsok/) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package io.github.artsok.internal; 18 | 19 | /** 20 | * Custom exception of rerunner-jupiter extension. 21 | * 22 | * @author Artem Sokovets 23 | */ 24 | public class RepeatedIfException extends RuntimeException { 25 | 26 | private static final long serialVersionUID = -453323380293211043L; 27 | 28 | public RepeatedIfException(String message) { 29 | super(message); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/io/github/artsok/MockitoTest.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok; 2 | 3 | import io.github.artsok.internal.RepeatedIfException; 4 | import io.github.artsok.internal.RepeatedIfExceptionsInvocationContext; 5 | import org.junit.jupiter.api.Test; 6 | 7 | import static org.junit.jupiter.api.Assertions.*; 8 | import static org.mockito.Mockito.mock; 9 | import static org.mockito.Mockito.when; 10 | 11 | class MockitoTest { 12 | 13 | @Test 14 | void testRepeatedIfException() { 15 | Throwable exception = assertThrows(RepeatedIfException.class, () -> { 16 | throw new RepeatedIfException("RepeatedIfException"); 17 | }); 18 | assertEquals("RepeatedIfException", exception.getMessage()); 19 | } 20 | 21 | @Test 22 | void testRepeatedIfExceptionsInvocationContext() { 23 | RepeatedIfExceptionsInvocationContext repeatedIfExceptionsInvocationContext 24 | = mock(RepeatedIfExceptionsInvocationContext.class); 25 | when(repeatedIfExceptionsInvocationContext.getDisplayName(2)) 26 | .thenReturn("Repetition if test failed 1 of 2"); 27 | assertSame(repeatedIfExceptionsInvocationContext.getDisplayName(2), 28 | "Repetition if test failed 1 of 2", "testRepeatedIfExceptionsInvocationContext"); 29 | } 30 | 31 | } -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/RepeatedIfExceptionsDisplayNameFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2017 Artem Sokovets (http://github.com/artsok/) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package io.github.artsok.internal; 18 | 19 | import io.github.artsok.RepeatedIfExceptionsTest; 20 | 21 | /** 22 | * Formatter for extension point @RepeatedIfExceptions 23 | * 24 | * @author Artem Sokovets 25 | */ 26 | public class RepeatedIfExceptionsDisplayNameFormatter { 27 | 28 | private final String pattern; 29 | private final String displayName; 30 | 31 | public RepeatedIfExceptionsDisplayNameFormatter(final String pattern, final String displayName) { 32 | this.pattern = pattern; 33 | this.displayName = displayName; 34 | } 35 | 36 | String format(final int currentRepetition, final int totalRepetitions) { 37 | if (currentRepetition > 1 && totalRepetitions > 0) { 38 | final String result = pattern 39 | .replace(RepeatedIfExceptionsTest.CURRENT_REPETITION_PLACEHOLDER, String.valueOf(currentRepetition - 1)) //Minus, because first run doesn't mean repetition 40 | .replace(RepeatedIfExceptionsTest.TOTAL_REPETITIONS_PLACEHOLDER, String.valueOf(totalRepetitions - 1)); 41 | return this.displayName.concat(" (").concat(result).concat(")"); 42 | } else { 43 | return this.displayName; 44 | } 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/ParameterizedTestInvocationContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2019 the original author or authors. 3 | * 4 | * All rights reserved. This program and the accompanying materials are 5 | * made available under the terms of the Eclipse Public License v2.0 which 6 | * accompanies this distribution and is available at 7 | * 8 | * http://www.eclipse.org/legal/epl-v20.html 9 | */ 10 | 11 | package io.github.artsok.internal; 12 | 13 | import org.junit.jupiter.api.extension.Extension; 14 | import org.junit.jupiter.api.extension.TestTemplateInvocationContext; 15 | 16 | import java.util.Collections; 17 | import java.util.List; 18 | 19 | /** 20 | * @since 5.0 - COPY PAST FROM ORIGINAL JUNIT 5 WITH SEVERAL CORRECTIONS 21 | */ 22 | public class ParameterizedTestInvocationContext implements TestTemplateInvocationContext { 23 | 24 | private final int currentRepetition; 25 | private final int totalRepetitions; 26 | 27 | 28 | private final ParameterizedRepeatedIfExceptionsTestNameFormatter formatter; 29 | private final ParameterizedRepeatedMethodContext methodContext; 30 | private final Object[] arguments; 31 | 32 | public ParameterizedTestInvocationContext(int currentRepetition, int totalRepetitions, ParameterizedRepeatedIfExceptionsTestNameFormatter formatter, 33 | ParameterizedRepeatedMethodContext methodContext, Object[] arguments) { 34 | this.currentRepetition = currentRepetition; 35 | this.totalRepetitions = totalRepetitions; 36 | this.formatter = formatter; 37 | this.methodContext = methodContext; 38 | this.arguments = arguments; 39 | } 40 | 41 | @Override 42 | public String getDisplayName(int invocationIndex) { 43 | return this.formatter.format(invocationIndex, this.currentRepetition, this.totalRepetitions, this.arguments); 44 | } 45 | 46 | @Override 47 | public List getAdditionalExtensions() { 48 | return Collections.singletonList(new ParameterizedTestParameterResolver(this.methodContext, this.arguments)); 49 | } 50 | } 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/ParameterizedTestParameterResolver.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok.internal; 2 | 3 | import org.junit.jupiter.api.extension.ExtensionContext; 4 | import org.junit.jupiter.api.extension.ParameterContext; 5 | import org.junit.jupiter.api.extension.ParameterResolutionException; 6 | import org.junit.jupiter.api.extension.ParameterResolver; 7 | 8 | import java.lang.reflect.Executable; 9 | import java.lang.reflect.Method; 10 | 11 | /** 12 | * @since 5.0 - FULL COPY PAST FROM ORIGINAL JUNIT 5 13 | */ 14 | public class ParameterizedTestParameterResolver implements ParameterResolver { 15 | 16 | private final ParameterizedRepeatedMethodContext methodContext; 17 | private final Object[] arguments; 18 | 19 | public ParameterizedTestParameterResolver(ParameterizedRepeatedMethodContext methodContext, Object[] arguments) { 20 | this.methodContext = methodContext; 21 | this.arguments = arguments; 22 | } 23 | 24 | @Override 25 | public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) { 26 | Executable declaringExecutable = parameterContext.getDeclaringExecutable(); 27 | Method testMethod = extensionContext.getTestMethod().orElse(null); 28 | 29 | // Not a @ParameterizedTest method? 30 | if (!declaringExecutable.equals(testMethod)) { 31 | return false; 32 | } 33 | 34 | // Current parameter is an aggregator? 35 | if (this.methodContext.isAggregator(parameterContext.getIndex())) { 36 | return true; 37 | } 38 | 39 | // Ensure that the current parameter is declared before aggregators. 40 | // Otherwise, a different ParameterResolver should handle it. 41 | if (this.methodContext.indexOfFirstAggregator() != -1) { 42 | return parameterContext.getIndex() < this.methodContext.indexOfFirstAggregator(); 43 | } 44 | 45 | // Else fallback to behavior for parameterized test methods without aggregators. 46 | return parameterContext.getIndex() < this.arguments.length; 47 | } 48 | 49 | @Override 50 | public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) 51 | throws ParameterResolutionException { 52 | return this.methodContext.resolve(parameterContext, this.arguments); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/RepeatedIfExceptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2017 Artem Sokovets (http://github.com/artsok/) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package io.github.artsok; 18 | 19 | import io.github.artsok.extension.RepeatIfExceptionsExtension; 20 | import org.junit.jupiter.api.TestInfo; 21 | import org.junit.jupiter.api.TestTemplate; 22 | import org.junit.jupiter.api.extension.ExtendWith; 23 | 24 | import java.lang.annotation.ElementType; 25 | import java.lang.annotation.Retention; 26 | import java.lang.annotation.RetentionPolicy; 27 | import java.lang.annotation.Target; 28 | 29 | 30 | @Target({ElementType.METHOD, ElementType.TYPE}) 31 | @Retention(RetentionPolicy.RUNTIME) 32 | @TestTemplate 33 | @ExtendWith(RepeatIfExceptionsExtension.class) 34 | public @interface RepeatedIfExceptionsTest { 35 | 36 | /** 37 | * Placeholder for the {@linkplain TestInfo#getDisplayName display name} of 38 | * a {@code @RepeatedTest} method: {displayName} 39 | */ 40 | String DISPLAY_NAME_PLACEHOLDER = "{displayName}"; 41 | 42 | /** 43 | * Placeholder for the current repetition count of a {@code @RepeatedTest} 44 | * method: {currentRepetition} 45 | */ 46 | String CURRENT_REPETITION_PLACEHOLDER = "{currentRepetition}"; 47 | 48 | /** 49 | * Placeholder for the total number of repetitions of a {@code @RepeatedTest} 50 | * method: {totalRepetitions} 51 | */ 52 | String TOTAL_REPETITIONS_PLACEHOLDER = "{totalRepetitions}"; 53 | 54 | /** 55 | * Short display name pattern for a repeated test: {@value #SHORT_DISPLAY_NAME} 56 | * 57 | * @see #CURRENT_REPETITION_PLACEHOLDER 58 | * @see #TOTAL_REPETITIONS_PLACEHOLDER 59 | * @see #LONG_DISPLAY_NAME 60 | */ 61 | String SHORT_DISPLAY_NAME = "Repetition " + CURRENT_REPETITION_PLACEHOLDER + " of " + TOTAL_REPETITIONS_PLACEHOLDER; 62 | 63 | /** 64 | * Long display name pattern for a repeated test: {@value #LONG_DISPLAY_NAME} 65 | * 66 | * @see #DISPLAY_NAME_PLACEHOLDER 67 | * @see #SHORT_DISPLAY_NAME 68 | */ 69 | String LONG_DISPLAY_NAME = DISPLAY_NAME_PLACEHOLDER + " :: " + SHORT_DISPLAY_NAME; 70 | 71 | /** 72 | * Pool of exceptions 73 | * 74 | * @return Exception that handlered 75 | */ 76 | Class[] exceptions() default Throwable.class; 77 | 78 | /** 79 | * Number of repeats 80 | * 81 | * @return N-times repeat test if it failed 82 | */ 83 | int repeats() default 1; 84 | 85 | /** 86 | * Minimum success 87 | * 88 | * @return After n-times of passed tests will disable all remaining repeats. 89 | */ 90 | int minSuccess() default 1; 91 | 92 | 93 | /** 94 | * Add break (cooldown) to each tests. 95 | * It matters, when you get some infrastructure problems and you want to run your tests through timeout. 96 | * 97 | * @return the length of time to sleep in milliseconds 98 | */ 99 | long suspend() default 0L; 100 | 101 | /** 102 | * Display name for test method 103 | * 104 | * @return Short name 105 | */ 106 | String name() default SHORT_DISPLAY_NAME; 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/ParameterizedRepeatedIfExceptionsTestNameFormatter.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok.internal; 2 | 3 | import org.junit.platform.commons.JUnitException; 4 | import org.junit.platform.commons.util.StringUtils; 5 | 6 | import java.text.Format; 7 | import java.text.MessageFormat; 8 | import java.util.Arrays; 9 | import java.util.stream.IntStream; 10 | 11 | import static io.github.artsok.ParameterizedRepeatedIfExceptionsTest.ARGUMENTS_PLACEHOLDER; 12 | import static io.github.artsok.ParameterizedRepeatedIfExceptionsTest.CURRENT_REPETITION_PLACEHOLDER; 13 | import static io.github.artsok.ParameterizedRepeatedIfExceptionsTest.DISPLAY_NAME_PLACEHOLDER; 14 | import static io.github.artsok.ParameterizedRepeatedIfExceptionsTest.INDEX_PLACEHOLDER; 15 | import static io.github.artsok.ParameterizedRepeatedIfExceptionsTest.TOTAL_REPETITIONS_PLACEHOLDER; 16 | import static java.util.stream.Collectors.joining; 17 | 18 | 19 | public class ParameterizedRepeatedIfExceptionsTestNameFormatter { 20 | 21 | private final String pattern; 22 | private final String displayName; 23 | private final String repeatedNamePattern; 24 | 25 | public ParameterizedRepeatedIfExceptionsTestNameFormatter(String pattern, String displayName, String repeatedNamePattern) { 26 | this.pattern = pattern; 27 | this.displayName = displayName; 28 | this.repeatedNamePattern = repeatedNamePattern; 29 | } 30 | 31 | 32 | String format(int invocationIndex, int currentRepetition, int totalRepetitions, Object... arguments) { 33 | try { 34 | return formatSafely(invocationIndex, currentRepetition, totalRepetitions, arguments); 35 | } catch (Exception ex) { 36 | String message = "The display name pattern defined for the parameterized test is invalid. " 37 | + "See nested exception for further details."; 38 | throw new JUnitException(message, ex); 39 | } 40 | } 41 | 42 | private String formatSafely(int invocationIndex, int currentRepetition, int totalRepetitions, Object[] arguments) { 43 | String pattern = prepareMessageFormatPattern(invocationIndex, currentRepetition, totalRepetitions, arguments); 44 | MessageFormat format = new MessageFormat(pattern); 45 | Object[] humanReadableArguments = makeReadable(format, arguments); 46 | return format.format(humanReadableArguments); 47 | } 48 | 49 | 50 | /** 51 | * Format display message. If exceptions appears use one or other pattern. 52 | * @param invocationIndex - Index of argument 53 | * @param currentRepetition - Indicate the current repeating index if exceptions appear 54 | * @param totalRepetitions - Amount of all repeats 55 | * @param arguments - Method arguments 56 | * @return {@link String} - Displayed text 57 | */ 58 | private String prepareMessageFormatPattern(int invocationIndex, int currentRepetition, int totalRepetitions, Object[] arguments) { 59 | String result; 60 | if (currentRepetition != 0 && totalRepetitions != 0) { 61 | result = pattern 62 | .replace(DISPLAY_NAME_PLACEHOLDER, this.displayName) 63 | .replace(INDEX_PLACEHOLDER, String.valueOf(invocationIndex)) 64 | .concat(repeatedNamePattern) 65 | .replace(CURRENT_REPETITION_PLACEHOLDER, String.valueOf(currentRepetition)) 66 | .replace(TOTAL_REPETITIONS_PLACEHOLDER, String.valueOf(totalRepetitions)); 67 | } else { 68 | result = pattern 69 | .replace(DISPLAY_NAME_PLACEHOLDER, this.displayName) 70 | .replace(INDEX_PLACEHOLDER, String.valueOf(invocationIndex)) 71 | .replace(repeatedNamePattern, ""); 72 | } 73 | 74 | 75 | if (result.contains(ARGUMENTS_PLACEHOLDER)) { 76 | // @formatter:off 77 | String replacement = IntStream.range(0, arguments.length) 78 | .mapToObj(index -> "{" + index + "}") 79 | .collect(joining(", ")); 80 | // @formatter:on 81 | result = result.replace(ARGUMENTS_PLACEHOLDER, replacement); 82 | } 83 | return result; 84 | } 85 | 86 | private Object[] makeReadable(MessageFormat format, Object[] arguments) { 87 | Format[] formats = format.getFormatsByArgumentIndex(); 88 | Object[] result = Arrays.copyOf(arguments, Math.min(arguments.length, formats.length), Object[].class); 89 | for (int i = 0; i < result.length; i++) { 90 | if (formats[i] == null) { 91 | result[i] = StringUtils.nullSafeToString(arguments[i]); 92 | } 93 | } 94 | return result; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/RepeatedIfExceptionsInvocationContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2017 Artem Sokovets (http://github.com/artsok/) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package io.github.artsok.internal; 18 | 19 | import org.junit.jupiter.api.extension.*; 20 | 21 | import java.util.List; 22 | 23 | import static java.util.Collections.singletonList; 24 | 25 | 26 | /** 27 | * Context for extension point @RepeatedIfExceptions 28 | * 29 | * @author Artem Sokovets 30 | */ 31 | public class RepeatedIfExceptionsInvocationContext implements TestTemplateInvocationContext { 32 | 33 | private final int currentRepetition; 34 | private final int totalTestRuns; 35 | private final int successfulTestRepetitionsCount; 36 | private final int minSuccess; 37 | private final boolean repeatableExceptionAppeared; 38 | private final RepeatedIfExceptionsDisplayNameFormatter formatter; 39 | 40 | public RepeatedIfExceptionsInvocationContext(int currentRepetition, int totalRepetitions, int successfulTestRepetitionsCount, 41 | int minSuccess, boolean repeatableExceptionAppeared, 42 | RepeatedIfExceptionsDisplayNameFormatter formatter) { 43 | this.currentRepetition = currentRepetition; 44 | this.totalTestRuns = totalRepetitions; 45 | this.successfulTestRepetitionsCount = successfulTestRepetitionsCount; 46 | this.minSuccess = minSuccess; 47 | this.repeatableExceptionAppeared = repeatableExceptionAppeared; 48 | this.formatter = formatter; 49 | } 50 | 51 | @Override 52 | public String getDisplayName(int invocationIndex) { 53 | return this.formatter.format(this.currentRepetition, this.totalTestRuns); 54 | } 55 | 56 | @Override 57 | public List getAdditionalExtensions() { 58 | return singletonList(new RepeatExecutionCondition(currentRepetition, totalTestRuns, minSuccess, 59 | successfulTestRepetitionsCount, repeatableExceptionAppeared)); 60 | } 61 | } 62 | 63 | /** 64 | * Implements ExecutionCondition interface. 65 | * With one method in this interface, we can control of on/off executing test 66 | */ 67 | class RepeatExecutionCondition implements ExecutionCondition { 68 | private final int totalTestRuns; 69 | private final int minSuccess; 70 | private final int successfulTestRepetitionsCount; 71 | private final int failedTestRepetitionsCount; 72 | private final boolean repeatableExceptionAppeared; 73 | 74 | RepeatExecutionCondition(int currentRepetition, int totalRepetitions, int minSuccess, 75 | int successfulTestRepetitionsCount, boolean repeatableExceptionAppeared) { 76 | this.totalTestRuns = totalRepetitions; 77 | this.minSuccess = minSuccess; 78 | this.successfulTestRepetitionsCount = successfulTestRepetitionsCount; 79 | this.failedTestRepetitionsCount = currentRepetition - successfulTestRepetitionsCount - 1; 80 | this.repeatableExceptionAppeared = repeatableExceptionAppeared; 81 | } 82 | 83 | @Override 84 | public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { 85 | if (testUltimatelyFailed()) { 86 | return ConditionEvaluationResult.disabled("Turn off the remaining repetitions as the test ultimately failed"); 87 | } else if (testUltimatelyPassed()) { 88 | return ConditionEvaluationResult.disabled("Turn off the remaining repetitions as the test ultimately passed"); 89 | } else { 90 | return ConditionEvaluationResult.enabled("Repeat the tests"); 91 | } 92 | } 93 | 94 | private boolean testUltimatelyFailed() { 95 | return aNonRepeatableExceptionAppeared() || minimalRequiredSuccessfulRunsCannotBeReachedAnymore(); 96 | } 97 | 98 | private boolean aNonRepeatableExceptionAppeared() { 99 | return failedTestRepetitionsCount > 0 && !repeatableExceptionAppeared; 100 | } 101 | 102 | private boolean minimalRequiredSuccessfulRunsCannotBeReachedAnymore() { 103 | return totalTestRuns - failedTestRepetitionsCount < minSuccess; 104 | } 105 | 106 | private boolean testUltimatelyPassed() { 107 | return successfulTestRepetitionsCount >= minSuccess; 108 | } 109 | } 110 | 111 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/ParameterizedRepeatedIfExceptionsTest.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok; 2 | 3 | import io.github.artsok.extension.ParameterizedRepeatedExtension; 4 | import org.apiguardian.api.API; 5 | import org.junit.jupiter.api.TestTemplate; 6 | import org.junit.jupiter.api.extension.ExtendWith; 7 | 8 | import java.lang.annotation.ElementType; 9 | import java.lang.annotation.Retention; 10 | import java.lang.annotation.RetentionPolicy; 11 | import java.lang.annotation.Target; 12 | 13 | import static org.apiguardian.api.API.Status.EXPERIMENTAL; 14 | 15 | /** 16 | * Annotation which you can put to parameterized test method. 17 | * Customize the number of repeats and set for what exception you want handled. 18 | * By default set Throwable.class. 19 | * All logic of this extension at {@link ParameterizedRepeatedExtension} 20 | * 21 | * @author Artem Sokovets 22 | */ 23 | @Target({ElementType.METHOD, ElementType.TYPE}) 24 | @Retention(RetentionPolicy.RUNTIME) 25 | @TestTemplate 26 | @ExtendWith(ParameterizedRepeatedExtension.class) 27 | public @interface ParameterizedRepeatedIfExceptionsTest { 28 | 29 | /** 30 | * Placeholder for the current repetition count of a {@code @RepeatedTest} 31 | * method: {currentRepetition} 32 | */ 33 | String CURRENT_REPETITION_PLACEHOLDER = "{currentRepetition}"; 34 | 35 | /** 36 | * Placeholder for the total number of repetitions of a {@code @RepeatedTest} 37 | * method: {totalRepetitions} 38 | */ 39 | String TOTAL_REPETITIONS_PLACEHOLDER = "{totalRepetitions}"; 40 | 41 | /** 42 | * Display name pattern for a repeated test 43 | * 44 | * @see #CURRENT_REPETITION_PLACEHOLDER 45 | * @see #TOTAL_REPETITIONS_PLACEHOLDER 46 | */ 47 | String REPEATED_DISPLAY_NAME = " (Repeated if the test failed " + CURRENT_REPETITION_PLACEHOLDER + " of " + TOTAL_REPETITIONS_PLACEHOLDER + ")"; 48 | 49 | /** 50 | * Placeholder for the {@linkplain org.junit.jupiter.api.TestInfo#getDisplayName 51 | * display name} of a {@code @ParameterizedTest} method: {displayName} 52 | * 53 | * @see #name 54 | * @since 5.3 55 | */ 56 | @API(status = EXPERIMENTAL, since = "5.3") 57 | String DISPLAY_NAME_PLACEHOLDER = "{displayName}"; 58 | 59 | /** 60 | * Placeholder for the current invocation index of a {@code @ParameterizedTest} 61 | * method (1-based): {index} 62 | * 63 | * @see #name 64 | * @since 5.3 65 | */ 66 | @API(status = EXPERIMENTAL, since = "5.3") 67 | String INDEX_PLACEHOLDER = "{index}"; 68 | 69 | /** 70 | * Placeholder for the complete, comma-separated arguments list of the 71 | * current invocation of a {@code @ParameterizedTest} method: 72 | * {arguments} 73 | * 74 | * @see #name 75 | * @since 5.3 76 | */ 77 | @API(status = EXPERIMENTAL, since = "5.3") 78 | String ARGUMENTS_PLACEHOLDER = "{arguments}"; 79 | 80 | /** 81 | * Default display name pattern for the current invocation of a 82 | * {@code @ParameterizedTest} method: {@value} 83 | * 84 | *

Note that the default pattern does not include the 85 | * {@linkplain #DISPLAY_NAME_PLACEHOLDER display name} of the 86 | * {@code @ParameterizedTest} method. 87 | * 88 | * @see #name 89 | * @see #DISPLAY_NAME_PLACEHOLDER 90 | * @see #INDEX_PLACEHOLDER 91 | * @see #ARGUMENTS_PLACEHOLDER 92 | * @since 5.3 93 | */ 94 | @API(status = EXPERIMENTAL, since = "5.3") 95 | String DEFAULT_DISPLAY_NAME = "[" + INDEX_PLACEHOLDER + "] " + ARGUMENTS_PLACEHOLDER; 96 | 97 | /** 98 | * The display name to be used for individual invocations of the 99 | * parameterized test; never blank or consisting solely of whitespace. 100 | * 101 | * @return {@link String} 102 | */ 103 | String name() default DEFAULT_DISPLAY_NAME; 104 | 105 | /** 106 | * The display name to be used for individual repeated invocations of the 107 | * parameterized test; never blank. 108 | * 109 | * @return {@link String} 110 | */ 111 | String repeatedName() default REPEATED_DISPLAY_NAME; 112 | 113 | 114 | /** 115 | * Pool of exceptions 116 | * 117 | * @return Exception that handlered 118 | */ 119 | Class[] exceptions() default Throwable.class; 120 | 121 | /** 122 | * Number of repeats 123 | * 124 | * @return N-times repeat test if it failed 125 | */ 126 | int repeats() default 1; 127 | 128 | /** 129 | * Minimum success 130 | * 131 | * @return After n-times of passed tests will disable all remaining repeats. 132 | */ 133 | int minSuccess() default 1; 134 | 135 | /** 136 | * Add break (cooldown) to each tests. 137 | * It matters, when you get some infrastructure problems and you want to run your tests through timeout. 138 | * 139 | * @return the length of time to sleep in milliseconds 140 | */ 141 | long suspend() default 0L; 142 | } 143 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | io.github.artsok 7 | rerunner-jupiter 8 | 2.1.7-SNAPSHOT 9 | Rerunner-Jupiter 10 | 11 | JUnit 5 Extension point which rerun failed tests certain number of times 12 | 13 | https://github.com/artsok/rerunner-jupiter 14 | 15 | 16 | artsok 17 | Artem Sokovets 18 | artem@sokovets.ru 19 | 20 | 21 | 22 | 23 | Apache 2.0 24 | http://www.apache.org/licenses/LICENSE-2.0 25 | 26 | 27 | 28 | https://github.com/artsok/rerunner-jupiter 29 | scm:git:git://github.com/artsok/rerunner-jupiter.git 30 | scm:git:git@github.com:artsok/rerunner-jupiter.git 31 | 32 | 33 | GitHub Issues 34 | https://github.com/artsok/rerunner-jupiter/issues 35 | 36 | 37 | 5.4.2 38 | 1.18.8 39 | 2.28.2 40 | 1.8 41 | 1.8 42 | 1.6 43 | 3.1.0 44 | 3.1.1 45 | UTF-8 46 | 47 | 48 | 49 | ossrh 50 | https://oss.sonatype.org/content/repositories/snapshots 51 | 52 | 53 | ossrh 54 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 55 | 56 | 57 | 58 | 59 | 60 | org.junit 61 | junit-bom 62 | ${junit.version} 63 | pom 64 | import 65 | 66 | 67 | 68 | 69 | 70 | org.junit.jupiter 71 | junit-jupiter-api 72 | 73 | 74 | org.junit.jupiter 75 | junit-jupiter-engine 76 | 77 | 78 | org.junit.jupiter 79 | junit-jupiter-params 80 | 81 | 82 | org.junit.platform 83 | junit-platform-runner 84 | test 85 | 86 | 87 | org.projectlombok 88 | lombok 89 | ${lombok.version} 90 | test 91 | 92 | 93 | org.mockito 94 | mockito-core 95 | ${mockito-core.version} 96 | test 97 | 98 | 99 | 100 | 101 | 102 | 103 | maven-compiler-plugin 104 | 3.8.1 105 | 106 | ${maven.compiler.source} 107 | ${maven.compiler.target} 108 | 109 | 110 | 111 | org.sonatype.plugins 112 | nexus-staging-maven-plugin 113 | 1.6.8 114 | true 115 | 116 | ossrh 117 | https://oss.sonatype.org/ 118 | true 119 | 120 | 121 | 122 | maven-surefire-plugin 123 | 2.22.2 124 | 125 | programmatic-tests 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | build-extras 134 | 135 | true 136 | 137 | 138 | 139 | 140 | maven-source-plugin 141 | ${maven-source-plugin.version} 142 | 143 | 144 | attach-sources 145 | 146 | jar-no-fork 147 | 148 | 149 | 150 | 151 | 152 | maven-javadoc-plugin 153 | ${maven-javadoc-plugin.version} 154 | 155 | 8 156 | 157 | 158 | 159 | attach-javadocs 160 | 161 | jar 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | release 171 | 172 | 173 | performRelease 174 | true 175 | 176 | 177 | 178 | 179 | 180 | org.apache.maven.plugins 181 | maven-gpg-plugin 182 | ${maven-gpg-plugin.version} 183 | 184 | ${gpg.passphrase} 185 | 186 | 187 | 188 | sign-artifacts 189 | verify 190 | 191 | sign 192 | 193 | 194 | 195 | 196 | 197 | maven-source-plugin 198 | ${maven-source-plugin.version} 199 | 200 | 201 | attach-sources 202 | 203 | jar-no-fork 204 | 205 | 206 | 207 | 208 | 209 | maven-javadoc-plugin 210 | ${maven-javadoc-plugin.version} 211 | 212 | 8 213 | 214 | 215 | 216 | attach-javadocs 217 | 218 | jar 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rerunner-jupiter 2 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.github.artsok/rerunner-jupiter/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.github.artsok/rerunner-jupiter) 3 | [![Build Status](https://travis-ci.org/artsok/rerunner-jupiter.svg?branch=master)](https://travis-ci.org/artsok/rerunner-jupiter) 4 | ![badge-jdk-8](https://img.shields.io/badge/jdk-8-yellow.svg "JDK-8") 5 | [![License badge](https://img.shields.io/badge/license-Apache2-green.svg)](http://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | *rerunner-jupiter* is a extension for Junit 5. 8 | Re-run failed JUnit-Jupiter tests immediately. Very useful when your UI/API tests don't stable. 9 | This library is open source, released under the terms of [Apache 2.0 License]. 10 | 11 | ## How To Use 12 | 13 | In order to include *rerunner-jupiter* in a Maven project, first add the following dependency to your `pom.xml` (Java 8 required): 14 | 15 | ```xml 16 | 17 | io.github.artsok 18 | rerunner-jupiter 19 | 2.1.6 20 | test 21 | 22 | ``` 23 | 24 | ## Examples 25 | ```java 26 | /** 27 | * Repeated three times if test failed. 28 | * By default Exception.class will be handled in test 29 | */ 30 | @RepeatedIfExceptionsTest(repeats = 3) 31 | void reRunTest() throws IOException { 32 | throw new IOException("Error in Test"); 33 | } 34 | 35 | 36 | /** 37 | * Repeated two times if test failed. Set IOException.class that will be handled in test 38 | * @throws IOException - error occurred 39 | */ 40 | @RepeatedIfExceptionsTest(repeats = 2, exceptions = IOException.class) 41 | void reRunTest2() throws IOException { 42 | throw new IOException("Exception in I/O operation"); 43 | } 44 | 45 | 46 | /** 47 | * Repeated ten times if test failed. Set IOException.class that will be handled in test 48 | * Set formatter for test. Like behavior as at {@link org.junit.jupiter.api.RepeatedTest} 49 | * @throws IOException - error occurred 50 | */ 51 | @RepeatedIfExceptionsTest(repeats = 10, exceptions = IOException.class, 52 | name = "Rerun failed test. Attempt {currentRepetition} of {totalRepetitions}") 53 | void reRunTest3() throws IOException { 54 | throw new IOException("Exception in I/O operation"); 55 | } 56 | 57 | /** 58 | * Repeated three times if selenium test failed. Use selenium-jupiter extension. 59 | */ 60 | @RepeatedIfExceptionsTest(repeats = 3, exceptions = NoSuchElementException.class) 61 | void testWithChrome(ChromeDriver chrome) { 62 | chrome.get("http://yandex.ru"); 63 | chrome.findElement(By.xpath("//span[@id='authors']")); 64 | } 65 | 66 | /** 67 | * Repeated 100 times with minimum success four times, then disabled all remaining repeats. 68 | * See image below how it works. Default exception is Exception.class 69 | */ 70 | @DisplayName("Test Case Name") 71 | @RepeatedIfExceptionsTest(repeats = 100, minSuccess = 4) 72 | void reRunTest4() { 73 | if(random.nextInt() % 2 == 0) { 74 | throw new RuntimeException("Error in Test"); 75 | } 76 | } 77 | 78 | /** 79 | * By default total repeats = 1 and minimum success = 1. 80 | * If the test failed by this way start to repeat it by one time with one minimum success. 81 | * 82 | * This example without exceptions. 83 | */ 84 | @ParameterizedRepeatedIfExceptionsTest 85 | @ValueSource(ints = {14, 15, 100, -10}) 86 | void successfulParameterizedTest(int argument) { 87 | System.out.println(argument); 88 | } 89 | 90 | /** 91 | * By default total repeats = 1 and minimum success = 1. 92 | * If the test failed by this way start to repeat it by one time with one minimum success. 93 | * This example with display name but without exceptions 94 | */ 95 | @DisplayName("Example of parameterized repeated without exception") 96 | @ParameterizedRepeatedIfExceptionsTest 97 | @ValueSource(ints = {1, 2, 3, 1001}) 98 | void successfulParameterizedTestWithDisplayName(int argument) { 99 | System.out.println(argument); 100 | } 101 | 102 | /** 103 | * By default total repeats = 1 and minimum success = 1. 104 | * If the test failed by this way start to repeat it by one time with one minimum success. 105 | * 106 | * This example with display name but with exception. Exception depends on random number generation. 107 | */ 108 | @DisplayName("Example of parameterized repeated with exception") 109 | @ParameterizedRepeatedIfExceptionsTest 110 | @ValueSource(strings = {"Hi", "Hello", "Bonjour", "Privet"}) 111 | void errorParameterizedTestWithDisplayName(String argument) { 112 | if (random.nextInt() % 2 == 0) { 113 | throw new RuntimeException("Exception " + argument); 114 | } 115 | } 116 | 117 | /** 118 | * By default total repeats = 1 and minimum success = 1. 119 | * If the test failed by this way start to repeat it by one time with one minimum success. 120 | * 121 | * This example with display name, repeated display name, 10 repeats and 2 minimum success with exceptions. 122 | * Exception depends on random number generation. 123 | */ 124 | @ParameterizedRepeatedIfExceptionsTest(name = "Argument was {0}", 125 | repeatedName = " (Repeat {currentRepetition} of {totalRepetitions})", 126 | repeats = 10, exceptions = RuntimeException.class, minSuccess = 2) 127 | @ValueSource(ints = {4, 5, 6, 7}) 128 | void errorParameterizedTestWithDisplayNameAndRepeatedName(int argument) { 129 | if (random.nextInt() % 2 == 0) { 130 | throw new RuntimeException("Exception in Test " + argument); 131 | } 132 | } 133 | 134 | /** 135 | * By default total repeats = 1 and minimum success = 1. 136 | * If the test failed by this way start to repeat it by one time with one minimum success. 137 | * 138 | * This example with display name, implicitly repeated display name, 4 repeats and 2 minimum success with exceptions. 139 | * Exception depends on random number generation. Also use {@link MethodSource} 140 | */ 141 | @DisplayName("Display name of container") 142 | @ParameterizedRepeatedIfExceptionsTest(name = "Year {0} is a leap year.", 143 | repeats = 4, exceptions = RuntimeException.class, minSuccess = 2) 144 | @MethodSource("stringIntAndListProvider") 145 | void errorParameterizedTestWithMultiArgMethodSource(String str, int num, List list) { 146 | assertEquals(5, str.length()); 147 | assertTrue(num >= 1 && num <= 2); 148 | assertEquals(2, list.size()); 149 | if (random.nextInt() % 2 == 0) { 150 | throw new RuntimeException("Exception in Test"); 151 | } 152 | } 153 | 154 | static Stream stringIntAndListProvider() { 155 | return Stream.of( 156 | arguments("apple", 1, Arrays.asList("a", "b")), 157 | arguments("lemon", 2, Arrays.asList("x", "y")) 158 | ); 159 | } 160 | 161 | 162 | /** 163 | * it's often caused by some infrastructure problems: network congestion, garbage collection etc. 164 | * These problems usually pass after some time. Use suspend option 165 | */ 166 | @RepeatedIfExceptionsTest(repeats = 3, exceptions = IOException.class, suspend = 5000L) 167 | void reRunTestWithSuspendOption() throws IOException { 168 | throw new IOException("Exception in I/O operation"); 169 | } 170 | 171 | 172 | /** 173 | * Example with suspend option for Parameterized Test 174 | * It matters, when you get some infrastructure problems and you want to run your tests through timeout. 175 | * 176 | * Set break to 5 seconds. If exception appeared for any arguments, repeating extension would runs tests with break. 177 | * If one result failed and other passed, does not matter we would wait 5 seconds throught each arguments of the repeated tests. 178 | * 179 | */ 180 | @DisplayName("Example of parameterized repeated with exception") 181 | @ParameterizedRepeatedIfExceptionsTest(suspend = 5000L, minSuccess = 2, repeats = 3) 182 | @ValueSource(strings = {"Hi", "Hello", "Bonjour", "Privet"}) 183 | void errorParameterizedTestWithSuspendOption(String argument) { 184 | if (random.nextInt() % 2 == 0) { 185 | throw new RuntimeException(argument); 186 | } 187 | } 188 | 189 | /** 190 | * Parameterized Test with the wrong exception. 191 | * Test throws AssertionError.class, but we wait for Exception.class. 192 | * In this case test with argument "1" runs ones without repeats. 193 | * 194 | * If you change 'exceptions = AssertionError.class', repeats will appear. 195 | */ 196 | @ValueSource(ints = {1, 2}) 197 | @ParameterizedRepeatedIfExceptionsTest(repeats = 2, exceptions = Exception.class) 198 | void testParameterizedRepeaterAssertionsFailure(int value) { 199 | assertThat(value, equalTo(2)); 200 | } 201 | ``` 202 | More examples you can find [here]. 203 | 204 | 205 | ## GitHub Star 206 | Push to the [star] if you like this JUnit 5 Extension. By this way, I will get feedback from you! 207 | 208 | 209 | [here]: https://github.com/artsok/rerunner-jupiter/blob/master/src/test/java/io/github/artsok/ReRunnerTest.java 210 | [star]: https://github.com/artsok/rerunner-jupiter/stargazers 211 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/extension/RepeatIfExceptionsExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2017 Artem Sokovets (http://github.com/artsok/) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package io.github.artsok.extension; 18 | 19 | 20 | import io.github.artsok.RepeatedIfExceptionsTest; 21 | import io.github.artsok.internal.RepeatedIfException; 22 | import io.github.artsok.internal.RepeatedIfExceptionsDisplayNameFormatter; 23 | import io.github.artsok.internal.RepeatedIfExceptionsInvocationContext; 24 | import org.junit.jupiter.api.extension.*; 25 | import org.junit.platform.commons.util.Preconditions; 26 | import org.junit.platform.commons.util.StringUtils; 27 | import org.opentest4j.TestAbortedException; 28 | 29 | import java.util.*; 30 | import java.util.stream.Collectors; 31 | import java.util.stream.Stream; 32 | 33 | import static java.lang.Math.toIntExact; 34 | import static java.util.Spliterators.spliteratorUnknownSize; 35 | import static java.util.stream.StreamSupport.stream; 36 | import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation; 37 | import static org.junit.platform.commons.util.AnnotationUtils.isAnnotated; 38 | 39 | 40 | /** 41 | * Main condition for extension point @RepeatedIfExceptions 42 | * All logic in this class. See TestTemplateIterator where handler logic of repeat tests 43 | * 44 | * @author Artem Sokovets 45 | */ 46 | public class RepeatIfExceptionsExtension implements TestTemplateInvocationContextProvider, BeforeTestExecutionCallback, 47 | AfterTestExecutionCallback, TestExecutionExceptionHandler { 48 | 49 | 50 | private int repeats = 0; 51 | private int minSuccess = 1; 52 | private int totalTestRuns = 0; 53 | private List> repeatableExceptions; 54 | private boolean repeatableExceptionAppeared = false; 55 | private RepeatedIfExceptionsDisplayNameFormatter formatter; 56 | private List historyExceptionAppear; 57 | private long suspend = 0L; 58 | private static final int CURRENT_RUN = 1; 59 | 60 | 61 | /** 62 | * Check that test method contain {@link RepeatedIfExceptionsTest} annotation 63 | * 64 | * @param extensionContext - encapsulates the context in which the current test or container is being executed 65 | * @return true/false 66 | */ 67 | @Override 68 | public boolean supportsTestTemplate(ExtensionContext extensionContext) { 69 | return isAnnotated(extensionContext.getTestMethod(), RepeatedIfExceptionsTest.class); 70 | } 71 | 72 | 73 | /** 74 | * Context call TestTemplateInvocationContext 75 | * 76 | * @param extensionContext - Test Class Context 77 | * @return Stream of TestTemplateInvocationContext 78 | */ 79 | @Override 80 | public Stream provideTestTemplateInvocationContexts(ExtensionContext extensionContext) { 81 | Preconditions.notNull(extensionContext.getTestMethod().orElse(null), "Test method must not be null"); 82 | 83 | RepeatedIfExceptionsTest annotationParams = extensionContext.getTestMethod() 84 | .flatMap(testMethods -> findAnnotation(testMethods, RepeatedIfExceptionsTest.class)) 85 | .orElseThrow(() -> new RepeatedIfException("The extension should not be executed " 86 | + "unless the test method is annotated with @RepeatedIfExceptionsTest.")); 87 | 88 | 89 | int totalRepeats = annotationParams.repeats(); 90 | minSuccess = annotationParams.minSuccess(); 91 | Preconditions.condition(totalRepeats > 0, "Total repeats must be higher than 0"); 92 | Preconditions.condition(minSuccess >= 1, "Total minimum success must be higher or equals than 1"); 93 | 94 | totalTestRuns = totalRepeats + CURRENT_RUN; 95 | suspend = annotationParams.suspend(); 96 | historyExceptionAppear = Collections.synchronizedList(new ArrayList<>()); 97 | 98 | String displayName = extensionContext.getDisplayName(); 99 | formatter = displayNameFormatter(annotationParams, displayName); 100 | 101 | //Convert logic of repeated handler to spliterator 102 | Spliterator spliterator = 103 | spliteratorUnknownSize(new TestTemplateIterator(), Spliterator.NONNULL); 104 | return stream(spliterator, false); 105 | } 106 | 107 | @Override 108 | public void beforeTestExecution(ExtensionContext context) { 109 | 110 | //get TotalTestRuns and minSuccess from system properties 111 | String strTotalRepeats = System.getProperty("totalRepeats"); 112 | if(strTotalRepeats != null) { 113 | try { 114 | totalTestRuns = Integer.parseInt(strTotalRepeats); 115 | }catch(Exception e){ 116 | } 117 | } 118 | 119 | 120 | String strMinSuccess = System.getProperty("minSuccess"); 121 | if(strMinSuccess != null) { 122 | try { 123 | minSuccess = Integer.parseInt(strMinSuccess); 124 | }catch(Exception e){ 125 | } 126 | } 127 | 128 | repeatableExceptions = Stream.of(context.getTestMethod() 129 | .flatMap(testMethods -> findAnnotation(testMethods, RepeatedIfExceptionsTest.class)) 130 | .orElseThrow(() -> new IllegalStateException("The extension should not be executed ")) 131 | .exceptions() 132 | ).collect(Collectors.toList()); 133 | repeatableExceptions.add(TestAbortedException.class); 134 | } 135 | 136 | /** 137 | * Check if exceptions that will appear in test same as we wait 138 | * 139 | * @param extensionContext - Test Class Context 140 | */ 141 | @Override 142 | public void afterTestExecution(ExtensionContext extensionContext) { 143 | boolean exceptionAppeared = exceptionAppeared(extensionContext); 144 | historyExceptionAppear.add(exceptionAppeared); 145 | } 146 | 147 | private boolean exceptionAppeared(ExtensionContext extensionContext) { 148 | Class exception = extensionContext.getExecutionException() 149 | .orElse(new RepeatedIfException("There is no exception in context")).getClass(); 150 | return repeatableExceptions.stream() 151 | .anyMatch(ex -> ex.isAssignableFrom(exception) && !RepeatedIfException.class.isAssignableFrom(exception)); 152 | } 153 | 154 | /** 155 | * Handler for display name 156 | * 157 | * @param test - RepeatedIfExceptionsTest annotation 158 | * @param displayName - Name that will be represent to report 159 | * @return RepeatedIfExceptionsDisplayNameFormatter {@link RepeatedIfExceptionsDisplayNameFormatter} 160 | */ 161 | private RepeatedIfExceptionsDisplayNameFormatter displayNameFormatter(RepeatedIfExceptionsTest test, String displayName) { 162 | String pattern = test.name().trim(); 163 | if (StringUtils.isBlank(pattern)) { 164 | pattern = Optional.of(test.name()) 165 | .orElseThrow(() -> new RepeatedIfException("Exception occurred with name parameter of RepeatedIfExceptionsTest annotation")); 166 | } 167 | return new RepeatedIfExceptionsDisplayNameFormatter(pattern, displayName); 168 | } 169 | 170 | @Override 171 | public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable { 172 | if (appearedExceptionDoesNotAllowRepetitions(throwable)) { 173 | throw throwable; 174 | } 175 | repeatableExceptionAppeared = true; 176 | 177 | long currentSuccessCount = historyExceptionAppear.stream().filter(exceptionAppeared -> !exceptionAppeared).count(); 178 | if (currentSuccessCount < minSuccess) { 179 | if (isMinSuccessTargetStillReachable(minSuccess)) { 180 | throw new TestAbortedException("Do not fail completely, but repeat the test", throwable); 181 | } else { 182 | throw throwable; 183 | } 184 | } 185 | } 186 | 187 | /** 188 | * If exception allowed, will return false 189 | * 190 | * @param appearedException - {@link Throwable} 191 | * @return true/false 192 | */ 193 | private boolean appearedExceptionDoesNotAllowRepetitions(final Throwable appearedException) { 194 | return repeatableExceptions.stream().noneMatch(ex -> ex.isAssignableFrom(appearedException.getClass())); 195 | } 196 | 197 | /** 198 | * If cannot reach a minimum success target, will return true 199 | * 200 | * @param minSuccessCount - minimum success count 201 | * @return true/false 202 | */ 203 | private boolean isMinSuccessTargetStillReachable(final long minSuccessCount) { 204 | return historyExceptionAppear.stream().filter(bool -> bool).count() < totalTestRuns - minSuccessCount; 205 | } 206 | 207 | /** 208 | * TestTemplateIterator (Repeat test if it failed) 209 | */ 210 | class TestTemplateIterator implements Iterator { 211 | int currentIndex = 0; 212 | 213 | @Override 214 | public boolean hasNext() { 215 | if (currentIndex == 0) { 216 | return true; 217 | } 218 | return historyExceptionAppear.stream().anyMatch(ex -> ex) && currentIndex < totalTestRuns; 219 | } 220 | 221 | @Override 222 | public TestTemplateInvocationContext next() { 223 | //If exception appeared would wait suspend time 224 | if (historyExceptionAppear.stream().anyMatch(ex -> ex) && suspend != 0L) { 225 | try { 226 | Thread.sleep(suspend); 227 | } catch (InterruptedException e) { 228 | Thread.currentThread().interrupt(); 229 | } 230 | } 231 | 232 | int successfulTestRepetitionsCount = toIntExact(historyExceptionAppear.stream().filter(b -> !b).count()); 233 | if (hasNext()) { 234 | currentIndex++; 235 | return new RepeatedIfExceptionsInvocationContext(currentIndex, totalTestRuns, 236 | successfulTestRepetitionsCount, minSuccess, repeatableExceptionAppeared, formatter); 237 | } 238 | throw new NoSuchElementException(); 239 | } 240 | 241 | @Override 242 | public void remove() { 243 | throw new UnsupportedOperationException(); 244 | } 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/internal/ParameterizedRepeatedMethodContext.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok.internal; 2 | 3 | import org.junit.jupiter.api.extension.ParameterContext; 4 | import org.junit.jupiter.api.extension.ParameterResolutionException; 5 | import org.junit.jupiter.params.aggregator.AggregateWith; 6 | import org.junit.jupiter.params.aggregator.ArgumentsAccessor; 7 | import org.junit.jupiter.params.aggregator.ArgumentsAggregator; 8 | import org.junit.jupiter.params.aggregator.DefaultArgumentsAccessor; 9 | import org.junit.jupiter.params.converter.ArgumentConverter; 10 | import org.junit.jupiter.params.converter.ConvertWith; 11 | import org.junit.jupiter.params.converter.DefaultArgumentConverter; 12 | import org.junit.jupiter.params.support.AnnotationConsumerInitializer; 13 | import org.junit.platform.commons.support.ReflectionSupport; 14 | import org.junit.platform.commons.util.AnnotationUtils; 15 | import org.junit.platform.commons.util.ReflectionUtils; 16 | import org.junit.platform.commons.util.StringUtils; 17 | 18 | import java.lang.reflect.Method; 19 | import java.lang.reflect.Parameter; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import static io.github.artsok.internal.ParameterizedRepeatedMethodContext.ResolverType.AGGREGATOR; 24 | import static io.github.artsok.internal.ParameterizedRepeatedMethodContext.ResolverType.CONVERTER; 25 | import static org.junit.platform.commons.util.AnnotationUtils.isAnnotated; 26 | 27 | /** 28 | * Encapsulates access to the parameters of a parameterized test method and 29 | * caches the converters and aggregators used to resolve them. 30 | * 31 | * @since 5.3 - FULL COPY PAST FROM ORIGINAL JUNIT 5 32 | */ 33 | public class ParameterizedRepeatedMethodContext { 34 | 35 | private final List resolverTypes; 36 | private final ParameterizedRepeatedMethodContext.Resolver[] resolvers; 37 | 38 | public ParameterizedRepeatedMethodContext(Method testMethod) { 39 | Parameter[] parameters = testMethod.getParameters(); 40 | this.resolverTypes = new ArrayList<>(parameters.length); 41 | this.resolvers = new ParameterizedRepeatedMethodContext.Resolver[parameters.length]; 42 | for (Parameter parameter : parameters) { 43 | this.resolverTypes.add(isAggregator(parameter) ? AGGREGATOR : CONVERTER); 44 | } 45 | } 46 | 47 | /** 48 | * Determine if the supplied {@link Parameter} is an aggregator (i.e., of 49 | * type {@link ArgumentsAccessor} or annotated with {@link AggregateWith}). 50 | * 51 | * @return {@code true} if the parameter is an aggregator 52 | */ 53 | private static boolean isAggregator(Parameter parameter) { 54 | return ArgumentsAccessor.class.isAssignableFrom(parameter.getType()) 55 | || isAnnotated(parameter, AggregateWith.class); 56 | } 57 | 58 | /** 59 | * Determine if the {@link Method} represented by this context has a 60 | * potentially valid signature (i.e., formal parameter 61 | * declarations) with regard to aggregators. 62 | * 63 | *

This method takes a best-effort approach at enforcing the following 64 | * policy for parameterized test methods that accept aggregators as arguments. 65 | * 66 | *

    67 | *
  1. zero or more indexed arguments come first.
    2. 68 | *
  2. zero or more aggregators come next.
    3. 69 | *
  3. zero or more arguments supplied by other {@code ParameterResolver} 70 | * implementations come last.
    4. 71 | *
72 | * 73 | * @return {@code true} if the method has a potentially valid signature 74 | */ 75 | public boolean hasPotentiallyValidSignature() { 76 | int indexOfPreviousAggregator = -1; 77 | for (int i = 0; i < getParameterCount(); i++) { 78 | if (isAggregator(i)) { 79 | if ((indexOfPreviousAggregator != -1) && (i != indexOfPreviousAggregator + 1)) { 80 | return false; 81 | } 82 | indexOfPreviousAggregator = i; 83 | } 84 | } 85 | return true; 86 | } 87 | 88 | /** 89 | * Get the number of parameters of the {@link Method} represented by this context. 90 | * @return - number of parameters 91 | */ 92 | public int getParameterCount() { 93 | return resolvers.length; 94 | } 95 | 96 | /** 97 | * Determine if the {@link Method} represented by this context declares at 98 | * least one {@link Parameter} that is an 99 | * {@linkplain #isAggregator aggregator}. 100 | * 101 | * @return {@code true} if the method has an aggregator 102 | */ 103 | public boolean hasAggregator() { 104 | return resolverTypes.contains(AGGREGATOR); 105 | } 106 | 107 | /** 108 | * Determine if the {@link Parameter} with the supplied index is an 109 | * aggregator (i.e., of type {@link ArgumentsAccessor} or annotated with 110 | * {@link AggregateWith}). 111 | * 112 | * @return {@code true} if the parameter is an aggregator 113 | */ 114 | boolean isAggregator(int parameterIndex) { 115 | return resolverTypes.get(parameterIndex) == AGGREGATOR; 116 | } 117 | 118 | /** 119 | * Find the index of the first {@linkplain #isAggregator aggregator} 120 | * {@link Parameter} in the {@link Method} represented by this context. 121 | * 122 | * @return the index of the first aggregator, or {@code -1} if not found 123 | */ 124 | int indexOfFirstAggregator() { 125 | return resolverTypes.indexOf(AGGREGATOR); 126 | } 127 | 128 | /** 129 | * Resolve the parameter for the supplied context using the supplied 130 | * arguments. 131 | */ 132 | Object resolve(ParameterContext parameterContext, Object[] arguments) { 133 | return getResolver(parameterContext).resolve(parameterContext, arguments); 134 | } 135 | 136 | private ParameterizedRepeatedMethodContext.Resolver getResolver(ParameterContext parameterContext) { 137 | int index = parameterContext.getIndex(); 138 | if (resolvers[index] == null) { 139 | resolvers[index] = resolverTypes.get(index).createResolver(parameterContext); 140 | } 141 | return resolvers[index]; 142 | } 143 | 144 | enum ResolverType { 145 | 146 | CONVERTER { 147 | @Override 148 | ParameterizedRepeatedMethodContext.Resolver createResolver(ParameterContext parameterContext) { 149 | try { // @formatter:off 150 | return AnnotationUtils.findAnnotation(parameterContext.getParameter(), ConvertWith.class) 151 | .map(ConvertWith::value) 152 | .map(clazz -> (ArgumentConverter) ReflectionUtils.newInstance(clazz)) 153 | .map(converter -> AnnotationConsumerInitializer.initialize(parameterContext.getParameter(), converter)) 154 | .map(ParameterizedRepeatedMethodContext.Converter::new) 155 | .orElse(ParameterizedRepeatedMethodContext.Converter.DEFAULT); 156 | } // @formatter:on 157 | catch (Exception ex) { 158 | throw parameterResolutionException("Error creating ArgumentConverter", ex, parameterContext); 159 | } 160 | } 161 | }, 162 | 163 | AGGREGATOR { 164 | @Override 165 | ParameterizedRepeatedMethodContext.Resolver createResolver(ParameterContext parameterContext) { 166 | try { // @formatter:off 167 | return AnnotationUtils.findAnnotation(parameterContext.getParameter(), AggregateWith.class) 168 | .map(AggregateWith::value) 169 | .map(clazz -> (ArgumentsAggregator) ReflectionSupport.newInstance(clazz)) 170 | .map(ParameterizedRepeatedMethodContext.Aggregator::new) 171 | .orElse(ParameterizedRepeatedMethodContext.Aggregator.DEFAULT); 172 | } // @formatter:on 173 | catch (Exception ex) { 174 | throw parameterResolutionException("Error creating ArgumentsAggregator", ex, parameterContext); 175 | } 176 | } 177 | }; 178 | 179 | abstract ParameterizedRepeatedMethodContext.Resolver createResolver(ParameterContext parameterContext); 180 | 181 | } 182 | 183 | interface Resolver { 184 | 185 | Object resolve(ParameterContext parameterContext, Object[] arguments); 186 | 187 | } 188 | 189 | static class Converter implements ParameterizedRepeatedMethodContext.Resolver { 190 | 191 | private static final ParameterizedRepeatedMethodContext.Converter DEFAULT = new ParameterizedRepeatedMethodContext.Converter(DefaultArgumentConverter.INSTANCE); 192 | 193 | private final ArgumentConverter argumentConverter; 194 | 195 | Converter(ArgumentConverter argumentConverter) { 196 | this.argumentConverter = argumentConverter; 197 | } 198 | 199 | @Override 200 | public Object resolve(ParameterContext parameterContext, Object[] arguments) { 201 | Object argument = arguments[parameterContext.getIndex()]; 202 | try { 203 | return this.argumentConverter.convert(argument, parameterContext); 204 | } catch (Exception ex) { 205 | throw parameterResolutionException("Error converting parameter", ex, parameterContext); 206 | } 207 | } 208 | 209 | } 210 | 211 | static class Aggregator implements ParameterizedRepeatedMethodContext.Resolver { 212 | 213 | private static final ParameterizedRepeatedMethodContext.Aggregator DEFAULT = new ParameterizedRepeatedMethodContext.Aggregator((accessor, context) -> accessor); 214 | 215 | private final ArgumentsAggregator argumentsAggregator; 216 | 217 | Aggregator(ArgumentsAggregator argumentsAggregator) { 218 | this.argumentsAggregator = argumentsAggregator; 219 | } 220 | 221 | @Override 222 | public Object resolve(ParameterContext parameterContext, Object[] arguments) { 223 | ArgumentsAccessor accessor = new DefaultArgumentsAccessor(arguments); 224 | try { 225 | return this.argumentsAggregator.aggregateArguments(accessor, parameterContext); 226 | } catch (Exception ex) { 227 | throw parameterResolutionException("Error aggregating arguments for parameter", ex, parameterContext); 228 | } 229 | } 230 | 231 | } 232 | 233 | private static ParameterResolutionException parameterResolutionException(String message, Exception cause, 234 | ParameterContext parameterContext) { 235 | String fullMessage = message + " at index " + parameterContext.getIndex(); 236 | if (StringUtils.isNotBlank(cause.getMessage())) { 237 | fullMessage += ": " + cause.getMessage(); 238 | } 239 | return new ParameterResolutionException(fullMessage, cause); 240 | } 241 | 242 | } -------------------------------------------------------------------------------- /src/main/java/io/github/artsok/extension/ParameterizedRepeatedExtension.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok.extension; 2 | 3 | import io.github.artsok.ParameterizedRepeatedIfExceptionsTest; 4 | import io.github.artsok.internal.RepeatedIfException; 5 | import io.github.artsok.internal.ParameterizedRepeatedIfExceptionsTestNameFormatter; 6 | import io.github.artsok.internal.ParameterizedRepeatedMethodContext; 7 | import io.github.artsok.internal.ParameterizedTestInvocationContext; 8 | import org.junit.jupiter.api.extension.*; 9 | import org.junit.jupiter.params.provider.Arguments; 10 | import org.junit.jupiter.params.provider.ArgumentsProvider; 11 | import org.junit.jupiter.params.provider.ArgumentsSource; 12 | import org.junit.jupiter.params.support.AnnotationConsumerInitializer; 13 | import org.junit.platform.commons.JUnitException; 14 | import org.junit.platform.commons.util.ExceptionUtils; 15 | import org.junit.platform.commons.util.Preconditions; 16 | import org.junit.platform.commons.util.ReflectionUtils; 17 | import org.opentest4j.TestAbortedException; 18 | 19 | import java.lang.reflect.Method; 20 | import java.util.*; 21 | import java.util.concurrent.atomic.AtomicLong; 22 | import java.util.stream.Collectors; 23 | import java.util.stream.Stream; 24 | 25 | import static java.lang.Math.toIntExact; 26 | import static java.util.Spliterators.spliteratorUnknownSize; 27 | import static java.util.stream.StreamSupport.stream; 28 | import static org.junit.platform.commons.util.AnnotationUtils.*; 29 | 30 | /** 31 | * Extension for {@link ParameterizedRepeatedIfExceptionsTest} 32 | */ 33 | public class ParameterizedRepeatedExtension implements TestTemplateInvocationContextProvider, 34 | BeforeTestExecutionCallback, AfterTestExecutionCallback, TestExecutionExceptionHandler { 35 | 36 | private int totalRepeats = 0; 37 | private int minSuccess = 1; 38 | private List> repeatableExceptions; 39 | private boolean repeatableExceptionAppeared = false; 40 | private final List historyExceptionAppear = Collections.synchronizedList(new ArrayList<>()); 41 | private static final String METHOD_CONTEXT_KEY = "context"; 42 | private long suspend = 0L; 43 | 44 | @Override 45 | public boolean supportsTestTemplate(ExtensionContext extensionContext) { 46 | if (!extensionContext.getTestMethod().isPresent()) { 47 | return false; 48 | } 49 | 50 | Method testMethod = extensionContext.getTestMethod().get(); 51 | if (!isAnnotated(testMethod, ParameterizedRepeatedIfExceptionsTest.class)) { 52 | return false; 53 | } 54 | 55 | ParameterizedRepeatedMethodContext methodContext = new ParameterizedRepeatedMethodContext(testMethod); 56 | 57 | Preconditions.condition(methodContext.hasPotentiallyValidSignature(), 58 | () -> String.format( 59 | "@ParameterizedRepeatedIfExceptionsTest method [%s] declares formal parameters in an invalid order: " 60 | + "argument aggregators must be declared after any indexed arguments " 61 | + "and before any arguments resolved by another ParameterResolver.", 62 | testMethod.toGenericString())); 63 | 64 | getStore(extensionContext).put(METHOD_CONTEXT_KEY, methodContext); 65 | return true; 66 | } 67 | 68 | @Override 69 | public Stream provideTestTemplateInvocationContexts(ExtensionContext extensionContext) { 70 | Method templateMethod = extensionContext.getRequiredTestMethod(); 71 | String displayName = extensionContext.getDisplayName(); 72 | ParameterizedRepeatedMethodContext methodContext = getStore(extensionContext)// 73 | .get(METHOD_CONTEXT_KEY, ParameterizedRepeatedMethodContext.class); 74 | ParameterizedRepeatedIfExceptionsTestNameFormatter formatter = createNameFormatter(templateMethod, displayName); 75 | 76 | ParameterizedRepeatedIfExceptionsTest annotationParams = extensionContext.getTestMethod() 77 | .flatMap(testMethods -> findAnnotation(testMethods, ParameterizedRepeatedIfExceptionsTest.class)) 78 | .orElseThrow(() -> new RepeatedIfException("The extension should not be executed " 79 | + "unless the test method is annotated with @ParameterizedRepeatedIfExceptionsTest.")); 80 | 81 | totalRepeats = annotationParams.repeats(); 82 | minSuccess = annotationParams.minSuccess(); 83 | suspend = annotationParams.suspend(); 84 | 85 | Preconditions.condition(totalRepeats > 0, "Total repeats must be higher than 0"); 86 | Preconditions.condition(minSuccess >= 1, "Total minimum success must be higher or equals than 1"); 87 | 88 | 89 | List collect = findRepeatableAnnotations(templateMethod, ArgumentsSource.class) 90 | .stream() 91 | .map(ArgumentsSource::value) 92 | .map(this::instantiateArgumentsProvider) 93 | .map(provider -> AnnotationConsumerInitializer.initialize(templateMethod, provider)) 94 | .flatMap(provider -> arguments(provider, extensionContext)) 95 | .map(Arguments::get) 96 | .map(arguments -> consumedArguments(arguments, methodContext)) 97 | .collect(Collectors.toList()); 98 | 99 | Spliterator spliterator = 100 | spliteratorUnknownSize(new TestTemplateIteratorParams(collect, formatter, methodContext), Spliterator.NONNULL); 101 | return stream(spliterator, false); 102 | 103 | } 104 | 105 | @Override 106 | public void beforeTestExecution(ExtensionContext context) { 107 | repeatableExceptions = Stream.of(context.getTestMethod() 108 | .flatMap(testMethods -> findAnnotation(testMethods, ParameterizedRepeatedIfExceptionsTest.class)) 109 | .orElseThrow(() -> new IllegalStateException("The extension should not be executed ")) 110 | .exceptions() 111 | ).collect(Collectors.toList()); 112 | repeatableExceptions.add(TestAbortedException.class); 113 | } 114 | 115 | //Записываем в historyExceptionAppear по конкретным аргументам! 116 | @Override 117 | public void afterTestExecution(ExtensionContext context) { 118 | boolean exceptionAppeared = exceptionAppeared(context); 119 | historyExceptionAppear.add(exceptionAppeared); 120 | } 121 | 122 | private boolean exceptionAppeared(ExtensionContext extensionContext) { 123 | if (extensionContext.getExecutionException().isPresent()) { 124 | Class exception = extensionContext.getExecutionException().get().getClass(); 125 | return repeatableExceptions.stream().anyMatch(ex -> ex.isAssignableFrom(exception)); 126 | } 127 | return false; 128 | } 129 | 130 | @Override 131 | public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable { 132 | if (appearedExceptionDoesNotAllowRepetitions(throwable)) { 133 | throw throwable; 134 | } 135 | repeatableExceptionAppeared = true; 136 | 137 | long currentSuccessCount = historyExceptionAppear.stream().filter(exceptionAppeared -> !exceptionAppeared).count(); 138 | if (currentSuccessCount < minSuccess) { 139 | if (isMinSuccessTargetStillReachable(minSuccess)) { 140 | throw new TestAbortedException("Do not fail completely, but repeat the test", throwable); 141 | } else { 142 | throw throwable; 143 | } 144 | } 145 | } 146 | 147 | /** 148 | * If cannot reach a minimum success target, will return true 149 | * 150 | * @param minSuccessCount - minimum success count 151 | * @return true/false 152 | */ 153 | private boolean isMinSuccessTargetStillReachable(final long minSuccessCount) { 154 | return historyExceptionAppear.stream().filter(bool -> bool).count() <= totalRepeats - minSuccessCount; 155 | } 156 | 157 | private boolean appearedExceptionDoesNotAllowRepetitions(Throwable appearedException) { 158 | return repeatableExceptions.stream().noneMatch(ex -> ex.isAssignableFrom(appearedException.getClass())); 159 | } 160 | 161 | private ParameterizedRepeatedIfExceptionsTestNameFormatter createNameFormatter(Method templateMethod, String displayName) { 162 | ParameterizedRepeatedIfExceptionsTest parameterizedTest = findAnnotation(templateMethod, ParameterizedRepeatedIfExceptionsTest.class).get(); 163 | String pattern = Preconditions.notBlank(parameterizedTest.name().trim(), 164 | () -> String.format( 165 | "Configuration error: @ParameterizedRepeatedIfExceptionsTest on method [%s] must be declared with a non-empty name.", 166 | templateMethod)); 167 | 168 | String repeatedPattern = Preconditions.notBlank(parameterizedTest.repeatedName(), () -> String.format( 169 | "Configuration error: @ParameterizedRepeatedIfExceptionsTest on method [%s] must be declared with a non-empty repeated name.", 170 | templateMethod)); 171 | 172 | return new ParameterizedRepeatedIfExceptionsTestNameFormatter(pattern, displayName, repeatedPattern); 173 | } 174 | 175 | protected static Stream arguments(ArgumentsProvider provider, ExtensionContext context) { 176 | try { 177 | return provider.provideArguments(context); 178 | } catch (Exception e) { 179 | throw ExceptionUtils.throwAsUncheckedException(e); 180 | } 181 | } 182 | 183 | private Object[] consumedArguments(Object[] arguments, ParameterizedRepeatedMethodContext methodContext) { 184 | int parameterCount = methodContext.getParameterCount(); 185 | return methodContext.hasAggregator() ? arguments 186 | : (arguments.length > parameterCount ? Arrays.copyOf(arguments, parameterCount) : arguments); 187 | } 188 | 189 | private ArgumentsProvider instantiateArgumentsProvider(Class clazz) { 190 | try { 191 | return ReflectionUtils.newInstance(clazz); 192 | } catch (Exception ex) { 193 | if (ex instanceof NoSuchMethodException) { 194 | String message = String.format("Failed to find a no-argument constructor for ArgumentsProvider [%s]. " 195 | + "Please ensure that a no-argument constructor exists and " 196 | + "that the class is either a top-level class or a static nested class", 197 | clazz.getName()); 198 | throw new JUnitException(message, ex); 199 | } 200 | throw ex; 201 | } 202 | } 203 | 204 | private ExtensionContext.Store getStore(ExtensionContext context) { 205 | return context.getStore(ExtensionContext.Namespace.create(ParameterizedRepeatedExtension.class, context.getRequiredTestMethod())); 206 | } 207 | 208 | /** 209 | * TestTemplateIteratorParams (Repeat test if it failed) 210 | */ 211 | class TestTemplateIteratorParams implements Iterator { 212 | 213 | private final List params; 214 | private final ParameterizedRepeatedIfExceptionsTestNameFormatter formatter; 215 | private final ParameterizedRepeatedMethodContext methodContext; 216 | private final AtomicLong invocationCount; 217 | private final AtomicLong paramsCount; 218 | private int currentIndex = 0; 219 | 220 | TestTemplateIteratorParams(List arguments, final ParameterizedRepeatedIfExceptionsTestNameFormatter formatter, final ParameterizedRepeatedMethodContext methodContext) { 221 | this.params = arguments; 222 | this.formatter = formatter; 223 | this.methodContext = methodContext; 224 | this.invocationCount = new AtomicLong(params.size() - 1); 225 | this.paramsCount = new AtomicLong(0); 226 | } 227 | 228 | @Override 229 | public boolean hasNext() { 230 | if (!historyExceptionAppear.isEmpty() 231 | && historyExceptionAppear.get(historyExceptionAppear.size() - 1) 232 | && currentIndex < totalRepeats) { 233 | return true; 234 | } 235 | return invocationCount.get() >= paramsCount.get(); 236 | } 237 | 238 | /** 239 | * Return next ParameterizedTestInvocationContext. Managing several situations: 240 | * 1) Exception in Parameterized Test appears 241 | * 2) When the count of tests for one argument (parameter) equal total repeats 242 | * 3) If no exception appears start to create new ParameterizedTestInvocationContext 243 | * 244 | * @return {@link ParameterizedTestInvocationContext} 245 | */ 246 | @Override 247 | public TestTemplateInvocationContext next() { 248 | 249 | if (hasNext()) { 250 | int currentParam = paramsCount.intValue(); 251 | int errorTestRepetitionsCountForOneArgument = toIntExact(historyExceptionAppear.stream().filter(b -> b).count()); 252 | int successfulTestRepetitionsCountForOneArgument = toIntExact(historyExceptionAppear 253 | .stream() 254 | .skip(historyExceptionAppear.size() - minSuccess <= 0 ? 0 : historyExceptionAppear.size() - minSuccess) 255 | .filter(b -> !b) 256 | .count()); 257 | 258 | if (errorTestRepetitionsCountForOneArgument >= 1 && currentIndex < totalRepeats && successfulTestRepetitionsCountForOneArgument != minSuccess) { 259 | 260 | //If exception appeared would wait suspend time 261 | if (historyExceptionAppear.stream().anyMatch(ex -> ex) && suspend != 0L) { 262 | try { 263 | Thread.sleep(suspend); 264 | } catch (InterruptedException e) { 265 | Thread.currentThread().interrupt(); 266 | } 267 | } 268 | 269 | currentIndex++; 270 | repeatableExceptionAppeared = false; 271 | return new ParameterizedTestInvocationContext(currentIndex, totalRepeats, formatter, methodContext, params.get(currentParam - 1)); 272 | } 273 | 274 | if (currentIndex == totalRepeats || !repeatableExceptionAppeared) { 275 | paramsCount.incrementAndGet(); 276 | repeatableExceptionAppeared = false; 277 | historyExceptionAppear.clear(); 278 | } 279 | 280 | currentIndex = 0; 281 | return new ParameterizedTestInvocationContext(0, 0, formatter, methodContext, params.get(currentParam)); 282 | } 283 | throw new NoSuchElementException(); 284 | } 285 | 286 | @Override 287 | public void remove() { 288 | throw new UnsupportedOperationException(); 289 | } 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /src/test/java/io/github/artsok/ReRunnerTest.java: -------------------------------------------------------------------------------- 1 | package io.github.artsok; 2 | 3 | 4 | import org.junit.jupiter.api.Disabled; 5 | import org.junit.jupiter.api.DisplayName; 6 | import org.junit.jupiter.api.Tag; 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.params.provider.Arguments; 9 | import org.junit.jupiter.params.provider.MethodSource; 10 | import org.junit.jupiter.params.provider.ValueSource; 11 | import org.junit.platform.launcher.Launcher; 12 | import org.junit.platform.launcher.LauncherDiscoveryRequest; 13 | import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; 14 | import org.junit.platform.launcher.core.LauncherFactory; 15 | import org.junit.platform.launcher.listeners.SummaryGeneratingListener; 16 | 17 | import java.io.IOException; 18 | import java.lang.annotation.ElementType; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | import java.lang.annotation.Target; 22 | import java.util.Arrays; 23 | import java.util.List; 24 | import java.util.Random; 25 | import java.util.concurrent.ThreadLocalRandom; 26 | import java.util.stream.Stream; 27 | 28 | import static org.hamcrest.CoreMatchers.equalTo; 29 | import static org.junit.Assert.assertThat; 30 | import static org.junit.jupiter.api.Assertions.assertEquals; 31 | import static org.junit.jupiter.api.Assertions.assertTrue; 32 | import static org.junit.jupiter.params.provider.Arguments.arguments; 33 | import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; 34 | 35 | 36 | 37 | /** 38 | * Examples how to use @RepeatedIfExceptionsTest 39 | * 40 | * @author Artem Sokovets 41 | */ 42 | public class ReRunnerTest { 43 | private ThreadLocalRandom random = ThreadLocalRandom.current(); 44 | 45 | @ProgrammaticTest 46 | @RepeatedIfExceptionsTest(repeats = 2) 47 | public void runTest() { 48 | assertTrue(true, "No exception, repeat one time"); 49 | } 50 | 51 | @Test 52 | void runRunTest() throws Exception { 53 | assertTestResults("runTest", true, 1, 0, 0); 54 | } 55 | 56 | /** 57 | * Repeated three times if test failed. 58 | * By default Exception.class will be handled in test 59 | */ 60 | @ProgrammaticTest 61 | @RepeatedIfExceptionsTest(repeats = 3) 62 | public void reRunTest() throws IOException { 63 | throw new IOException("Error in Test"); 64 | } 65 | 66 | @Test 67 | void runReRunTest() throws Exception { 68 | assertTestResults("reRunTest", false, 4, 3, 0); 69 | } 70 | 71 | /** 72 | * Repeated two times if test failed. Set IOException.class that will be handled in test 73 | * 74 | * @throws IOException - error if occurred 75 | */ 76 | @ProgrammaticTest 77 | @RepeatedIfExceptionsTest(repeats = 2, exceptions = IOException.class) 78 | public void reRunTest2() throws IOException { 79 | throw new IOException("Exception in I/O operation"); 80 | } 81 | 82 | @Test 83 | void runReRun2Test() throws Exception { 84 | assertTestResults("reRunTest2", false, 3, 2, 0); 85 | } 86 | 87 | /** 88 | * Repeated ten times if test failed. Set IOException.class that will be handled in test 89 | * Set formatter for test. Like behavior as at {@link org.junit.jupiter.api.RepeatedTest} 90 | * 91 | * @throws IOException - error if occurred 92 | */ 93 | @ProgrammaticTest 94 | @RepeatedIfExceptionsTest(repeats = 10, exceptions = IOException.class, 95 | name = "Rerun failed test. Attempt {currentRepetition} of {totalRepetitions}") 96 | public void reRunTest3() throws IOException { 97 | throw new IOException("Exception in I/O operation"); 98 | } 99 | 100 | @Test 101 | void runReRun3Test() throws Exception { 102 | assertTestResults("reRunTest3", false, 11, 10, 0); 103 | } 104 | 105 | @DisplayName("Name for our test") 106 | @RepeatedIfExceptionsTest(repeats = 105, exceptions = RuntimeException.class, 107 | name = "Rerun failed Test. Repetition {currentRepetition} of {totalRepetitions}") 108 | void reRunTest4() throws IOException { 109 | if (random.nextInt() % 2 == 0) { //Исключение бросается рандомно 110 | throw new RuntimeException("Error in Test"); 111 | } 112 | } 113 | 114 | /** 115 | * Repeated 100 times with minimum success four times, then disabled all remaining repeats. 116 | * See image below how it works. Default exception is Exception.class 117 | */ 118 | @ProgrammaticTest 119 | @DisplayName("Test Case Name") 120 | @RepeatedIfExceptionsTest(repeats = 100, minSuccess = 4) 121 | public void reRunTest5() { 122 | if (random.nextInt() % 2 == 0) { 123 | throw new RuntimeException("Error in Test"); 124 | } 125 | } 126 | 127 | @ProgrammaticTest 128 | @DisplayName("Do not ultimately fail a test if there are still enough repetitions possible.") 129 | @RepeatedIfExceptionsTest(repeats = 2) 130 | public void reRunTest6() { 131 | throw new RuntimeException("Error in Test"); 132 | } 133 | 134 | @Test 135 | void runReRunTest6() throws Exception { 136 | assertTestResults("reRunTest6", false, 3, 2, 0); 137 | } 138 | 139 | @ProgrammaticTest 140 | @DisplayName("Stop repetitions if 'minSuccess' cannot be reached anymore") 141 | @RepeatedIfExceptionsTest(repeats = 10, minSuccess = 4) 142 | public void reRunTest7() { 143 | throw new RuntimeException("Error in Test"); 144 | } 145 | 146 | @Test 147 | void runReRunTest7() throws Exception { 148 | assertTestResults("reRunTest7", false, 8, 7, 3); 149 | } 150 | 151 | @ProgrammaticTest 152 | @DisplayName("Ultimately fail a test as soon as an unrepeatable exception occurs.") 153 | @RepeatedIfExceptionsTest(repeats = 2, exceptions = NumberFormatException.class) 154 | public void reRunTest8() { 155 | throw new RuntimeException("Error in Test"); 156 | } 157 | 158 | @Test 159 | void runReRunTest8() throws Exception { 160 | assertTestResults("reRunTest8", false, 1, 0, 0); 161 | } 162 | 163 | @Disabled 164 | @RepeatedIfExceptionsTest(repeats = 3, exceptions = IOException.class, suspend = 5000L) 165 | void reRunTestWithSuspendOption() throws IOException { 166 | throw new IOException("Exception in I/O operation"); 167 | } 168 | 169 | /** 170 | * By default total repeats = 1 and minimum success = 1. 171 | * If the test failed by this way start to repeat it by one time with one minimum success. 172 | * 173 | * This example without exceptions. 174 | */ 175 | @Disabled 176 | @ParameterizedRepeatedIfExceptionsTest 177 | @ValueSource(ints = {14, 15, 100, -10}) 178 | void successfulParameterizedTest(int argument) { 179 | System.out.println(argument); 180 | } 181 | 182 | /** 183 | * By default total repeats = 1 and minimum success = 1. 184 | * If the test failed by this way start to repeat it by one time with one minimum success. 185 | * This example with display name but without exceptions 186 | */ 187 | @Disabled 188 | @DisplayName("Example of parameterized repeated without exception") 189 | @ParameterizedRepeatedIfExceptionsTest 190 | @ValueSource(ints = {1, 2, 3, 1001}) 191 | void successfulParameterizedTestWithDisplayName(int argument) { 192 | System.out.println(argument); 193 | } 194 | 195 | /** 196 | * By default total repeats = 1 and minimum success = 1. 197 | * If the test failed by this way start to repeat it by one time with one minimum success. 198 | * 199 | * This example with display name but with exception. Exception depends on random number generation. 200 | */ 201 | @Disabled 202 | @DisplayName("Example of parameterized repeated with exception") 203 | @ParameterizedRepeatedIfExceptionsTest 204 | @ValueSource(strings = {"Hi", "Hello", "Bonjour", "Privet"}) 205 | void errorParameterizedTestWithDisplayName(String argument) { 206 | if (random.nextInt() % 2 == 0) { 207 | throw new RuntimeException("Exception " + argument); 208 | } 209 | } 210 | 211 | /** 212 | * By default total repeats = 1 and minimum success = 1. 213 | * If the test failed by this way start to repeat it by one time with one minimum success. 214 | * 215 | * This example with display name, repeated display name, 10 repeats and 2 minimum success with exceptions. 216 | * Exception depends on random number generation. 217 | */ 218 | @Disabled 219 | @ParameterizedRepeatedIfExceptionsTest(name = "Argument was {0}", 220 | repeatedName = " (Repeat {currentRepetition} of {totalRepetitions})", 221 | repeats = 10, exceptions = RuntimeException.class, minSuccess = 2) 222 | @ValueSource(ints = {4, 5, 6, 7}) 223 | void errorParameterizedTestWithDisplayNameAndRepeatedName(int argument) { 224 | if (random.nextInt() % 2 == 0) { 225 | throw new RuntimeException("Exception in Test " + argument); 226 | } 227 | } 228 | 229 | /** 230 | * By default total repeats = 1 and minimum success = 1. 231 | * If the test failed by this way start to repeat it by one time with one minimum success. 232 | * 233 | * This example with display name, implicitly repeated display name, 4 repeats and 2 minimum success with exceptions. 234 | * Exception depends on random number generation. Also use {@link MethodSource} 235 | */ 236 | @Disabled 237 | @DisplayName("Display name of container") 238 | @ParameterizedRepeatedIfExceptionsTest(name = "Year {0} is a leap year.", 239 | repeats = 4, exceptions = RuntimeException.class, minSuccess = 2) 240 | @MethodSource("stringIntAndListProvider") 241 | void errorParameterizedTestWithMultiArgMethodSource(String str, int num, List list) { 242 | assertEquals(5, str.length()); 243 | assertTrue(num >= 1 && num <= 2); 244 | assertEquals(2, list.size()); 245 | if (random.nextInt() % 2 == 0) { 246 | throw new RuntimeException("Exception in Test"); 247 | } 248 | } 249 | 250 | 251 | /** 252 | * Example with suspend option for Parameterized Test 253 | * It matters, when you get some infrastructure problems and you want to run your tests through timeout. 254 | * 255 | * Set break to 5 seconds. If exception appeared for any arguments, repeating extension would runs tests with break. 256 | * If one result failed and other passed, does not matter we would wait 5 seconds throught each arguments of the repeated tests. 257 | * 258 | */ 259 | @Disabled 260 | @DisplayName("Example of parameterized repeated with exception") 261 | @ParameterizedRepeatedIfExceptionsTest(suspend = 5000L, minSuccess = 2, repeats = 3) 262 | @ValueSource(strings = {"Hi", "Hello", "Bonjour", "Privet"}) 263 | void errorParameterizedTestWithSuspendOption(String argument) { 264 | if (random.nextInt() % 2 == 0) { 265 | throw new RuntimeException(argument); 266 | } 267 | } 268 | 269 | /** 270 | * Parameterized Test with the wrong exception. 271 | * Test throws AssertionError.class, but we wait for Exception.class. 272 | * In this case test with argument "1" runs ones without repeats. 273 | * 274 | * If you change exceptions = AssertionError.class, repeats will appear. 275 | * 276 | */ 277 | @Disabled 278 | @ValueSource(ints = {1, 2}) 279 | @ParameterizedRepeatedIfExceptionsTest(repeats = 2, exceptions = Exception.class) 280 | void testParameterizedRepeaterAssertionsFailure(int value) { 281 | assertThat(value, equalTo(2)); 282 | } 283 | 284 | static Stream stringIntAndListProvider() { 285 | return Stream.of( 286 | arguments("apple", 1, Arrays.asList("a", "b")), 287 | arguments("lemon", 2, Arrays.asList("x", "y")) 288 | ); 289 | } 290 | 291 | @ProgrammaticTest 292 | @ParameterizedRepeatedIfExceptionsTest(repeats = 2) 293 | @ValueSource(ints = {1}) 294 | public void reRunTestParameterized1(int number) { 295 | assertThat(number, equalTo(5)); 296 | } 297 | 298 | @Test 299 | void runReRunParameterized1Test() throws Exception { 300 | assertTestResults("reRunTestParameterized1", 0, 1, 3, 2, 0, int.class); 301 | } 302 | 303 | @ProgrammaticTest 304 | @ParameterizedRepeatedIfExceptionsTest(repeats = 2) 305 | @ValueSource(ints = {1, 3, 2}) 306 | public void reRunTestParameterized2(int number) { 307 | assertThat(number, equalTo(3)); 308 | } 309 | 310 | @Test 311 | void runReRunParameterized2Test() throws Exception { 312 | assertTestResults("reRunTestParameterized2", 1, 2, 7, 4, 0, int.class); 313 | } 314 | 315 | @ProgrammaticTest 316 | @ParameterizedRepeatedIfExceptionsTest(repeats = 30) 317 | @ValueSource(ints = {1, 3, 2}) 318 | public void reRunTestParameterized3(int number) { 319 | assertThat(number, equalTo(new Random().nextInt(500) % 3)); 320 | } 321 | 322 | @Test 323 | void runReRunParameterized3Test() throws NoSuchMethodException { 324 | SummaryGeneratingListener testListener = runTest("reRunTestParameterized3", int.class); 325 | 326 | assertEquals(2, testListener.getSummary().getTestsSucceededCount(), "successful test runs"); 327 | assertEquals(1, testListener.getSummary().getTestsFailedCount(), "failed test runs"); 328 | assertGreaterThan(31, testListener.getSummary().getTestsStartedCount(), "started test runs"); 329 | assertGreaterThan(29, testListener.getSummary().getTestsAbortedCount(), "aborted test runs"); 330 | assertEquals(0, testListener.getSummary().getTestsSkippedCount(), "skipped test runs"); 331 | } 332 | 333 | private void assertGreaterThan(int expected, long actual, String message) { 334 | assertTrue(expected < actual, String.format("expected %s to be greater than %s,%nbut was %s", message, expected, actual)); 335 | } 336 | 337 | private void assertTestResults(String methodName, boolean successfulTestRun, int startedTests, int abortedTests, 338 | int skippedTests) throws Exception { 339 | assertTestResults(methodName, 340 | successfulTestRun ? 1 : 0, 341 | !successfulTestRun ? 1 : 0, 342 | startedTests, abortedTests, skippedTests); 343 | } 344 | 345 | private void assertTestResults(String methodName, int successfulTestRuns, int failedTestRuns, int startedTests, int abortedTests, 346 | int skippedTests, Class... parameterTypes) throws Exception { 347 | SummaryGeneratingListener listener = runTest(methodName, parameterTypes); 348 | 349 | assertEquals(successfulTestRuns, listener.getSummary().getTestsSucceededCount(), "successful test runs"); 350 | assertEquals(failedTestRuns, listener.getSummary().getTestsFailedCount(), "failed test runs"); 351 | assertEquals(startedTests, listener.getSummary().getTestsStartedCount(), "started test runs"); 352 | assertEquals(abortedTests, listener.getSummary().getTestsAbortedCount(), "aborted test runs"); 353 | assertEquals(skippedTests, listener.getSummary().getTestsSkippedCount(), "skipped test runs"); 354 | } 355 | 356 | private SummaryGeneratingListener runTest(String methodName, Class... parameterTypes) throws NoSuchMethodException { 357 | SummaryGeneratingListener listener = new SummaryGeneratingListener(); 358 | LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() 359 | .selectors(selectMethod(getClass(), getClass().getMethod(methodName, parameterTypes))) 360 | .build(); 361 | Launcher launcher = LauncherFactory.create(); 362 | launcher.registerTestExecutionListeners(listener); 363 | launcher.execute(request); 364 | return listener; 365 | } 366 | 367 | @Tag("programmatic-tests") 368 | @Target(ElementType.METHOD) 369 | @Retention(RetentionPolicy.RUNTIME) 370 | private @interface ProgrammaticTest { 371 | } 372 | } 373 | --------------------------------------------------------------------------------