├── .gitignore ├── README.md ├── pom.xml ├── testrail-connector ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── nullin │ │ └── testrail │ │ ├── ResultStatus.java │ │ ├── TestRailArgs.java │ │ ├── TestRailListener.java │ │ ├── TestRailReporter.java │ │ ├── annotations │ │ └── TestRailCase.java │ │ ├── client │ │ ├── APIClient.java │ │ ├── ClientException.java │ │ └── TestRailClient.java │ │ ├── dto │ │ ├── Case.java │ │ ├── Milestone.java │ │ ├── Plan.java │ │ ├── PlanEntry.java │ │ ├── Result.java │ │ ├── Run.java │ │ ├── Section.java │ │ ├── Suite.java │ │ └── Test.java │ │ └── internal │ │ ├── GenerateTestCases.java │ │ └── TestTheListener.java │ └── test │ ├── java │ └── com │ │ └── nullin │ │ └── testrail │ │ └── sampleproj │ │ ├── TestClassA.java │ │ ├── TestClassB.java │ │ └── pkg │ │ └── TestClassC.java │ └── resources │ └── testng.xml └── testrail-utils ├── pom.xml └── src └── main └── java └── com └── nullin └── testrail ├── tools ├── InvalidJiraReferenceFinder.java └── UnstableTestsFinder.java └── util ├── SoftReportingAssertion.java └── TestRailScreenshotListener.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | pom.xml.tag 3 | pom.xml.releaseBackup 4 | pom.xml.next 5 | release.properties 6 | .idea/ 7 | deploy.py 8 | *.iml 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | testrail-integration 2 | ==================== 3 | 4 | A library for connecting to TestRail and reporting results for automated tests executed using TestNG. 5 | 6 | Requirements/Assumptions 7 | ------------------------ 8 | 9 | ### TestRail Configuration 10 | 11 | * Project should be setup as a single repository with baseline support. 12 | * A custom string field named `automation_id` should be added to the `Case` fields for the project. 13 | * ~~All suite names should be unique~~ 14 | * Automation IDs need to be unique within a TestRail test suite 15 | 16 | ### Test Code 17 | 18 | * Annotations need to be added to test cases in Java code. See `TestRailCase` annotation 19 | * Automation IDs need to be added as first parameter of the data-driven tests 20 | 21 | ### Other Assumptions 22 | 23 | * TestRail Test Plan needs to be pre-created and associated with the required suite. 24 | * TestRail Test Plan can contain multiple plan entries (runs), but each of that run is associated with the same Suite. e.g. 25 | this allows us to separate API and UI tests into two different entries with in the same Test Plan. 26 | * If the same automation id appears twice for the same, result will only be reported once. This is an invalid configuration 27 | and should be fixed quickly. 28 | 29 | How to use `automation_id` 30 | --------------------- 31 | 32 | The custom string field `automation_id` in TestRail contains unique strings that are referenced by test code to report 33 | results. This unique string is associated to tests in multiple ways: 34 | 35 | ### Sing Test Method 36 | 37 | Simply add `TestRailCase` annotation to the test method and the listener will take care of reporting the result 38 | 39 | ```java 40 | 41 | @TestRailCase("testC1") 42 | @Test 43 | public void test1() { 44 | Assert.assertEquals(1, 1); 45 | } 46 | ``` 47 | 48 | In above example, a `PASS` will be reported for test with automation id `testC1`. 49 | 50 | ### Data-driven Test 51 | 52 | No annotation needs to be added in this case. We assume that the first parameter passed into the data driven test is 53 | the unique automation id for the data-driven tests. 54 | 55 | ```java 56 | 57 | @DataProvider(name = "DP") 58 | public Object[][] getData() { 59 | return new Object[][] { 60 | {"testA2", 10, 10}, 61 | {"testA3", 10, 5} 62 | }; 63 | } 64 | 65 | @Test(dataProvider = "DP") 66 | public void test2(String testId, int x, int y) { 67 | Assert.assertEquals(x, y); 68 | } 69 | ``` 70 | 71 | In above example, a `PASS` will be reported for test case with automation id `testA2` and a `FAILURE` for `testA3`. 72 | 73 | ### Test Method for Multiple TestRail Test Cases 74 | 75 | By adding `TestRailCase` annotation with `selfReporting` set to true, we tell the `TestRailReporter` to skip this 76 | test when processing it as it's responsible for it's own reporting of results. 77 | 78 | ```java 79 | 80 | @TestRailCase(selfReporting = true) 81 | @Test 82 | public void test5() { 83 | 84 | //do something 85 | 86 | Map result = new HashMap(); 87 | result.put(TestRailReporter.KEY_STATUS, ResultStatus.PASS); 88 | TestRailReporter.getInstance().reportResult("testC3", result); 89 | 90 | //do something 91 | 92 | result.put(TestRailReporter.KEY_STATUS, ResultStatus.FAIL); 93 | result.put(TestRailReporter.KEY_THROWABLE, new IOException("Something very bad happened!!")); 94 | TestRailReporter.getInstance().reportResult("testC4", result); 95 | } 96 | ``` 97 | 98 | In above example, a `PASS` will be reported for test case with automation id `testC3` and a `FAILURE` for `testC4`. This 99 | is a simplified example; in actual code, you might want to extend TestNG's `SoftAssert` and delegate the reporting to 100 | that class. 101 | 102 | Listeners and Reporters 103 | ----------------------- 104 | 105 | `com.nullin.testrail.TestRailListener` implements `ITestListener` and `IConfigurationListener` interfaces from TestNG, 106 | and is responsible for reporting results to TestRail. `TestRailListener` internally uses `com.nullin.testrail.TestRailReporter` 107 | to report results to TestRail. Test code can also directly use methods in `TestRailReporter` as shown in example above. 108 | 109 | ### Listener Configuration 110 | 111 | Following system properties need to be configured during startup to configure/enable TestRail reporter 112 | 113 | * `testRail.enabled` : boolean (true|false) value to enable/disable reporting to TestRail. 114 | * `testRail.url` : Base URL to TestRail. 115 | * `testRail.username` : User to use to authenticate to TestRail instance. 116 | * `testRail.password` : Password to authenticate to TestRail instance. 117 | * `testRail.testPlanId` : ID of a pre-created Test Plan in TestRail. This is an integer that you can get via APIs. Via, 118 | TestRail UI, this is the integer part of an ID that is shown next to a Test Plan. E.g. for `R3285` id displayed in UI, it will 119 | be `3285`. 120 | 121 | ### Listener Startup 122 | 123 | During startup, we try and connect to TestRail and get the Test Plan using the user specified id. If any of this fails, 124 | we disable the `TestRailReporter`, log this condition, but do not fail the test execution. 125 | 126 | Also, during startup, we create a cache of all Test Cases and associated `automation id`s. During this process, we log 127 | all instances when we encounter the same `automation id` multiple times. 128 | 129 | ### Listener Logging 130 | 131 | We use `java.util.logging`. 132 | 133 | 134 | Extending `TestRailListener` 135 | ---------------------------- 136 | 137 | We have two methods that can be overridden by extending `TestRailListener` 138 | 139 | 1. `public String getScreenshotUrl(ITestResult result)` 140 | Extend this method to return a URL for the captured screen shot for UI tests. If specified, we inline the image into TestRail result. 141 | 142 | 2. `public Map getMoreInformation(ITestResult result)` 143 | Extend this method to provide more information about the test result or the test run in general. We have used this 144 | method to specify information about environment variables, target environment etc. 145 | 146 | __NOTE__: the test class/method/parameter information is automatically logged and doesn't need to be returned by this method. 147 | 148 | Workflow 149 | -------- 150 | 151 | ``` 152 | 153 | - Connect and get a list of all test suites in the project and cache in a map 154 | - Also, get the list of all tests cases within the suites and create a map of the automation IDs to case IDs 155 | - If same automation id is associated with multiple case ids in same suite, raise error and quit 156 | - Get the test plan based on the specified plan name or id 157 | - For each result to be reported, get the method and it's enclosing class (multiple runs for same suite, 158 | not supported yet. Need to handle configurationIds for that to work) 159 | - get the annotations and figure out the automation id 160 | - if the test was DD, the first parameter should be automation id 161 | - from automation id, get the case id 162 | - now we have case id and test run id 163 | - report the result 164 | - Finish 165 | 166 | ``` 167 | 168 | Future work 169 | -------------------- 170 | 171 | - Support creating test runs with configurations 172 | - Output tests that were not reported into file w/ reason for failure 173 | - Create a new test plan (if an id isn't already specified) and associated the specified test suites 174 | - When done with all results, close the test plan (by default or based on user specified parameter) 175 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.nullin 6 | testrail-integration 7 | 2.3.5-SNAPHSOT 8 | pom 9 | 10 | TestRail Integration 11 | https://github.com/nullin/testrail-integration 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | testrail-connector 19 | testrail-utils 20 | 21 | 22 | 23 | 24 | 25 | com.google.guava 26 | guava 27 | 18.0 28 | 29 | 30 | org.testng 31 | testng 32 | 6.8.8 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | org.apache.maven.plugins 41 | maven-surefire-plugin 42 | 2.9 43 | 44 | true 45 | 46 | 47 | 48 | org.apache.maven.plugins 49 | maven-compiler-plugin 50 | 3.3 51 | 52 | -Xlint:all 53 | -Xlint:-serial 54 | true 55 | lines,vars,source 56 | true 57 | true 58 | true 59 | 1.7 60 | 1.7 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | my-repo 69 | https://github.com/nullin/mvn-repo 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /testrail-connector/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.nullin 7 | testrail-integration 8 | 2.3.5-SNAPHSOT 9 | 10 | 11 | testrail-connector 12 | jar 13 | 14 | TestRail Connector 15 | https://github.com/nullin/testrail-integration/testrail-connector 16 | 17 | 18 | 19 | com.google.guava 20 | guava 21 | 22 | 23 | org.testng 24 | testng 25 | 26 | 27 | org.apache.httpcomponents 28 | httpclient 29 | 4.3.2 30 | 31 | 32 | com.fasterxml.jackson.core 33 | jackson-core 34 | 2.4.1.1 35 | 36 | 37 | com.fasterxml.jackson.dataformat 38 | jackson-dataformat-xml 39 | 2.4.1 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/ResultStatus.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail; 2 | 3 | /** 4 | * Result status that corresponds to TestRail 5 | * 6 | * @author nullin 7 | */ 8 | public enum ResultStatus { 9 | PASS, 10 | FAIL, 11 | SKIP, //shows up as blocked in TestRail 12 | UNTESTED 13 | } 14 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/TestRailArgs.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * Arguments for {@link com.nullin.testrail.TestRailListener} 8 | * 9 | * @author nullin 10 | */ 11 | public class TestRailArgs { 12 | 13 | //if the listener is enabled or not 14 | private Boolean enabled; 15 | //test plan id (if one already exists) 16 | private Integer testPlanId; 17 | //suite names 18 | private List suiteNames; 19 | //url to the TestRail instance 20 | private String url; 21 | //username to login to TestRail 22 | private String username; 23 | //password to login to TestRail 24 | private String password; 25 | 26 | private TestRailArgs() {} 27 | 28 | public static TestRailArgs getNewTestRailListenerArgs() { 29 | TestRailArgs args = new TestRailArgs(); 30 | args.enabled = Boolean.valueOf(System.getProperty("testRail.enabled")); 31 | 32 | if (args.enabled == null || !args.enabled) { 33 | return args; //no need to process further. TestRail reporting is not enabled 34 | } 35 | 36 | String planId = System.getProperty("testRail.testPlanId"); 37 | if (planId == null) { 38 | throw new IllegalArgumentException("TestRail Test Plan ID not specified"); 39 | } else { 40 | try { 41 | args.testPlanId = Integer.valueOf(planId); 42 | } catch(NumberFormatException ex) { 43 | throw new IllegalArgumentException("Plan Id is not an integer as expected"); 44 | } 45 | } 46 | 47 | String suiteNamesStr = System.getProperty("testRail.suiteNames"); 48 | if (suiteNamesStr != null) { 49 | try { 50 | String[] suiteNamesArr = suiteNamesStr.split(","); 51 | args.suiteNames = new ArrayList(); 52 | for (String suiteName : suiteNamesArr) { 53 | if (suiteName != null && !suiteName.trim().isEmpty()) { 54 | args.suiteNames.add(suiteName.trim()); 55 | } 56 | } 57 | 58 | } catch(NumberFormatException ex) { 59 | throw new IllegalArgumentException("Plan Id is not an integer as expected"); 60 | } 61 | } 62 | 63 | if ((args.url = System.getProperty("testRail.url")) == null) { 64 | throw new IllegalArgumentException("TestRail URL not specified (testRail.url)"); 65 | } 66 | 67 | if ((args.username = System.getProperty("testRail.username")) == null) { 68 | throw new IllegalArgumentException("TestRail user not specified (testRail.username)"); 69 | } 70 | 71 | if ((args.password = System.getProperty("testRail.password")) == null) { 72 | throw new IllegalArgumentException("TestRail password not specified (testRail.password)"); 73 | } 74 | 75 | return args; 76 | } 77 | 78 | public Boolean getEnabled() { 79 | return enabled; 80 | } 81 | 82 | public Integer getTestPlanId() { 83 | return testPlanId; 84 | } 85 | 86 | public List getSuiteNames() { 87 | return suiteNames; 88 | } 89 | 90 | public String getUrl() { 91 | return url; 92 | } 93 | 94 | public String getUsername() { 95 | return username; 96 | } 97 | 98 | public String getPassword() { 99 | return password; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/TestRailListener.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail; 2 | 3 | import java.lang.reflect.Method; 4 | import java.util.Arrays; 5 | import java.util.Collections; 6 | import java.util.HashMap; 7 | import java.util.LinkedHashMap; 8 | import java.util.Map; 9 | import java.util.logging.Logger; 10 | 11 | import com.nullin.testrail.annotations.TestRailCase; 12 | import org.testng.IConfigurationListener; 13 | import org.testng.ITestContext; 14 | import org.testng.ITestListener; 15 | import org.testng.ITestResult; 16 | import org.testng.annotations.Test; 17 | 18 | /** 19 | * A TestNG listener to report results to TestRail instance 20 | * 21 | * @author nullin 22 | */ 23 | public class TestRailListener implements ITestListener, IConfigurationListener { 24 | 25 | private Logger logger = Logger.getLogger(TestRailListener.class.getName()); 26 | 27 | private TestRailReporter reporter; 28 | private boolean enabled; 29 | 30 | /** 31 | * Store the result associated with a failed configuration here. This can 32 | * then be used when reporting the result of a skipped test to provide 33 | * additional information in TestRail 34 | */ 35 | private ThreadLocal testSkipResult = new ThreadLocal(); 36 | 37 | public TestRailListener() { 38 | try { 39 | reporter = TestRailReporter.getInstance(); 40 | enabled = reporter.isEnabled(); 41 | } catch (Throwable ex) { 42 | logger.severe("Ran into exception initializing reporter: " + ex.getMessage()); 43 | ex.printStackTrace(); 44 | } 45 | } 46 | 47 | /** 48 | * Reports the result for the test method to TestRail 49 | * @param result TestNG test result 50 | */ 51 | private void reportResult(ITestResult result) { 52 | if (!enabled) { 53 | return; //do nothing 54 | } 55 | 56 | try { 57 | Method method = result.getMethod().getConstructorOrMethod().getMethod(); 58 | String className = result.getTestClass().getName(); 59 | String methodName = result.getMethod().getMethodName(); 60 | String id = className + "#" + methodName; 61 | Object[] params = result.getParameters(); 62 | String firstParam = null; 63 | if (params != null && params.length > 0) { 64 | id += "(" + params[0] + ")"; 65 | firstParam = String.valueOf(params[0]); 66 | } 67 | int status = result.getStatus(); 68 | Throwable throwable = result.getThrowable(); 69 | 70 | TestRailCase trCase = method.getAnnotation(TestRailCase.class); 71 | Test test = method.getAnnotation(Test.class); 72 | String automationId; 73 | if (trCase == null) { 74 | if (null != test.dataProvider() && !test.dataProvider().isEmpty()) { 75 | if (firstParam == null) { 76 | logger.severe("Didn't find the first parameter for DD test " + id + ". Result not reported."); 77 | return; //nothing more to do 78 | } 79 | automationId = firstParam; 80 | } else { 81 | logger.severe(String.format("Test case %s is not annotated with TestRailCase annotation. " + 82 | "Result not reported", id)); 83 | return; //nothing more to do 84 | } 85 | } else { 86 | automationId = trCase.value(); 87 | } 88 | 89 | if (automationId == null || automationId.isEmpty()) { 90 | //case id not specified on method, check if this is a DD method 91 | if (!trCase.selfReporting()) { 92 | //self reporting test cases are responsible of reporting results on their own 93 | logger.warning("Didn't find automation id nor is the test self reporting for test " + id + 94 | ". Please check test configuration."); 95 | return; //nothing more to do 96 | } else { 97 | return; //nothing to do as the test is marked as self reporting 98 | } 99 | } 100 | 101 | Map props = new HashMap(); 102 | long elapsed = (result.getEndMillis() - result.getStartMillis()) / 1000; 103 | elapsed = elapsed == 0 ? 1 : elapsed; //we can only track 1 second as the smallest unit 104 | props.put("elapsed", elapsed + "s"); 105 | props.put("status", getStatus(status)); 106 | props.put("throwable", throwable); 107 | //override if needed 108 | if (status == ITestResult.SKIP) { 109 | ITestResult skipResult = testSkipResult.get(); 110 | if (skipResult != null) { 111 | props.put("throwable", skipResult.getThrowable()); 112 | } 113 | } 114 | props.put("screenshotUrl", getScreenshotUrl(result)); 115 | Map moreInfo = new LinkedHashMap(); 116 | moreInfo.put("class", result.getMethod().getRealClass().getCanonicalName()); 117 | moreInfo.put("method", result.getMethod().getMethodName()); 118 | if (result.getParameters() != null) { 119 | moreInfo.put("parameters", Arrays.toString(result.getParameters())); 120 | } 121 | moreInfo.putAll(getMoreInformation(result)); 122 | props.put("moreInfo", moreInfo); 123 | reporter.reportResult(automationId, props); 124 | } catch(Exception ex) { 125 | //only log and do nothing else 126 | logger.severe("Ran into exception " + ex.getMessage()); 127 | } 128 | } 129 | 130 | public void onTestStart(ITestResult result) { 131 | //not reporting a started status 132 | } 133 | 134 | public void onTestSuccess(ITestResult result) { 135 | reportResult(result); 136 | } 137 | 138 | public void onTestFailure(ITestResult result) { 139 | reportResult(result); 140 | } 141 | 142 | public void onTestSkipped(ITestResult result) { 143 | if (result.getThrowable() != null) { 144 | //test failed, but is reported as skipped because of RetryAnalyzer. 145 | //so, changing result status and reporting this as failure instead. 146 | result.setStatus(ITestResult.FAILURE); 147 | } 148 | reportResult(result); 149 | } 150 | 151 | public void onTestFailedButWithinSuccessPercentage(ITestResult result) { 152 | //nothing here 153 | } 154 | 155 | public void onStart(ITestContext context) { 156 | //nothing here 157 | } 158 | 159 | public void onFinish(ITestContext context) { 160 | //nothing here 161 | } 162 | 163 | /** 164 | * TestRail currently doesn't support uploading screenshots via APIs. Suggested method is 165 | * to upload screenshots to another server and provide a URL in the test comments. 166 | * 167 | * This method should be overridden in a sub-class to provide the URL for the screenshot. 168 | * 169 | * @param result result of test execution 170 | * @return the URL to where the screenshot can be accessed 171 | */ 172 | public String getScreenshotUrl(ITestResult result) { 173 | return null; //should be extended & overridden if needed 174 | } 175 | 176 | /** 177 | * In case, we want to log more information about the test execution, this method can be used. 178 | * 179 | * NOTE: the test class/method/parameter information is automatically logged. 180 | * 181 | * This method should be overridden in a sub-class to provide map containing information 182 | * that should be displayed for each test result in TestRail 183 | */ 184 | public Map getMoreInformation(ITestResult result) { 185 | return Collections.emptyMap(); //should be extended & overridden if needed 186 | } 187 | 188 | /** 189 | * @param status TestNG specific status code 190 | * @return TestRail specific status IDs 191 | */ 192 | private ResultStatus getStatus(int status) { 193 | switch (status) { 194 | case ITestResult.SUCCESS: 195 | return ResultStatus.PASS; 196 | case ITestResult.FAILURE: 197 | return ResultStatus.FAIL; 198 | case ITestResult.SUCCESS_PERCENTAGE_FAILURE: 199 | return ResultStatus.FAIL; 200 | case ITestResult.SKIP: 201 | return ResultStatus.SKIP; 202 | default: 203 | return ResultStatus.UNTESTED; 204 | } 205 | } 206 | 207 | public void onConfigurationSuccess(ITestResult iTestResult) { 208 | testSkipResult.remove(); 209 | } 210 | 211 | public void onConfigurationFailure(ITestResult iTestResult) { 212 | testSkipResult.set(iTestResult); 213 | } 214 | 215 | public void onConfigurationSkip(ITestResult iTestResult) { 216 | //nothing here 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/TestRailReporter.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.IOException; 5 | import java.io.PrintStream; 6 | import java.io.UnsupportedEncodingException; 7 | import java.nio.charset.Charset; 8 | import java.nio.charset.StandardCharsets; 9 | import java.util.HashMap; 10 | import java.util.HashSet; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.Set; 14 | import java.util.logging.Logger; 15 | 16 | import com.nullin.testrail.client.ClientException; 17 | import com.nullin.testrail.client.TestRailClient; 18 | import com.nullin.testrail.dto.Case; 19 | import com.nullin.testrail.dto.Plan; 20 | import com.nullin.testrail.dto.PlanEntry; 21 | import com.nullin.testrail.dto.Run; 22 | import com.nullin.testrail.dto.Test; 23 | 24 | /** 25 | * This class is responsible with communicating with TestRail and reporting results to it. 26 | * 27 | * It's invoked by {@link com.nullin.testrail.TestRailListener} and has methods that can be 28 | * directly invoked from the test code as well. 29 | * 30 | * @author nullin 31 | */ 32 | public class TestRailReporter { 33 | 34 | private Logger logger = Logger.getLogger(TestRailReporter.class.getName()); 35 | private TestRailClient client; 36 | private Map caseIdLookupMap; 37 | private Map testToRunIdMap; 38 | private Boolean enabled; 39 | private String config; 40 | 41 | //keys for the properties map that is used to pass test information into this reporter 42 | public static final String KEY_MORE_INFO = "moreInfo"; 43 | public static final String KEY_SCREENSHOT_URL = "screenshotUrl"; 44 | public static final String KEY_STATUS = "status"; 45 | public static final String KEY_ELAPSED = "elapsed"; 46 | public static final String KEY_THROWABLE = "throwable"; 47 | 48 | private static class Holder { 49 | private static final TestRailReporter INSTANCE = new TestRailReporter(); 50 | } 51 | 52 | public static TestRailReporter getInstance() { 53 | return Holder.INSTANCE; 54 | } 55 | 56 | /** 57 | * Initializes the required state including any required caches. 58 | * 59 | * This method is allowed to throw exception so that we can stop test execution 60 | * to fix invalid configuration before proceeding further 61 | * 62 | * @throws java.io.IOException 63 | * @throws com.nullin.testrail.client.ClientException 64 | */ 65 | private TestRailReporter() { 66 | TestRailArgs args = TestRailArgs.getNewTestRailListenerArgs(); 67 | enabled = args.getEnabled(); 68 | enabled = enabled == null ? false : enabled; 69 | 70 | if (!enabled) { 71 | logger.info("TestRail listener is not enabled. Results will not be reported to TestRail."); 72 | return; 73 | } 74 | 75 | logger.info("TestRail listener is enabled. Configuring..."); 76 | try { 77 | client = new TestRailClient(args.getUrl(), args.getUsername(), args.getPassword()); 78 | 79 | //prepare the test plan and stuff 80 | Plan plan = client.getPlan(args.getTestPlanId()); 81 | 82 | /* 83 | We will make an assumption that the plan can contains multiple entries, but all the 84 | entries would be associated with the same suite. This helps simplify the automated 85 | reporting of results. 86 | 87 | Another assumption is that a test with a given automation id will not re-appear twice 88 | for the same configuration set. Multiple instances of the same configuration set 89 | is possible. If same automation id and configuration set combination is repeated, the 90 | result would only be reported once. 91 | */ 92 | Set suiteIdSet = new HashSet(); 93 | List planEntries = plan.entries; 94 | 95 | int projectId = 0; 96 | int suiteId = 0; 97 | testToRunIdMap = new HashMap(); 98 | for (PlanEntry entry : planEntries) { 99 | suiteIdSet.add(suiteId = entry.suiteId); 100 | for (Run run : entry.runs) { 101 | projectId = run.projectId; 102 | List tests = client.getTests(run.id); 103 | for (Test test : tests) { 104 | testToRunIdMap.put(test.automationId + run.config, run.id); 105 | } 106 | } 107 | } 108 | 109 | caseIdLookupMap = cacheCaseIdLookupMap(client, projectId, suiteId); 110 | 111 | //check some constraints 112 | if (suiteIdSet.size() != 1) { 113 | throw new IllegalStateException("Referenced plan " + plan.id + " has multiple test suites (" + 114 | suiteIdSet + "). This configuration is currently not supported."); 115 | } 116 | 117 | /* 118 | This should be specified when starting the JVM for test execution. It should match exactly at least 119 | one of the configurations used in the test runs. This, along with the automation id of the test 120 | are used to identify the run id. 121 | */ 122 | config = System.getProperty("testRail.runConfig"); 123 | } catch(Exception ex) { 124 | //wrap in a Runtime and throw again 125 | //why? because we don't want to handle it and we want 126 | //to stop the execution asap. 127 | throw new RuntimeException(ex); 128 | } 129 | } 130 | 131 | /** 132 | * Gets all the test cases associated with the test run and caches a map of the 133 | * associated automation id's to the case ids 134 | * 135 | * @param projectId 136 | * @param suiteId 137 | * @return Map with keys as automation ids and corresponding values as the case ids. 138 | */ 139 | private Map cacheCaseIdLookupMap(TestRailClient client, int projectId, int suiteId) 140 | throws IOException, ClientException { 141 | List cases = client.getCases(projectId, suiteId, 0, null); 142 | Map lookupMap = new HashMap(); 143 | for (Case c : cases) { 144 | if (c.automationId == null || c.automationId.isEmpty()) { 145 | continue; //ignore empty automation IDs 146 | } 147 | 148 | if (lookupMap.get(c.automationId) != null) { 149 | logger.severe("Found multiple tests cases with same automation id. " + 150 | "Case Ids " + lookupMap.get(c.automationId) + " & " + c.id); 151 | } else { 152 | lookupMap.put(c.automationId, c.id); 153 | } 154 | } 155 | return lookupMap; 156 | } 157 | 158 | /** 159 | * Reports results to testrail 160 | * @param automationId test automation id 161 | * @param properties test properties. Following values are supported: 162 | *
    163 | *
  • {@link #KEY_ELAPSED}: elapsed time as string
    • 164 | *
  • {@link #KEY_MORE_INFO}: more test information as Map
    • 165 | *
  • {@link #KEY_SCREENSHOT_URL}: screen shot url as string
    • 166 | *
  • {@link #KEY_STATUS}: {@link ResultStatus} of the test
    • 167 | *
  • {@link #KEY_THROWABLE}: any associated {@link Throwable}
    • 168 | *
169 | */ 170 | public void reportResult(String automationId, Map properties) { 171 | if (!enabled) { 172 | return; //do nothing 173 | } 174 | 175 | ResultStatus resultStatus = (ResultStatus)properties.get(KEY_STATUS); 176 | Throwable throwable = (Throwable)properties.get(KEY_THROWABLE); 177 | String elapsed = (String)properties.get(KEY_ELAPSED); 178 | String screenshotUrl = (String)properties.get(KEY_SCREENSHOT_URL); 179 | Map moreInfo = (Map)properties.get(KEY_MORE_INFO); 180 | 181 | try { 182 | Integer caseId = caseIdLookupMap.get(automationId); 183 | if (caseId == null) { 184 | logger.severe("Didn't find case id for test with automation id " + automationId); 185 | return; //nothing more to do 186 | } 187 | 188 | StringBuilder comment = new StringBuilder("More Info (if any):\n"); 189 | if (moreInfo != null && !moreInfo.isEmpty()) { 190 | for (Map.Entry entry: moreInfo.entrySet()) { 191 | comment.append("- ").append(entry.getKey()).append(" : ") 192 | .append('`').append(entry.getValue()).append("`\n"); 193 | } 194 | } else { 195 | comment.append("- `none`\n"); 196 | } 197 | comment.append("\n"); 198 | if (screenshotUrl != null && !screenshotUrl.isEmpty()) { 199 | comment.append("![](").append(screenshotUrl).append(")\n\n"); 200 | } 201 | if (resultStatus.equals(ResultStatus.SKIP)) { 202 | comment.append("Test skipped because of configuration method failure. " + 203 | "Related config error (if captured): \n\n"); 204 | comment.append(getStackTraceAsString(throwable)); 205 | } 206 | if (resultStatus.equals(ResultStatus.FAIL)) { 207 | comment.append("Test failed with following exception (if captured): \n\n"); 208 | comment.append(getStackTraceAsString(throwable)); 209 | } 210 | 211 | //add the result 212 | Map body = new HashMap(); 213 | body.put("status_id", getStatus(resultStatus)); 214 | body.put("comment", new String(comment.toString().getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); 215 | body.put("elapsed", elapsed); 216 | 217 | Integer runId = testToRunIdMap.get(automationId + config); 218 | if (runId == null) { 219 | throw new IllegalArgumentException("Unable to find run id for test with automation id " 220 | + automationId + " and configuration set as " + config); 221 | } 222 | client.addResultForCase(runId, caseId, body); 223 | } catch(Exception ex) { 224 | //only log and do nothing else 225 | logger.severe("Ran into exception " + ex.getMessage()); 226 | } 227 | } 228 | 229 | private String getStackTraceAsString(Throwable throwable) throws UnsupportedEncodingException { 230 | if (throwable == null) { 231 | return ""; 232 | } 233 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 234 | throwable.printStackTrace(new PrintStream(os)); 235 | String str = new String(os.toByteArray(), "UTF-8"); 236 | str = " " + str.replace("\n", "\n ").replace("\t", " "); //better printing 237 | return str; 238 | } 239 | 240 | /** 241 | * @param status TestNG specific status code 242 | * @return TestRail specific status IDs 243 | */ 244 | private int getStatus(ResultStatus status) { 245 | switch (status) { 246 | case PASS: 247 | return 1; //Passed 248 | case FAIL: 249 | return 5; //Failed 250 | case SKIP: 251 | return 2; //Blocked 252 | default: 253 | return 3; //Untested 254 | } 255 | } 256 | 257 | public boolean isEnabled() { 258 | return enabled; 259 | } 260 | 261 | } 262 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/annotations/TestRailCase.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.annotations; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | /** 7 | * Represents one or more cases in TestRail 8 | * 9 | * @author nullin 10 | */ 11 | @Retention(RetentionPolicy.RUNTIME) 12 | public @interface TestRailCase { 13 | 14 | //automation id in TestRail (if any). 15 | //NOTE: "automation_id" custom field needs to be added in TestRail 16 | String value() default ""; 17 | //if true, any value for automation id is ignored 18 | //lets the listener know that it should not raise a warning for no automation id 19 | boolean selfReporting() default false; 20 | 21 | } 22 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/client/APIClient.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.client; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.IOException; 5 | import java.io.UnsupportedEncodingException; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.logging.Logger; 9 | 10 | import org.apache.http.Header; 11 | import org.apache.http.HttpEntity; 12 | import org.apache.http.HttpResponse; 13 | import org.apache.http.client.HttpClient; 14 | import org.apache.http.client.methods.HttpGet; 15 | import org.apache.http.client.methods.HttpPost; 16 | import org.apache.http.entity.StringEntity; 17 | import org.apache.http.impl.client.HttpClientBuilder; 18 | import org.apache.http.message.BasicHeader; 19 | 20 | /** 21 | * Client to talk to TestRail API end points 22 | * 23 | * @author nullin 24 | */ 25 | public class APIClient { 26 | 27 | private HttpClient httpClient; 28 | private String url; 29 | private Logger logger = Logger.getLogger(APIClient.class.getName()); 30 | 31 | public APIClient(String url, String user, String password) { 32 | try { 33 | List
headerList = new ArrayList
(); 34 | headerList.add(new BasicHeader("Content-Type", "application/json")); 35 | headerList.add(new BasicHeader("Authorization", "Basic " + getAuthorization(user, password))); 36 | 37 | httpClient = HttpClientBuilder.create().setDefaultHeaders(headerList).build(); 38 | this.url = url + "/index.php?/api/v2/"; 39 | logger.fine("Created API client for " + url); 40 | } catch (Exception e) { 41 | throw new RuntimeException(e); 42 | } 43 | } 44 | 45 | private String getAuthorization(String user, String password) { 46 | try { 47 | return getBase64((user + ":" + password).getBytes("UTF-8")); 48 | } catch (UnsupportedEncodingException e) { 49 | // Not thrown 50 | } 51 | 52 | return ""; 53 | } 54 | 55 | /** 56 | * @see https://github.com/gurock/testrail-api/blob/master/java/com/gurock/testrail/APIClient.java 57 | */ 58 | private String getBase64(byte[] buffer) { 59 | final char[] map = { 60 | 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 61 | 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 62 | 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 63 | 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 64 | 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 65 | 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', 66 | '8', '9', '+', '/' 67 | }; 68 | 69 | StringBuffer sb = new StringBuffer(); 70 | for (int i = 0; i < buffer.length; i++) { 71 | byte b0 = buffer[i++], b1 = 0, b2 = 0; 72 | 73 | int bytes = 3; 74 | if (i < buffer.length) { 75 | b1 = buffer[i++]; 76 | if (i < buffer.length) { 77 | b2 = buffer[i]; 78 | } else { 79 | bytes = 2; 80 | } 81 | } else { 82 | bytes = 1; 83 | } 84 | 85 | int total = (b0 << 16) | (b1 << 8) | b2; 86 | 87 | switch (bytes) { 88 | case 3: 89 | sb.append(map[(total >> 18) & 0x3f]); 90 | sb.append(map[(total >> 12) & 0x3f]); 91 | sb.append(map[(total >> 6) & 0x3f]); 92 | sb.append(map[total & 0x3f]); 93 | break; 94 | 95 | case 2: 96 | sb.append(map[(total >> 18) & 0x3f]); 97 | sb.append(map[(total >> 12) & 0x3f]); 98 | sb.append(map[(total >> 6) & 0x3f]); 99 | sb.append('='); 100 | break; 101 | 102 | case 1: 103 | sb.append(map[(total >> 18) & 0x3f]); 104 | sb.append(map[(total >> 12) & 0x3f]); 105 | sb.append('='); 106 | sb.append('='); 107 | break; 108 | } 109 | } 110 | 111 | return sb.toString(); 112 | } 113 | 114 | public String invokeHttpGet(String uriSuffix) throws IOException, ClientException { 115 | logger.fine("Invoking " + uriSuffix); 116 | HttpGet httpGet = new HttpGet(url + uriSuffix); 117 | return consumeResponse(httpClient.execute(httpGet)); 118 | } 119 | 120 | public String invokeHttpPost(String uriSuffix, String jsonData) throws IOException, ClientException { 121 | logger.fine("Invoking " + uriSuffix + " with jsonData " + jsonData); 122 | HttpPost httpPost = new HttpPost(url + uriSuffix); 123 | StringEntity reqEntity = new StringEntity(jsonData); 124 | httpPost.setEntity(reqEntity); 125 | return consumeResponse(httpClient.execute(httpPost)); 126 | } 127 | 128 | public String consumeResponse(HttpResponse response) throws ClientException, IOException { 129 | int status = response.getStatusLine().getStatusCode(); 130 | HttpEntity entity = response.getEntity(); 131 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 132 | entity.writeTo(os); 133 | String content = os.toString("UTF-8"); 134 | 135 | logger.fine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); 136 | logger.fine(response.getStatusLine().toString()); 137 | logger.fine(content); 138 | logger.fine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); 139 | 140 | if (status != 200) { 141 | throw new ClientException("Received status code " + status + " with content '" + content + "'"); 142 | } 143 | 144 | return content; 145 | } 146 | 147 | } -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/client/ClientException.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.client; 2 | 3 | /** 4 | * Exception raised something goes wrong communicating with TestRail server 5 | * 6 | * @author nullin 7 | */ 8 | public class ClientException extends Exception 9 | { 10 | public ClientException(String message) 11 | { 12 | super(message); 13 | } 14 | 15 | public ClientException(String message, Exception ex) 16 | { 17 | super(message, ex); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/client/TestRailClient.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.client; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import com.fasterxml.jackson.annotation.JsonInclude; 9 | import com.fasterxml.jackson.core.type.TypeReference; 10 | import com.fasterxml.jackson.databind.DeserializationFeature; 11 | import com.fasterxml.jackson.databind.ObjectMapper; 12 | import com.fasterxml.jackson.databind.SerializationFeature; 13 | import com.nullin.testrail.dto.Case; 14 | import com.nullin.testrail.dto.Milestone; 15 | import com.nullin.testrail.dto.Plan; 16 | import com.nullin.testrail.dto.PlanEntry; 17 | import com.nullin.testrail.dto.Result; 18 | import com.nullin.testrail.dto.Run; 19 | import com.nullin.testrail.dto.Section; 20 | import com.nullin.testrail.dto.Suite; 21 | import com.nullin.testrail.dto.Test; 22 | 23 | /** 24 | * TestRail Client for endpoints described at 25 | * {@link http://docs.gurock.com/testrail-api2/start} 26 | * 27 | * Contains methods to talk to all the various endpoints for all the 28 | * different object types within this one class. The method parameters 29 | * translate to the fields accepted as part of the request URL as well as a 30 | * map of fields that can be passed as body of the POST requests 31 | * 32 | * @author nullin 33 | */ 34 | public class TestRailClient { 35 | 36 | //underlying api client 37 | private APIClient client; 38 | //(de)-serializes objects to/from json 39 | private ObjectMapper objectMapper; 40 | 41 | /** 42 | * Creates an instance of the client and setups up required state 43 | * 44 | * @param url 45 | * @param username 46 | * @param password 47 | */ 48 | public TestRailClient(String url, String username, String password) { 49 | client = new APIClient(url, username, password); 50 | objectMapper = new ObjectMapper(); 51 | objectMapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); 52 | objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); 53 | //TODO: should probably remove this 54 | objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); 55 | } 56 | 57 | /* 58 | Plans 59 | */ 60 | 61 | public Plan getPlan(int planId) throws IOException, ClientException { 62 | return objectMapper.readValue(client.invokeHttpGet("get_plan/" + planId), Plan.class); 63 | } 64 | 65 | public List getPlans(int projectId, Map filters) 66 | throws IOException, ClientException { 67 | String url = "get_plans/" + projectId; 68 | if (filters != null) { 69 | for (Map.Entry entry : filters.entrySet()) { 70 | url += "&" + entry.getKey() + "=" + entry.getValue(); 71 | } 72 | } 73 | return objectMapper.readValue(client.invokeHttpGet(url), new TypeReference>(){}); 74 | } 75 | 76 | public Plan addPlan(int projectId, String name, Integer milestoneId, List entries) 77 | throws IOException, ClientException { 78 | Map body = new HashMap(); 79 | body.put("name", name); 80 | if (milestoneId != null) { 81 | body.put("milestone_id", String.valueOf(milestoneId)); 82 | } 83 | if (entries != null) { 84 | body.put("entries", entries); 85 | } 86 | return objectMapper.readValue( 87 | client.invokeHttpPost("add_plan/" + projectId, objectMapper.writeValueAsString(body)), 88 | Plan.class); 89 | } 90 | 91 | public PlanEntry addPlanEntry(int planId, int suiteId) throws IOException, ClientException { 92 | Map body = new HashMap(); 93 | body.put("suite_id", String.valueOf(suiteId)); 94 | return objectMapper.readValue( 95 | client.invokeHttpPost("add_plan_entry/" + planId, objectMapper.writeValueAsString(body)), 96 | PlanEntry.class); 97 | } 98 | 99 | public Plan closePlan(int planId) throws IOException, ClientException { 100 | return objectMapper.readValue(client.invokeHttpPost("close_plan/" + planId, ""), Plan.class); 101 | } 102 | 103 | public void deletePlan(int planId) throws IOException, ClientException { 104 | client.invokeHttpPost("delete_plan/" + planId, ""); 105 | } 106 | 107 | /* 108 | Results 109 | */ 110 | 111 | public List getResults(int testId) throws IOException, ClientException { 112 | String url = "get_results/" + testId; 113 | return objectMapper.readValue(client.invokeHttpGet(url), new TypeReference>(){}); 114 | } 115 | 116 | public List getResultsForRun(int runId, Map filters) throws IOException, ClientException { 117 | String url = "get_results_for_run/" + runId; 118 | if (filters != null) { 119 | for (Map.Entry entry : filters.entrySet()) { 120 | url += "&" + entry.getKey() + "=" + entry.getValue(); 121 | } 122 | } 123 | return objectMapper.readValue(client.invokeHttpGet(url), new TypeReference>(){}); 124 | } 125 | 126 | /** 127 | * 128 | * @return 129 | */ 130 | public List getResultsForCase(int runId, int caseId) throws IOException, ClientException { 131 | String url = "get_results_for_case/" + runId + "/" + caseId + "&limit=10"; 132 | return objectMapper.readValue(client.invokeHttpGet(url), new TypeReference>(){}); 133 | } 134 | 135 | public Result addResultForCase(int runId, int caseId, int statusId, String comment) 136 | throws IOException, ClientException { 137 | String url = "add_result_for_case/" + runId + "/" + caseId; 138 | Map body = new HashMap(); 139 | body.put("status_id", String.valueOf(statusId)); 140 | body.put("comment", comment); 141 | return objectMapper.readValue(client.invokeHttpPost(url, objectMapper.writeValueAsString(body)), Result.class); 142 | } 143 | 144 | public Result addResultForCase(int runId, int caseId, Map properties) 145 | throws IOException, ClientException { 146 | String url = "add_result_for_case/" + runId + "/" + caseId; 147 | return objectMapper.readValue(client.invokeHttpPost(url, objectMapper.writeValueAsString(properties)), Result.class); 148 | } 149 | 150 | /* 151 | Tests 152 | */ 153 | 154 | public Test getTest(int testId) throws IOException, ClientException { 155 | return objectMapper.readValue(client.invokeHttpGet("get_test/" + testId), Test.class); 156 | } 157 | 158 | public List getTests(int runId) throws IOException, ClientException { 159 | return objectMapper.readValue(client.invokeHttpGet("get_tests/" + runId), new TypeReference>(){}); 160 | } 161 | 162 | /* 163 | Cases 164 | */ 165 | 166 | public Case addCase(int sectionId, String title, Map fields) 167 | throws IOException, ClientException { 168 | Map body = new HashMap(); 169 | body.put("title", title); 170 | if (fields != null) { 171 | body.putAll(fields); 172 | } 173 | return objectMapper.readValue( 174 | client.invokeHttpPost("add_case/" + sectionId, objectMapper.writeValueAsString(body)), 175 | Case.class); 176 | } 177 | 178 | public Case getCase(int caseId) throws IOException, ClientException { 179 | return objectMapper.readValue(client.invokeHttpGet("get_case/" + caseId), Case.class); 180 | } 181 | 182 | public List getCases(int projectId, int suiteId, int sectionId, Map filters) 183 | throws IOException, ClientException { 184 | String url = "get_cases/" + projectId; 185 | if (suiteId > 0) { 186 | url += "&suite_id=" + suiteId; 187 | } 188 | if (sectionId > 0) { 189 | url += "§ion_id=" + sectionId; 190 | } 191 | if (filters != null) { 192 | for (Map.Entry entry : filters.entrySet()) { 193 | url += "&" + entry.getKey() + "=" + entry.getValue(); 194 | } 195 | } 196 | return objectMapper.readValue(client.invokeHttpGet(url), new TypeReference>(){}); 197 | } 198 | 199 | /** 200 | * Needed when you need to work with custom fields that are not part of the {@link Case} class 201 | * @param projectId 202 | * @param suiteId 203 | * @param sectionId 204 | * @param filters 205 | * @return 206 | * @throws IOException 207 | * @throws ClientException 208 | */ 209 | public List> getCasesAsMap(int projectId, int suiteId, int sectionId, Map filters) 210 | throws IOException, ClientException { 211 | String url = "get_cases/" + projectId; 212 | if (suiteId > 0) { 213 | url += "&suite_id=" + suiteId; 214 | } 215 | if (sectionId > 0) { 216 | url += "§ion_id=" + sectionId; 217 | } 218 | if (filters != null) { 219 | for (Map.Entry entry : filters.entrySet()) { 220 | url += "&" + entry.getKey() + "=" + entry.getValue(); 221 | } 222 | } 223 | return objectMapper.readValue(client.invokeHttpGet(url), new TypeReference>>(){}); 224 | } 225 | 226 | public Case updateCase(int caseId, Map fields) throws IOException, ClientException { 227 | return objectMapper.readValue(client.invokeHttpPost("update_case/" + caseId, 228 | objectMapper.writeValueAsString(fields)), Case.class); 229 | } 230 | 231 | /* 232 | Sections 233 | */ 234 | 235 | public Section addSection(int projectId, String name, int parentId, int suiteId) 236 | throws IOException, ClientException { 237 | Map body = new HashMap(); 238 | if (suiteId > 0) { 239 | body.put("suite_id", String.valueOf(suiteId)); 240 | } 241 | if (parentId > 0) { 242 | body.put("parent_id", String.valueOf(parentId)); 243 | } 244 | body.put("name", name); 245 | return objectMapper.readValue( 246 | client.invokeHttpPost("add_section/" + projectId, objectMapper.writeValueAsString(body)), 247 | Section.class); 248 | } 249 | 250 | /* 251 | Suites 252 | */ 253 | 254 | public Suite addSuite(int projectId, String name) throws IOException, ClientException { 255 | Map body = new HashMap(); 256 | body.put("name", name); 257 | return objectMapper.readValue( 258 | client.invokeHttpPost("add_suite/" + projectId, objectMapper.writeValueAsString(body)), Suite.class); 259 | } 260 | 261 | public Suite getSuite(int suiteId) throws IOException, ClientException { 262 | return objectMapper.readValue(client.invokeHttpGet("get_suite/" + suiteId), Suite.class); 263 | } 264 | 265 | public List getSuites(int projectId) throws IOException, ClientException { 266 | return objectMapper.readValue(client.invokeHttpGet("get_suites/" + projectId), 267 | new TypeReference>(){}); 268 | } 269 | 270 | /* 271 | Milestones 272 | */ 273 | 274 | public Milestone getMilestone(int milestoneId) throws IOException, ClientException { 275 | return objectMapper.readValue(client.invokeHttpGet("get_milestone/" + milestoneId), Milestone.class); 276 | } 277 | 278 | public List getMilestones(int projectId) throws IOException, ClientException { 279 | return objectMapper.readValue(client.invokeHttpGet("get_milestones/" + projectId), 280 | new TypeReference>(){}); 281 | } 282 | 283 | public Milestone addMilestone(int projectId, String name, String description) throws IOException, ClientException { 284 | Map body = new HashMap(); 285 | body.put("name", name); 286 | if (description != null) { 287 | body.put("description", description); 288 | } 289 | return objectMapper.readValue( 290 | client.invokeHttpPost("add_milestone/" + projectId, objectMapper.writeValueAsString(body)), Milestone.class); 291 | } 292 | 293 | /* 294 | Runs 295 | */ 296 | 297 | public Run getRun(int runId) throws IOException, ClientException { 298 | return objectMapper.readValue(client.invokeHttpGet("get_run/" + runId), Run.class); 299 | } 300 | 301 | } 302 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Case.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | /** 6 | * Case represents a single test case in TestRail 7 | * 8 | * @author nullin 9 | */ 10 | public class Case { 11 | 12 | public int id; 13 | public String title; 14 | @JsonProperty("suite_id") 15 | public int suiteId; 16 | @JsonProperty("type_id") 17 | public int typeId; 18 | @JsonProperty("milestone_id") 19 | public int milestoneId; 20 | @JsonProperty("section_id") 21 | public int sectionId; 22 | //following property is not present in TestRail by default 23 | //we need to add it as a custom field 24 | @JsonProperty("custom_automation_id") 25 | public String automationId; 26 | public String refs; 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Milestone.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | /** 4 | * Milestone 5 | * 6 | * @author nullin 7 | */ 8 | public class Milestone { 9 | 10 | public int id; 11 | public String name; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Plan.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | import java.util.List; 4 | 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | /** 8 | * Test Plan 9 | * 10 | * @author nullin 11 | */ 12 | public class Plan { 13 | 14 | public int id; 15 | public String name; 16 | @JsonProperty("milestone_id") 17 | public Integer milestoneId; 18 | public List entries; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/PlanEntry.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | import java.util.List; 4 | 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | /** 8 | * Test Plan entry 9 | * 10 | * @author nullin 11 | */ 12 | public class PlanEntry { 13 | 14 | public String id; 15 | @JsonProperty("suite_id") 16 | public int suiteId; 17 | public String name; 18 | public List runs; 19 | @JsonProperty("assignedto_id") 20 | public Integer assignedTo; 21 | @JsonProperty("include_all") 22 | public boolean includeAll; 23 | @JsonProperty("case_ids") 24 | public List caseIds; 25 | @JsonProperty("config_ids") 26 | public List configIds; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Result.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | /** 6 | * Test Result 7 | * 8 | * @author nullin 9 | */ 10 | public class Result { 11 | 12 | public int id; 13 | @JsonProperty("test_id") 14 | public int testId; 15 | @JsonProperty("status_id") 16 | public Integer statusId; 17 | public String comment; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Run.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | import java.util.List; 4 | 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | /** 8 | * Test run 9 | * 10 | * @author nullin 11 | */ 12 | public class Run { 13 | 14 | public Integer id; 15 | @JsonProperty("project_id") 16 | public Integer projectId; 17 | @JsonProperty("suite_id") 18 | public Integer suiteId; 19 | @JsonProperty("plan_id") 20 | public Integer planId; 21 | public String name; 22 | @JsonProperty("assignedto_id") 23 | public Integer assignedTo; 24 | @JsonProperty("include_all") 25 | public boolean includeAll; 26 | @JsonProperty("case_ids") 27 | public List caseIds; 28 | @JsonProperty("config_ids") 29 | public List configIds; 30 | public String config; 31 | 32 | } 33 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Section.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | /** 4 | * Sections within test suites 5 | * 6 | * @author nullin 7 | */ 8 | public class Section { 9 | 10 | public int id; 11 | public String name; 12 | public int depth; 13 | public int parent_id; 14 | public int suite_id; 15 | public int display_order; 16 | public String description; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Suite.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | /** 4 | * Test suite 5 | * 6 | * @author nullin 7 | */ 8 | public class Suite { 9 | 10 | public int id; 11 | public String name; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/dto/Test.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | /** 6 | * Represents a test instance (an instance of a test case) 7 | * 8 | * @author nullin 9 | */ 10 | public class Test { 11 | 12 | public int id; 13 | @JsonProperty("case_id") 14 | public int caseId; 15 | @JsonProperty("statusId") 16 | public int status_id; 17 | @JsonProperty("run_id") 18 | public int runId; 19 | //following property is not present in TestRail by default 20 | //we need to add it as a custom field 21 | @JsonProperty("custom_automation_id") 22 | public String automationId; 23 | 24 | } 25 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/internal/GenerateTestCases.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.internal; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.Random; 9 | import java.util.UUID; 10 | 11 | import com.nullin.testrail.client.TestRailClient; 12 | import com.nullin.testrail.dto.Section; 13 | import com.nullin.testrail.dto.Suite; 14 | 15 | /** 16 | * Simple class to generate some different kinds of projects with ~10000 17 | * test cases. Used it for testing out some different scenarios while evaluating 18 | * TestRail 19 | * 20 | * @author nullin 21 | */ 22 | public class GenerateTestCases { 23 | 24 | private TestRailClient client; 25 | 26 | public GenerateTestCases(String[] args) { 27 | client = new TestRailClient(args[0], args[1], args[2]); 28 | } 29 | 30 | public static void main(String[] args) throws Exception { 31 | GenerateTestCases tcs = new GenerateTestCases(args); 32 | tcs.generateType1(); 33 | tcs.generateType2(); 34 | tcs.generateType3(); 35 | tcs.generateType4(); 36 | } 37 | 38 | private void generateType1() throws Exception { 39 | List sizes = new ArrayList(); 40 | sizes.add(100); 41 | sizes.add(50); 42 | sizes.add(25); 43 | sizes.add(25); 44 | 45 | int projectId = 8; 46 | for (int i = 0 ; i < 50 ; i++) { 47 | Section section = client.addSection(projectId, "Section " + UUID.randomUUID(), 0, 0); 48 | Collections.shuffle(sizes); 49 | for (int j = 0 ; j < 4 ; j++) { 50 | Section childSection = client.addSection(projectId, "Section " + UUID.randomUUID(), section.id, 0); 51 | for (int k = 0 ; k < sizes.get(j) ; k++) { 52 | addCase(client, childSection.id); 53 | } 54 | } 55 | } 56 | } 57 | 58 | private void generateType2() throws Exception { 59 | List sizes = new ArrayList(); 60 | sizes.add(100); 61 | sizes.add(50); 62 | sizes.add(25); 63 | sizes.add(25); 64 | 65 | int projectId = 9; 66 | int suiteId = 13; 67 | for (int i = 0 ; i < 50 ; i++) { 68 | Section section = client.addSection(projectId, "Section " + UUID.randomUUID(), 0, suiteId); 69 | Collections.shuffle(sizes); 70 | for (int j = 0 ; j < 4 ; j++) { 71 | Section childSection = client.addSection(projectId, "Section " + UUID.randomUUID(), section.id, suiteId); 72 | for (int k = 0 ; k < sizes.get(j) ; k++) { 73 | addCase(client, childSection.id); 74 | } 75 | } 76 | } 77 | } 78 | 79 | private void generateType3() throws Exception { 80 | List sizes = new ArrayList(); 81 | sizes.add(100); 82 | sizes.add(50); 83 | sizes.add(25); 84 | sizes.add(25); 85 | 86 | int projectId = 11; 87 | int suiteId = 15; 88 | for (int i = 0 ; i < 5 ; i++) { 89 | Section section = client.addSection(projectId, "Section " + UUID.randomUUID(), 0, suiteId); 90 | Collections.shuffle(sizes); 91 | for (int j = 0 ; j < 4 ; j++) { 92 | Section childSection = client.addSection(projectId, "Section " + UUID.randomUUID(), section.id, suiteId); 93 | for (int k = 0 ; k < sizes.get(j) ; k++) { 94 | addCase(client, childSection.id); 95 | } 96 | } 97 | } 98 | } 99 | 100 | private void generateType4() throws Exception { 101 | List sizes = new ArrayList(); 102 | sizes.add(100); 103 | sizes.add(50); 104 | sizes.add(25); 105 | sizes.add(25); 106 | 107 | int projectId = 12; 108 | for (int i = 0 ; i < 6 ; i++) { 109 | Suite suite = client.addSuite(projectId, "Suite " + UUID.randomUUID()); 110 | Collections.shuffle(sizes); 111 | for (int j = 0 ; j < 4 ; j++) { 112 | Section childSection = client.addSection(projectId, "Section " + UUID.randomUUID(), 0, suite.id); 113 | for (int k = 0 ; k < sizes.get(j) ; k++) { 114 | addCase(client, childSection.id); 115 | } 116 | } 117 | } 118 | } 119 | 120 | private void addCase(TestRailClient client, int sectionId) throws Exception { 121 | Map fields = new HashMap(); 122 | fields.put("custom_expected", "Expected\n\n" + getString(20)); 123 | fields.put("custom_preconds", "Preconds\n\n" + getString(5)); 124 | fields.put("custom_steps", "Steps\n\n" + getString(10)); 125 | client.addCase(sectionId, "Case " + UUID.randomUUID(), fields); 126 | } 127 | 128 | private String getString(int i) { 129 | StringBuilder sb = new StringBuilder(); 130 | for (int c = 0 ; c < i ; c++) { 131 | sb.append(UUID.randomUUID().toString()); 132 | if (c % (new Random().nextInt(i) + 1) == 0) { 133 | sb.append("\n"); 134 | } 135 | } 136 | return sb.toString(); 137 | } 138 | 139 | } 140 | -------------------------------------------------------------------------------- /testrail-connector/src/main/java/com/nullin/testrail/internal/TestTheListener.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.internal; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import com.nullin.testrail.client.ClientException; 9 | import com.nullin.testrail.client.TestRailClient; 10 | import com.nullin.testrail.dto.Case; 11 | import com.nullin.testrail.dto.Plan; 12 | import com.nullin.testrail.dto.PlanEntry; 13 | import com.nullin.testrail.dto.Result; 14 | import com.nullin.testrail.dto.Run; 15 | import com.nullin.testrail.dto.Suite; 16 | import org.testng.Assert; 17 | 18 | /** 19 | * Test the implementation of our API calls 20 | * 21 | * @author nullin 22 | */ 23 | public class TestTheListener { 24 | 25 | public static void main(String[] args) throws ClientException, IOException { 26 | 27 | TestRailClient client = new TestRailClient(args[0], args[1], args[2]); 28 | 29 | int projectId = 6; //should be read from a configuration 30 | 31 | List suiteList = client.getSuites(projectId); 32 | Map suiteMap = getSuiteMap(suiteList); 33 | System.out.println("Suite Map: " + suiteMap); 34 | 35 | Plan plan = client.getPlan(6); 36 | // Plan plan = client.addPlan(projectId, "Test Plan_" + new Date().toString(), null); 37 | 38 | try { 39 | int suiteId = suiteMap.get("Master"); 40 | // int caseId = 226; 41 | 42 | Assert.assertEquals(plan.entries.size(), 0, "Plan Entries"); 43 | 44 | List cases = client.getCases(projectId, 0, 0, null); 45 | System.out.println(cases.size()); 46 | int caseId = findCase(cases, "test1"); 47 | Assert.assertEquals(caseId, 226, "Case id"); 48 | 49 | PlanEntry planEntry = findPlanEntry(plan.entries, suiteId); 50 | if (planEntry == null) { 51 | planEntry = client.addPlanEntry(plan.id, suiteId); 52 | } 53 | 54 | //int runId = getRunId(plan.entries, suiteId); 55 | int runId = planEntry.runs.get(0).id; 56 | 57 | Result result = client.addResultForCase(runId, caseId, 1, "Nalin Added this comment Yo!! Yolo!"); 58 | System.out.println(result); 59 | } finally { 60 | client.closePlan(plan.id); // don't complete, because completed plans can't be deleted 61 | client.deletePlan(plan.id); 62 | } 63 | System.out.println(""); 64 | } 65 | 66 | private static int findCase(List cases, String test1) { 67 | for (Case c : cases) { 68 | if (c.automationId != null && c.automationId.equals(test1)) { 69 | return c.id; 70 | } 71 | } 72 | return -1; 73 | } 74 | 75 | private static PlanEntry findPlanEntry(List entries, int suiteId) { 76 | for (PlanEntry planEntry : entries) { 77 | if (planEntry.suiteId == suiteId) { 78 | return planEntry; 79 | } 80 | } 81 | return null; 82 | } 83 | 84 | private static int getRunId(List entries, int suiteId) { 85 | for (PlanEntry planEntry : entries) { 86 | if (planEntry.suiteId == suiteId) { 87 | List runs = planEntry.runs; 88 | return runs.get(0).id; 89 | } 90 | } 91 | throw new IllegalArgumentException("Didn't find entry for suite " + suiteId); 92 | } 93 | 94 | private static Map getSuiteMap(List suites) { 95 | Map suiteMap = new HashMap(suites.size()); 96 | for (Suite suite : suites) { 97 | suiteMap.put(suite.name, suite.id); 98 | } 99 | return suiteMap; 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /testrail-connector/src/test/java/com/nullin/testrail/sampleproj/TestClassA.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.sampleproj; 2 | 3 | import java.util.Random; 4 | 5 | import com.nullin.testrail.annotations.TestRailCase; 6 | import org.testng.Assert; 7 | import org.testng.annotations.DataProvider; 8 | import org.testng.annotations.Test; 9 | 10 | /** 11 | * 12 | * @author nullin 13 | */ 14 | public class TestClassA { 15 | 16 | @TestRailCase("testA1") 17 | @Test 18 | public void test1() { 19 | Assert.assertTrue(getResult(10, 2)); 20 | } 21 | 22 | @DataProvider(name = "MYDP") 23 | public Object[][] getData() { 24 | return new Object[][] { 25 | {"testA2", 10, 3}, 26 | {"testA3", 10, 5} 27 | }; 28 | } 29 | 30 | //@TestRailCase(dataDriven = true) 31 | @Test(dataProvider = "MYDP") 32 | public void test2(String testId, int x, int y) { 33 | Assert.assertTrue(getResult(x, y)); 34 | } 35 | 36 | public static boolean getResult(int max, int r) { 37 | return new Random(max).nextInt() % r != 0; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /testrail-connector/src/test/java/com/nullin/testrail/sampleproj/TestClassB.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.sampleproj; 2 | 3 | import com.nullin.testrail.annotations.TestRailCase; 4 | import org.testng.annotations.Test; 5 | 6 | /** 7 | * 8 | * @author nullin 9 | */ 10 | public class TestClassB { 11 | 12 | @TestRailCase("testB1") 13 | @Test 14 | public void test1() { 15 | // do nothing always passes 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /testrail-connector/src/test/java/com/nullin/testrail/sampleproj/pkg/TestClassC.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.sampleproj.pkg; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import com.nullin.testrail.ResultStatus; 8 | import com.nullin.testrail.TestRailReporter; 9 | import com.nullin.testrail.annotations.TestRailCase; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | /** 14 | * 15 | * @author nullin 16 | */ 17 | public class TestClassC { 18 | 19 | @TestRailCase("testC1") 20 | @Test 21 | public void test1() { 22 | // do nothing always passes 23 | } 24 | 25 | @TestRailCase("testC2") 26 | @Test 27 | public void test4() { 28 | Assert.fail("Always fails!!"); 29 | } 30 | 31 | @TestRailCase(selfReporting = true) 32 | @Test 33 | public void test5() { 34 | Map result = new HashMap(); 35 | result.put(TestRailReporter.KEY_STATUS, ResultStatus.PASS); 36 | TestRailReporter.getInstance().reportResult("testC3", result); 37 | result.put(TestRailReporter.KEY_STATUS, ResultStatus.FAIL); 38 | result.put(TestRailReporter.KEY_THROWABLE, new IOException("Something very bad happened!!")); 39 | TestRailReporter.getInstance().reportResult("testC4", result); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /testrail-connector/src/test/resources/testng.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /testrail-utils/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.nullin 7 | testrail-integration 8 | 2.3.5-SNAPHSOT 9 | 10 | 11 | testrail-tools 12 | jar 13 | 14 | TestRail Utils 15 | https://github.com/nullin/testrail-integration/testrail-tools 16 | 17 | 18 | 19 | com.google.guava 20 | guava 21 | 22 | 23 | org.testng 24 | testng 25 | 26 | 27 | com.nullin 28 | testrail-connector 29 | ${project.version} 30 | 31 | 32 | com.atlassian.jira 33 | jira-rest-java-client-core 34 | 3.0.0 35 | 36 | 37 | 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-surefire-plugin 43 | 2.9 44 | 45 | true 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | my-repo 54 | https://github.com/nullin/mvn-repo 55 | 56 | 57 | 58 | 59 | 60 | atlassian-public 61 | atlassian-public 62 | https://m2proxy.atlassian.com/repository/public 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /testrail-utils/src/main/java/com/nullin/testrail/tools/InvalidJiraReferenceFinder.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.tools; 2 | 3 | import com.atlassian.jira.rest.client.api.JiraRestClient; 4 | import com.atlassian.jira.rest.client.api.JiraRestClientFactory; 5 | import com.atlassian.jira.rest.client.api.domain.Issue; 6 | import com.atlassian.jira.rest.client.api.domain.Status; 7 | import com.atlassian.jira.rest.client.internal.async.AsynchronousJiraRestClientFactory; 8 | import com.nullin.testrail.client.TestRailClient; 9 | import com.nullin.testrail.dto.*; 10 | 11 | import java.net.URI; 12 | import java.util.*; 13 | 14 | /** 15 | * Find and prints out JIRA bug status for bugs referenced in the 'Reference' field of test cases 16 | * 17 | * When using references to point to bugs that are causing test cases to fail, this tool is useful for finding 18 | * references to resolved or closed bugs. 19 | * 20 | * @author nullin 21 | */ 22 | public class InvalidJiraReferenceFinder { 23 | 24 | /** 25 | * currently takes 6 args: 26 | * 27 | * {testrail URL} {testrail user} {testrail passwd} {jira URL} {jira user} {jira passwd} 28 | * 29 | * @param args 30 | * @throws Exception 31 | */ 32 | public static void main(String[] args) throws Exception { 33 | TestRailClient client = new TestRailClient(args[0], args[1], args[2]); 34 | final JiraRestClientFactory factory = new AsynchronousJiraRestClientFactory(); 35 | final URI jiraServerUri = new URI(args[3]); 36 | final JiraRestClient restClient = factory.createWithBasicHttpAuthentication(jiraServerUri, args[4], args[5]); 37 | 38 | int projectId = 1; 39 | String suiteName = "master"; //TODO: make configurable 40 | 41 | List suites = client.getSuites(projectId); 42 | 43 | int suiteId = -1; 44 | for (Suite suite : suites) { 45 | if (suite.name.equals(suiteName)) { 46 | suiteId = suite.id; 47 | } 48 | } 49 | 50 | List cases = client.getCases(projectId, suiteId, 0, null); 51 | for (Case _case : cases) { 52 | String refs = _case.refs; 53 | if (refs == null || refs.isEmpty()) { 54 | continue; 55 | } 56 | String[] refsArr = refs.split(","); 57 | for (String ref : refsArr) { 58 | ref = ref.trim(); 59 | checkReference(restClient, _case, ref); 60 | } 61 | } 62 | 63 | System.exit(0); 64 | } 65 | 66 | private static void checkReference(JiraRestClient client, Case aCase, String ref) { 67 | Issue issue = client.getIssueClient().getIssue(ref).claim(); 68 | Status status = issue.getStatus(); 69 | System.out.println(String.format("Case %8d (%75s), %d, %10s, %12s", aCase.id, aCase.automationId, 70 | aCase.typeId, ref, status.getName())); 71 | } 72 | 73 | } -------------------------------------------------------------------------------- /testrail-utils/src/main/java/com/nullin/testrail/tools/UnstableTestsFinder.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.tools; 2 | 3 | import com.google.common.base.Function; 4 | import com.google.common.collect.Lists; 5 | import com.google.common.collect.Maps; 6 | import com.nullin.testrail.client.ClientException; 7 | import com.nullin.testrail.client.TestRailClient; 8 | import com.nullin.testrail.dto.*; 9 | 10 | 11 | import java.io.IOException; 12 | import java.util.*; 13 | 14 | /** 15 | * Attempts to find unstable tests by looking at past test results. 16 | * 17 | * Milestone name is used to find associated TestPlans and then we get all test results for each test case (based on 18 | * automation id) and finally check if we see unstable results for the tests. 19 | * 20 | * @author nullin 21 | */ 22 | public class UnstableTestsFinder { 23 | 24 | /** 25 | * currently takes 3 args: 26 | * 27 | * {testrail URL} {testrail user} {testrail passwd} 28 | * 29 | * @param args 30 | * @throws Exception 31 | */ 32 | public static void main(String[] args) throws Exception { 33 | TestRailClient client = new TestRailClient(args[0], args[1], args[2]); 34 | 35 | int projectId = 1; 36 | String milestoneName = "master"; //TODO: make configurable 37 | 38 | List milestones = client.getMilestones(projectId); 39 | 40 | int milestoneId = -1; 41 | for (Milestone milestone : milestones) { 42 | if (milestone.name.equals(milestoneName)) { 43 | milestoneId = milestone.id; 44 | } 45 | } 46 | 47 | Map filters = new HashMap<>(); 48 | filters.put("milestone_id", String.valueOf(milestoneId)); 49 | filters.put("limit", "5"); //TODO: make configurable 50 | List plans = client.getPlans(projectId, filters); 51 | 52 | Map automationIdMap = Maps.newHashMap(); 53 | Map> resultMap = Maps.newLinkedHashMap(); 54 | 55 | for (Plan plan : plans) { 56 | System.out.println("Plan: " + plan.name); 57 | List planEntries = client.getPlan(plan.id).entries; 58 | for (PlanEntry planEntry : planEntries) { 59 | for (Run run : planEntry.runs) { 60 | List results = getResults(client, run.id); 61 | List tests = client.getTests(run.id); 62 | System.out.println("Run: " + run.name + ", Tests Size: " + tests.size() + ", Results Size: " + results.size()); 63 | Map> testResultMap = new LinkedHashMap<>(); 64 | 65 | for (Result result : results) { 66 | if (result.statusId != null) { 67 | if (testResultMap.get(result.testId) != null) { 68 | testResultMap.get(result.testId).add(result); 69 | } else { 70 | testResultMap.put(result.testId, Lists.newArrayList(result)); 71 | } 72 | } 73 | } 74 | for (Test test : tests) { 75 | if (!automationIdMap.containsKey(test.caseId)) { 76 | automationIdMap.put(test.caseId, test.automationId); 77 | } 78 | 79 | List testResults = testResultMap.get(test.id); 80 | if (testResults != null) { 81 | if (resultMap.get(test.caseId) != null) { 82 | resultMap.get(test.caseId).addAll(testResults); 83 | } else { 84 | resultMap.put(test.caseId, testResults); 85 | } 86 | } 87 | } 88 | } 89 | } 90 | } 91 | 92 | for (Map.Entry> entry : resultMap.entrySet()) { 93 | List results = entry.getValue(); 94 | List statuses = Lists.transform(results, new Function() { 95 | @Override 96 | public Integer apply(Result result) { 97 | return result.statusId; 98 | } 99 | }); 100 | boolean isUnstable = check(statuses); 101 | if (isUnstable) { 102 | System.out.println("Case " + entry.getKey() + " (" + automationIdMap.get(entry.getKey()) + "): " + statuses); 103 | } 104 | } 105 | 106 | } 107 | 108 | private static List getResults(TestRailClient client, Integer id) throws IOException, ClientException { 109 | Map filters = Maps.newHashMap(); 110 | List results = Lists.newArrayList(); 111 | List tmp; 112 | int offset = 250; 113 | while (!(tmp = client.getResultsForRun(id, filters)).isEmpty()) { 114 | results.addAll(tmp); 115 | filters.put("offset", String.valueOf(offset)); 116 | offset += 250; 117 | } 118 | return results; 119 | } 120 | 121 | private static boolean check(List statuses) { 122 | int count = 0; 123 | for (int i = 0 ; i < statuses.size() - 1; i++) { 124 | if (!statuses.get(i).equals(statuses.get(i + 1))) { 125 | count++; 126 | } 127 | } 128 | return count >= 3; 129 | } 130 | 131 | } -------------------------------------------------------------------------------- /testrail-utils/src/main/java/com/nullin/testrail/util/SoftReportingAssertion.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.util; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import com.nullin.testrail.ResultStatus; 7 | import com.nullin.testrail.TestRailReporter; 8 | import org.testng.asserts.IAssert; 9 | import org.testng.asserts.SoftAssert; 10 | 11 | /** 12 | * Soft Reporting Assertion class extends the functionality of {@link org.testng.asserts.SoftAssert} to 13 | * allow us to report results to TestRail using {@link com.nullin.testrail.TestRailReporter}. 14 | * 15 | * There are still issues here. If the test is not written correctly and there 16 | * are exceptions raised, we could have a situation where this error is not reported correctly to 17 | * TestRail. We are relying on the fact that there are no exceptions raised during the running of the 18 | * test method, except when {@code assertAll()} method is invoked. 19 | * 20 | * Usage: 21 | *
    22 | *
  • Should be used only with tests with {@link com.nullin.testrail.annotations.TestRailCase} annotation 23 | * marking the test as Self Reporting
    • 24 | *
  • Create an instance of this class for each test that needs to be reported on
    • 25 | *
  • Use that instance for running all asserts associated with a given test
    • 26 | *
  • At end of the overall test, use {@link SoftReportingAssertion#assertAll()} to run {@code assertAll()} methods 27 | * for all these instances in the finally block.
    • 28 | *
29 | * 30 | * @author nullin 31 | */ 32 | public class SoftReportingAssertion extends SoftAssert { 33 | 34 | //Automation Id for test to be reported on 35 | private String testAutomationId; 36 | private boolean hasFailedBefore; 37 | private boolean hasPassedBefore; 38 | private boolean hasRunAtLeastOnce; 39 | 40 | public SoftReportingAssertion(String testAutomationId) { 41 | this.testAutomationId = testAutomationId; 42 | } 43 | 44 | @Override 45 | public void onAssertSuccess(IAssert iAssert) { 46 | hasRunAtLeastOnce = true; 47 | super.onAssertSuccess(iAssert); 48 | if (!hasFailedBefore && !hasPassedBefore) { 49 | Map properties = new HashMap<>(); 50 | properties.put(TestRailReporter.KEY_STATUS, ResultStatus.PASS); 51 | properties.put(TestRailReporter.KEY_MORE_INFO, getMoreInfo()); 52 | TestRailReporter.getInstance().reportResult(testAutomationId, properties); 53 | hasPassedBefore = true; 54 | } 55 | } 56 | 57 | @Override 58 | public void onAssertFailure(IAssert iAssert, AssertionError ex) { 59 | hasRunAtLeastOnce = true; 60 | super.onAssertFailure(iAssert, ex); 61 | Map properties = new HashMap<>(); 62 | properties.put(TestRailReporter.KEY_STATUS, ResultStatus.FAIL); 63 | properties.put(TestRailReporter.KEY_THROWABLE, ex); 64 | properties.put(TestRailReporter.KEY_MORE_INFO, getMoreInfo()); 65 | TestRailReporter.getInstance().reportResult(testAutomationId, properties); 66 | hasFailedBefore = true; 67 | } 68 | 69 | private Map getMoreInfo() { 70 | return getMoreInfo(6); 71 | } 72 | 73 | private Map getMoreInfo(int stackTraceElIdx) { 74 | Map moreInfo = new HashMap<>(); 75 | StackTraceElement stackTraceElement = Thread.currentThread().getStackTrace()[stackTraceElIdx]; 76 | moreInfo.put("class", stackTraceElement.getClassName()); 77 | moreInfo.put("method", stackTraceElement.getMethodName()); 78 | return moreInfo; 79 | } 80 | 81 | private boolean hasRunAtLeastOnce() { 82 | return hasRunAtLeastOnce; 83 | } 84 | 85 | private String getTestAutomationId() { 86 | return testAutomationId; 87 | } 88 | 89 | /** 90 | * Helper method that runs the {@code assertAll()} method on all the instances passed in 91 | *

92 | * Note: This methods throws AssertionError so it should be called in try-finally and 93 | * all cleanups are done in finally block. 94 | * 

 95 |      *     try {
 96 |      *         test code
 97 |      *     } finally {
 98 |      *         try {
 99 |      *             SoftReportingAssertion.assertAll(assert_1, assert_2, assert_3);
100 |      *         } finally {
101 |      *             cleanup code
102 |      *         }
103 |      *     }
104 |      *
105 | *

106 | * @param asserts test asserts 107 | */ 108 | public static void assertAll(SoftReportingAssertion... asserts) { 109 | StringBuilder allAssertsMessages = new StringBuilder(); 110 | for (SoftReportingAssertion sr_assert : asserts) { 111 | if (sr_assert.hasRunAtLeastOnce()) { 112 | try { 113 | sr_assert.assertAll(); 114 | } catch (AssertionError assertionError) { 115 | allAssertsMessages.append("\n").append(sr_assert.getTestAutomationId()) 116 | .append("\n\t").append(assertionError.getMessage()).append("\n"); 117 | } 118 | } else { 119 | //not been invoked at all 120 | Map properties = new HashMap<>(); 121 | properties.put(TestRailReporter.KEY_STATUS, ResultStatus.SKIP); 122 | properties.put(TestRailReporter.KEY_THROWABLE, 123 | new AssertionError(sr_assert.getTestAutomationId() + " assert was not executed")); 124 | properties.put(TestRailReporter.KEY_MORE_INFO, sr_assert.getMoreInfo(3)); 125 | TestRailReporter.getInstance().reportResult(sr_assert.getTestAutomationId(), properties); 126 | } 127 | } 128 | 129 | String allAssertMsgsStr = allAssertsMessages.toString(); 130 | if (!allAssertMsgsStr.isEmpty()) { 131 | throw new AssertionError(allAssertsMessages.toString()); 132 | } 133 | } 134 | 135 | @Override 136 | public void assertAll() { 137 | super.assertAll(); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /testrail-utils/src/main/java/com/nullin/testrail/util/TestRailScreenshotListener.java: -------------------------------------------------------------------------------- 1 | package com.nullin.testrail.util; 2 | 3 | import org.testng.ITestResult; 4 | import com.google.common.collect.Lists; 5 | 6 | /** 7 | * An example of TestRail listener extended to take screenshots and pass this information to. 8 | * 9 | * @author nullin 10 | */ 11 | public class TestRailScreenshotListener extends com.nullin.testrail.TestRailListener { 12 | 13 | public String getScreenshotUrl(ITestResult result) { 14 | if (!Lists.newArrayList(ITestResult.FAILURE, 15 | ITestResult.SUCCESS_PERCENTAGE_FAILURE).contains(result.getStatus())) { 16 | //if the result doesn't represent a failure, no need to take screenshots 17 | return null; 18 | } 19 | 20 | try { 21 | // Do stuff to take screen shot 22 | // WebDriver driver = TWebDriverHolder.getWebDriver(); 23 | // String base64 = ((TakesScreenshot) driver).getScreenshotAs(OutputType.BASE64); 24 | // byte[] decodedScreenshot = Base64.decodeBase64(base64.getBytes()); 25 | 26 | String testname = getTestName(result); 27 | String timestamp = String.valueOf(System.currentTimeMillis()); 28 | String fileName = timestamp + "_" + testname + ".png"; 29 | 30 | return "http://some.hostname.com/" + fileName; 31 | } catch (Exception e) { 32 | //do nothing and only log the exception here 33 | } 34 | return null; 35 | } 36 | 37 | private String getTestName(ITestResult result) { 38 | String testName = result.getTestClass().getRealClass().getName() + "." + result.getMethod().getMethodName(); 39 | if (result.getParameters() != null && result.getParameters().length > 0) { 40 | testName += result.getParameters()[0]; 41 | } 42 | 43 | return testName; 44 | } 45 | 46 | } 47 | --------------------------------------------------------------------------------