├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── src ├── main │ └── java │ │ └── ai │ │ └── test │ │ └── sdk │ │ ├── package-info.java │ │ ├── CollectionUtils.java │ │ ├── JsonUtils.java │ │ ├── NetUtils.java │ │ ├── TestAiElement.java │ │ └── TestAiDriver.java └── test │ └── java │ └── ai │ └── test │ └── LibraryTest.java ├── .gitignore ├── README.md ├── gradlew.bat ├── gradlew └── LICENSE.md /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'test-ai-appium' -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testdotai/java-appium-sdk/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Contains the main classes for the Test.ai (Fluffy Dragon) SDK. These classes provide simple wrappers around existing appium functionality to seamlessly incorporate Test.ai's powerful element 3 | * classification technology. 4 | */ 5 | 6 | package ai.test.sdk; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | build/ 3 | 4 | # OS Generated files 5 | .DS_Store 6 | ._* 7 | .Spotlight-V100 8 | .Trashes 9 | Icon? 10 | ehthumbs.db 11 | Thumbs.db 12 | 13 | # Eclipse 14 | .project 15 | .classpath 16 | .settings 17 | 18 | # intellij 19 | .idea 20 | 21 | # Gradle 22 | .gradle/ 23 | /bin/ 24 | 25 | Scratch*.java -------------------------------------------------------------------------------- /src/test/java/ai/test/LibraryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This Java source file was generated by the Gradle 'init' task. 3 | */ 4 | package ai.test; 5 | 6 | import org.junit.jupiter.api.Test; 7 | //import static org.junit.jupiter.api.Assertions.*; 8 | 9 | class LibraryTest { 10 | @Test void someLibraryMethodReturnsTrue() { 11 | // Library classUnderTest = new Library(); 12 | // assertTrue(classUnderTest.someLibraryMethod(), "someLibraryMethod should return 'true'"); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/CollectionUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.util.HashMap; 4 | 5 | /** 6 | * Shared classes and methods enhancing collections functionality. 7 | * 8 | * @author Alexander Wu (alec@test.ai) 9 | * 10 | */ 11 | final class CollectionUtils 12 | { 13 | /** 14 | * Builds a {@code HashMap} out of a list of {@code String}s. Pass in values such that {@code [ k1, v1, k2, v2, k3, v3... ]}. 15 | * 16 | * @param sl The {@code String}s to use 17 | * @return A {@code HashMap} derived from the values in {@code sl} 18 | */ 19 | public static HashMap keyValuesToHM(String... sl) 20 | { 21 | HashMap m = new HashMap<>(); 22 | 23 | for (int i = 0; i < sl.length; i += 2) 24 | m.put(sl[i], sl[i + 1]); 25 | 26 | return m; 27 | } 28 | 29 | /** 30 | * Simple Tuple implementation. A Tuple is an immutable two-pair of values. It may consist of any two Objects, which may or may not be in of the same type. 31 | * 32 | * @author Alexander Wu (alec@test.ai) 33 | * 34 | * @param The type of Object allowed for the first Object in the tuple. 35 | * @param The type of Object allowed for the second Object in the tuple. 36 | */ 37 | public static class Tuple 38 | { 39 | /** 40 | * The k value of the tuple 41 | */ 42 | public final K k; 43 | 44 | /** 45 | * The y value of the tuple 46 | */ 47 | public final V v; 48 | 49 | /** 50 | * Constructor, creates a new Tuple from the specified values. 51 | * 52 | * @param k The first entry in the Tuple. 53 | * @param v The second entry in the Tuple. 54 | */ 55 | public Tuple(K k, V v) 56 | { 57 | this.k = k; 58 | this.v = v; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![test.ai sdk logo](https://testdotai.github.io/static-assets/logo-sdk.png)](https://adoptium.net) 2 | 3 | [![JDK-11+](https://img.shields.io/badge/JDK-11%2B-blue)](https://adoptium.net) 4 | [![Apache 2.0](https://img.shields.io/badge/Apache-2.0-blue)](https://www.apache.org/licenses/LICENSE-2.0) 5 | [![javadoc](https://javadoc.io/badge2/ai.test.sdk/test-ai-appium/javadoc.svg)](https://javadoc.io/doc/ai.test.sdk/test-ai-appium) 6 | [![Discord](https://img.shields.io/discord/853669216880295946?&logo=discord)](https://sdk.test.ai/discord) 7 | 8 | The test.ai Appium SDK is a simple library that makes it easy to write robust cross-platform mobile application tests backed by computer vision and artificial intelligence. 9 | 10 | test.ai integrates seamelessly with your existing tests, and will act as backup if your selectors break/fail by attempting to visually (computer vision) identify elements. This ability can also be leveraged to to write a single test suite that works on both iOS & Android. 11 | 12 | The test.ai SDK is able to accomplish this by automatically ingesting your Appium elements (using both screenshots and element names) when you run your test cases with test.ai for the first time. 13 | 14 | The SDK is accompanied by a [web-based editor](https://sdk.test.ai/) which makes building visual test cases easy; you can draw boxes around your elements instead of using fragile CSS or XPath selectors. 15 | 16 | ## Install 17 | 18 | Add the following line(s) to the dependencies section in your 19 | 20 | **pom.xml (Maven)** 21 | ```xml 22 | 23 | ai.test.sdk 24 | test-ai-appium 25 | 0.1.0 26 | 27 | ```` 28 | 29 | **build.gradle (Gradle)** 30 | ```groovy 31 | implementation 'ai.test.sdk:test-ai-appium:0.1.0' 32 | ``` 33 | 34 | ## Tutorial 35 | We have a detailed step-by-step tutorial which will help you get set up with the SDK: https://github.com/testdotai/java-appium-sdk-demo 36 | 37 | ## Resources 38 | * [Register/Login to your test.ai account](https://sdk.test.ai/login) 39 | * [API Docs](https://www.javadoc.io/doc/ai.test.sdk/test-ai-appium) 40 | * [Another Tutorial](https://sdk.test.ai/tutorial) -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.google.gson.JsonObject; 7 | import com.google.gson.JsonParser; 8 | 9 | import okhttp3.Response; 10 | 11 | /** 12 | * Shared utility methods for common tasks 13 | * 14 | * @author Alexander Wu (alec@test.ai) 15 | * 16 | */ 17 | final class JsonUtils 18 | { 19 | /** 20 | * The logger for this class 21 | */ 22 | private static Logger log = LoggerFactory.getLogger(JsonUtils.class); 23 | 24 | /** 25 | * Convenience method, extract the body of a {@code Response} as a {@code JsonObject}. 26 | * 27 | * @param r The Response object to use 28 | * @return The body of {@code r} as a {@code JsonObject}. 29 | */ 30 | public static JsonObject responseAsJson(Response r) 31 | { 32 | try 33 | { 34 | String body = r.body().string(); 35 | log.debug("Status: {} ----- Body: {}", r.code(), body); 36 | 37 | return JsonParser.parseString(body).getAsJsonObject(); 38 | } 39 | catch (Throwable e) 40 | { 41 | e.printStackTrace(); 42 | return null; 43 | } 44 | } 45 | 46 | /** 47 | * Convenience method, extract a String value associated with the specified key on a JsonObject. 48 | * 49 | * @param jo The JsonObject to extract a String from 50 | * @param key The key associated with the value to extract 51 | * @return The value associated with {@code key}, or the empty String if {@code key} was not in {@code jo}. 52 | */ 53 | public static String stringFromJson(JsonObject jo, String key) 54 | { 55 | return jo.has(key) ? jo.get(key).getAsString() : ""; 56 | } 57 | 58 | /** 59 | * Convenience method, extract a double value associated with the specified key on a JsonObject. 60 | * 61 | * @param jo The JsonObject to extract a double from 62 | * @param key The key associated with the value to extract 63 | * @return The value associated with {@code key}, or 0.0 if {@code key} was not in {@code jo}. 64 | */ 65 | public static double doubleFromJson(JsonObject jo, String key) 66 | { 67 | return jo.has(key) ? jo.get(key).getAsDouble() : 0; 68 | } 69 | 70 | /** 71 | * Convenience method, extract an int value associated with the specified key on a JsonObject. 72 | * 73 | * @param jo The JsonObject to extract an int from 74 | * @param key The key associated with the value to extract 75 | * @return The value associated with {@code key}, or 0 if {@code key} was not in {@code jo}. 76 | */ 77 | public static int intFromJson(JsonObject jo, String key) 78 | { 79 | return jo.has(key) ? jo.get(key).getAsInt() : 0; 80 | } 81 | 82 | /** 83 | * Convenience method, extract a boolean value associated with the specified key on a JsonObject. 84 | * 85 | * @param jo The JsonObject to extract a boolean from 86 | * @param key The key associated with the value to extract 87 | * @return The value associated with {@code key}, or false if {@code key} was not in {@code jo}. 88 | */ 89 | public static boolean booleanFromJson(JsonObject jo, String key) 90 | { 91 | return jo.has(key) ? jo.get(key).getAsBoolean() : false; 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/NetUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.io.IOException; 4 | import java.security.SecureRandom; 5 | import java.security.cert.CertificateException; 6 | import java.security.cert.X509Certificate; 7 | import java.time.Duration; 8 | import java.util.HashMap; 9 | 10 | import javax.net.ssl.HostnameVerifier; 11 | import javax.net.ssl.SSLContext; 12 | import javax.net.ssl.SSLSession; 13 | import javax.net.ssl.TrustManager; 14 | import javax.net.ssl.X509TrustManager; 15 | 16 | import okhttp3.FormBody; 17 | import okhttp3.HttpUrl; 18 | import okhttp3.OkHttpClient; 19 | import okhttp3.Request; 20 | import okhttp3.Response; 21 | 22 | /** 23 | * Shared network/http-related utilities and functionality 24 | * 25 | * @author Alexander Wu (alec@test.ai) 26 | * 27 | */ 28 | final class NetUtils 29 | { 30 | /** 31 | * Performs a simple form POST to the specified url with the provided client and form data. 32 | * 33 | * @param client The OkHTTP client to use 34 | * @param baseURL The base URL to target 35 | * @param endpoint The endpoint on the baseURL to target. 36 | * @param form The form data to POST 37 | * @return The response from the server, in the form of a {@code Response} object 38 | * @throws IOException Network error 39 | */ 40 | public static Response basicPOST(OkHttpClient client, HttpUrl baseURL, String endpoint, HashMap form) throws IOException 41 | { 42 | FormBody.Builder fb = new FormBody.Builder(); 43 | form.forEach(fb::add); 44 | 45 | return client.newCall(new Request.Builder().url(baseURL.newBuilder().addPathSegment(endpoint).build()).post(fb.build()).build()).execute(); 46 | } 47 | 48 | /** 49 | * Convenience method, creates a new OkHttpBuilder with timeouts configured. 50 | * 51 | * @return A OkHttpClient builder with reasonable timeouts configured. 52 | */ 53 | static OkHttpClient.Builder basicClient() 54 | { 55 | Duration d = Duration.ofSeconds(30); 56 | return new OkHttpClient.Builder().connectTimeout(d).writeTimeout(d).readTimeout(d).callTimeout(d); 57 | 58 | } 59 | 60 | /** 61 | * Creates a new {@code OkHttpClient} which ignores expired/invalid ssl certificates. Normally, OkHttp will raise an exception if it encounters bad certificates. 62 | * 63 | * @return A new {@code OkHttpClient} which ignores expired/invalid ssl certificates. 64 | */ 65 | public static OkHttpClient unsafeClient() 66 | { 67 | try 68 | { 69 | TrustManager tl[] = { new TrustAllX509Manager() }; 70 | 71 | SSLContext sslContext = SSLContext.getInstance("SSL"); 72 | sslContext.init(null, tl, new SecureRandom()); 73 | 74 | return basicClient().sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) tl[0]).hostnameVerifier(new TrustAllHostnameVerifier()).build(); 75 | } 76 | catch (Throwable e) // highly unlikely, shut up compiler 77 | { 78 | return null; 79 | } 80 | } 81 | 82 | /** 83 | * A dummy {@code HostnameVerifier} which doesn't actually do any hostname checking. 84 | * 85 | * @author Alexander Wu (alec@test.ai) 86 | * 87 | */ 88 | private static class TrustAllHostnameVerifier implements HostnameVerifier 89 | { 90 | @Override 91 | public boolean verify(String hostname, SSLSession session) 92 | { 93 | return true; 94 | } 95 | } 96 | 97 | /** 98 | * A dummy {@code X509TrustManager} which doesn't actually do any certificate verification. 99 | * 100 | * @author Alexander Wu (alec@test.ai) 101 | * 102 | */ 103 | private static class TrustAllX509Manager implements X509TrustManager 104 | { 105 | @Override 106 | public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException 107 | { 108 | } 109 | 110 | @Override 111 | public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException 112 | { 113 | } 114 | 115 | @Override 116 | public X509Certificate[] getAcceptedIssuers() 117 | { 118 | return new X509Certificate[0]; 119 | } 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/TestAiElement.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import com.google.gson.JsonObject; 4 | 5 | import io.appium.java_client.AppiumDriver; 6 | import io.appium.java_client.MobileElement; 7 | import io.appium.java_client.TouchAction; 8 | import io.appium.java_client.touch.offset.PointOption; 9 | 10 | import org.openqa.selenium.Dimension; 11 | import org.openqa.selenium.Point; 12 | import org.openqa.selenium.Rectangle; 13 | import org.openqa.selenium.WebElement; 14 | import org.openqa.selenium.interactions.Actions; 15 | 16 | /** 17 | * An enhanced RemoteWebElement which uses the results of the Test.ai classifier for improved accuracy. 18 | * 19 | * @author Alexander Wu (alec@test.ai) 20 | * 21 | */ 22 | @SuppressWarnings("unchecked") 23 | public class TestAiElement extends MobileElement 24 | { 25 | /** 26 | * The webdriver the user is using. We wrap this for when the user calls methods that interact with appium. 27 | */ 28 | @SuppressWarnings("rawtypes") 29 | private AppiumDriver driver; 30 | 31 | /** 32 | * The text in this element, as determined by test.ai's classifier 33 | */ 34 | private String text; 35 | 36 | /** 37 | * The size of this element, in pixels 38 | */ 39 | private Dimension size; 40 | 41 | /** 42 | * The location of this element, in pixels (offset from the upper left corner of the screen) 43 | */ 44 | private Point location; 45 | 46 | /** 47 | * The rectangle that can be drawn around this element. Basically combines size and location. 48 | */ 49 | private Rectangle rectangle; 50 | 51 | /** 52 | * The tag name of this element, as determined by test.ai's classifier 53 | */ 54 | private String tagName; 55 | 56 | /** 57 | * Coordinates for clicking/taping this element. 58 | * 59 | * @see #click() 60 | */ 61 | private int cX, cY; 62 | 63 | /** 64 | * Constructor, creates a new TestAiElement 65 | * 66 | * @param elem The element data returned by the FD API, as JSON 67 | * @param driver The driver the user is using to interact with their app 68 | * @param multiplier The screen density multiplier to use 69 | */ 70 | TestAiElement(JsonObject elem, @SuppressWarnings("rawtypes") AppiumDriver driver, double multiplier) 71 | { 72 | this.driver = driver; 73 | setParent(driver); 74 | 75 | text = JsonUtils.stringFromJson(elem, "text"); 76 | size = new Dimension(JsonUtils.intFromJson(elem, "width") / (int) multiplier, JsonUtils.intFromJson(elem, "height") / (int) multiplier); 77 | 78 | location = new Point(JsonUtils.intFromJson(elem, "x") / (int) multiplier, JsonUtils.intFromJson(elem, "y") / (int) multiplier); 79 | 80 | // this.property = property //TODO: not referenced/implemented on python side?? 81 | rectangle = new Rectangle(location, size); 82 | tagName = JsonUtils.stringFromJson(elem, "class"); 83 | 84 | cX = location.x / (int) multiplier + size.width / (int) multiplier / 2; 85 | cY = location.y / (int) multiplier + size.height / (int) multiplier / 2; 86 | 87 | } 88 | 89 | @Override 90 | public String getText() 91 | { 92 | return text; 93 | } 94 | 95 | @Override 96 | public Dimension getSize() 97 | { 98 | return size; 99 | } 100 | 101 | @Override 102 | public Point getLocation() 103 | { 104 | return location; 105 | } 106 | 107 | @Override 108 | public Rectangle getRect() 109 | { 110 | return rectangle; 111 | } 112 | 113 | @Override 114 | public String getTagName() 115 | { 116 | return tagName; 117 | } 118 | 119 | @Override 120 | @SuppressWarnings("rawtypes") 121 | public void click() 122 | { 123 | new TouchAction(driver).tap(PointOption.point(new Point(cX, cY))).perform(); 124 | } 125 | 126 | @Override 127 | public void sendKeys(CharSequence... keysToSend) 128 | { 129 | sendKeys(String.join("", keysToSend), true); 130 | } 131 | 132 | /** 133 | * Attempts to type the specified String ({@code value}) into this element. 134 | * 135 | * @param value The String to type into this element. 136 | * @param clickFirst Set {@code true} to tap this element (e.g. to focus it) first before sending keys. 137 | */ 138 | public void sendKeys(String value, boolean clickFirst) 139 | { 140 | if (clickFirst) 141 | click(); 142 | 143 | new Actions(driver).sendKeys(value).perform(); 144 | } 145 | 146 | @Override 147 | public void submit() 148 | { 149 | sendKeys("\n", false); 150 | } 151 | 152 | @Override 153 | public String getAttribute(String name) 154 | { 155 | return null; 156 | } 157 | 158 | @Override 159 | public String getCssValue(String propertyName) 160 | { 161 | throw new UnsupportedOperationException(); 162 | } 163 | 164 | @Override 165 | public boolean isDisplayed() 166 | { 167 | throw new UnsupportedOperationException(); 168 | } 169 | 170 | @Override 171 | public boolean isEnabled() 172 | { 173 | throw new UnsupportedOperationException(); 174 | } 175 | 176 | @Override 177 | public boolean isSelected() 178 | { 179 | throw new UnsupportedOperationException(); 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 10 | 11 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 14 | 15 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 16 | 17 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 18 | 19 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 20 | 21 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 22 | 23 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 24 | 25 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 26 | 27 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 28 | 29 | 2. Grant of Copyright License. 30 | 31 | Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 32 | 33 | 3. Grant of Patent License. 34 | 35 | Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 36 | 37 | 4. Redistribution. 38 | 39 | You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 40 | 41 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 42 | You must cause any modified files to carry prominent notices stating that You changed the files; and 43 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 44 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 45 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 46 | 47 | 5. Submission of Contributions. 48 | 49 | Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 50 | 51 | 6. Trademarks. 52 | 53 | This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 54 | 55 | 7. Disclaimer of Warranty. 56 | 57 | Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 58 | 59 | 8. Limitation of Liability. 60 | 61 | In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 62 | 63 | 9. Accepting Warranty or Additional Liability. 64 | 65 | While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 66 | 67 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/TestAiDriver.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.io.IOException; 4 | import java.net.URL; 5 | import java.util.Collection; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Objects; 10 | import java.util.Set; 11 | import java.util.UUID; 12 | import java.util.concurrent.TimeUnit; 13 | import java.util.function.Function; 14 | import java.util.logging.Level; 15 | 16 | import javax.imageio.ImageIO; 17 | 18 | import org.openqa.selenium.By; 19 | import org.openqa.selenium.Capabilities; 20 | import org.openqa.selenium.DeviceRotation; 21 | import org.openqa.selenium.NoSuchElementException; 22 | import org.openqa.selenium.OutputType; 23 | import org.openqa.selenium.Rectangle; 24 | import org.openqa.selenium.ScreenOrientation; 25 | import org.openqa.selenium.WebDriver; 26 | import org.openqa.selenium.WebDriver.Navigation; 27 | import org.openqa.selenium.WebDriver.Options; 28 | import org.openqa.selenium.WebDriver.TargetLocator; 29 | import org.openqa.selenium.WebElement; 30 | import org.openqa.selenium.html5.Location; 31 | import org.openqa.selenium.interactions.Keyboard; 32 | import org.openqa.selenium.interactions.Mouse; 33 | import org.openqa.selenium.interactions.Sequence; 34 | import org.openqa.selenium.remote.CommandExecutor; 35 | import org.openqa.selenium.remote.ErrorHandler; 36 | import org.openqa.selenium.remote.ExecuteMethod; 37 | import org.openqa.selenium.remote.FileDetector; 38 | import org.openqa.selenium.remote.SessionId; 39 | import org.slf4j.Logger; 40 | import org.slf4j.LoggerFactory; 41 | 42 | import com.google.gson.JsonObject; 43 | 44 | import io.appium.java_client.AppiumDriver; 45 | import io.appium.java_client.MobileElement; 46 | import okhttp3.HttpUrl; 47 | import okhttp3.OkHttpClient; 48 | import okhttp3.Response; 49 | 50 | /** 51 | * A convenient wrapper around {@code AppiumDriver} which calls out to Test.ai to improve the accuracy of identified elements. 52 | * 53 | * @author Alexander Wu (alec@test.ai) 54 | * 55 | * @param The element type to return, must be a subclass of MobileElement. 56 | */ 57 | @SuppressWarnings({ "unchecked", "deprecation" }) 58 | public class TestAiDriver // extends AppiumDriver 59 | { 60 | /** 61 | * The logger for this class 62 | */ 63 | private static Logger log = LoggerFactory.getLogger(TestAiDriver.class); 64 | 65 | /** 66 | * The client to use for making http requests 67 | */ 68 | private OkHttpClient client; 69 | 70 | /** 71 | * The driver used by the user that we're wrapping. 72 | */ 73 | private AppiumDriver driver; 74 | 75 | /** 76 | * The user's fluffy dragon API key 77 | */ 78 | private String apiKey; 79 | 80 | /** 81 | * The base URL of the target server (e.g. {@code https://sdk.test.ai}) 82 | */ 83 | private HttpUrl serverURL; 84 | 85 | /** 86 | * The test case name. Used in live/interactive mode. 87 | */ 88 | private String testCaseName; 89 | 90 | /** 91 | * Indicates whether Test.ai should be used to improve the accuracy of returned elements 92 | */ 93 | // private boolean train; 94 | 95 | /** 96 | * The run id. This should be randomly generated each run. 97 | */ 98 | private String runID = UUID.randomUUID().toString(); 99 | 100 | /** 101 | * The UUID of the last screenshot in live/interactive mode. 102 | */ 103 | // private String lastTestCaseScreenshotUUID; 104 | 105 | /** 106 | * The screen density multiplier 107 | */ 108 | private double multiplier; 109 | 110 | /** 111 | * Constructor, creates a new TestAiDriver. 112 | * 113 | * @param driver The AppiumDriver to wrap 114 | * @param apiKey Your API key, acquired from sdk.test.ai. 115 | * @param serverURL The server URL. Set {@code null} to use the default of sdk.test.ai. 116 | * @param testCaseName The test case name to use for interactive mode. Setting this to something other than {@code null} enables interactive mode. 117 | * @param train Set `true` to enable training for each encountered element. 118 | * @throws IOException If there was an initialization error. 119 | */ 120 | public TestAiDriver(AppiumDriver driver, String apiKey, String serverURL, String testCaseName, boolean train) throws IOException 121 | { 122 | // super(driver.getCapabilities()); 123 | 124 | this.driver = driver; 125 | this.apiKey = apiKey; 126 | this.testCaseName = testCaseName; 127 | // this.train = train; 128 | 129 | this.serverURL = HttpUrl.parse(serverURL != null ? serverURL : Objects.requireNonNullElse(System.getenv("TESTAI_FLUFFY_DRAGON_URL"), "https://sdk.test.ai")); 130 | client = this.serverURL.equals(HttpUrl.parse("https://sdk.dev.test.ai")) ? NetUtils.unsafeClient() : NetUtils.basicClient().build(); 131 | 132 | multiplier = 1.0 * ImageIO.read(driver.getScreenshotAs(OutputType.FILE)).getWidth() / driver.manage().window().getSize().width; 133 | 134 | log.debug("The screen multiplier is {}", multiplier); 135 | } 136 | 137 | /** 138 | * Constructor, creates a new TestAiDriver with the default server url (sdk.test.ai), non-interactive mode, and with training enabled. 139 | * 140 | * @param driver The AppiumDriver to wrap 141 | * @param apiKey Your API key, acquired from sdk.test.ai. 142 | * @throws IOException If there was an initialization error. 143 | */ 144 | public TestAiDriver(AppiumDriver driver, String apiKey) throws IOException 145 | { 146 | this(driver, apiKey, null, null, true); 147 | } 148 | 149 | /** 150 | * Convenience method, implicitly wait for the specified amount of time. 151 | * 152 | * @param waitTime The number of seconds to implicitly wait. 153 | * @return This {@code TestAiDriver}, for chaining convenience. 154 | */ 155 | public TestAiDriver implicitlyWait(long waitTime) 156 | { 157 | driver.manage().timeouts().implicitlyWait(waitTime, TimeUnit.SECONDS); 158 | return this; 159 | } 160 | 161 | public WebDriver context(String name) 162 | { 163 | return driver.context(name); 164 | } 165 | 166 | public org.openqa.selenium.remote.Response execute(String command) 167 | { 168 | return driver.execute(command); 169 | } 170 | 171 | public org.openqa.selenium.remote.Response execute(String command, Map parameters) 172 | { 173 | return driver.execute(command, parameters); 174 | } 175 | 176 | public T findElement(By locator) 177 | { 178 | return driver.findElement(locator); 179 | } 180 | 181 | public T findElement(String by, String using) 182 | { 183 | return driver.findElement(by, using); 184 | } 185 | 186 | public String getContext() 187 | { 188 | return driver.getContext(); 189 | } 190 | 191 | public Set getContextHandles() 192 | { 193 | return driver.getContextHandles(); 194 | } 195 | 196 | public ExecuteMethod getExecuteMethod() 197 | { 198 | return driver.getExecuteMethod(); 199 | } 200 | 201 | public ScreenOrientation getOrientation() 202 | { 203 | return driver.getOrientation(); 204 | } 205 | 206 | public URL getRemoteAddress() 207 | { 208 | return driver.getRemoteAddress(); 209 | } 210 | 211 | public Map getStatus() 212 | { 213 | return driver.getStatus(); 214 | } 215 | 216 | public boolean isBrowser() 217 | { 218 | return driver.isBrowser(); 219 | } 220 | 221 | public Location location() 222 | { 223 | return driver.location(); 224 | } 225 | 226 | public void rotate(DeviceRotation rotation) 227 | { 228 | driver.rotate(rotation); 229 | } 230 | 231 | public void rotate(ScreenOrientation orientation) 232 | { 233 | driver.rotate(orientation); 234 | } 235 | 236 | public DeviceRotation rotation() 237 | { 238 | return driver.rotation(); 239 | } 240 | 241 | public void setLocation(Location location) 242 | { 243 | driver.setLocation(location); 244 | } 245 | 246 | public String toString() 247 | { 248 | return driver.toString(); 249 | } 250 | 251 | public Object executeAsyncScript(String script, Object... args) 252 | { 253 | return driver.executeAsyncScript(script, args); 254 | } 255 | 256 | public Object executeScript(String script, Object... args) 257 | { 258 | return driver.executeScript(script, args); 259 | } 260 | 261 | public Capabilities getCapabilities() 262 | { 263 | return driver.getCapabilities(); 264 | } 265 | 266 | public CommandExecutor getCommandExecutor() 267 | { 268 | return driver.getCommandExecutor(); 269 | } 270 | 271 | public String getCurrentUrl() 272 | { 273 | return driver.getCurrentUrl(); 274 | } 275 | 276 | public ErrorHandler getErrorHandler() 277 | { 278 | return driver.getErrorHandler(); 279 | } 280 | 281 | public FileDetector getFileDetector() 282 | { 283 | return driver.getFileDetector(); 284 | } 285 | 286 | public Keyboard getKeyboard() 287 | { 288 | return driver.getKeyboard(); 289 | } 290 | 291 | public Mouse getMouse() 292 | { 293 | return driver.getMouse(); 294 | } 295 | 296 | public String getPageSource() 297 | { 298 | return driver.getPageSource(); 299 | } 300 | 301 | public X getScreenshotAs(OutputType outputType) 302 | { 303 | return driver.getScreenshotAs(outputType); 304 | } 305 | 306 | public SessionId getSessionId() 307 | { 308 | return driver.getSessionId(); 309 | } 310 | 311 | public String getTitle() 312 | { 313 | return driver.getTitle(); 314 | } 315 | 316 | public String getWindowHandle() 317 | { 318 | return driver.getWindowHandle(); 319 | } 320 | 321 | public Set getWindowHandles() 322 | { 323 | return driver.getWindowHandles(); 324 | } 325 | 326 | public Options manage() 327 | { 328 | return driver.manage(); 329 | } 330 | 331 | public Navigation navigate() 332 | { 333 | return driver.navigate(); 334 | } 335 | 336 | public void perform(Collection actions) 337 | { 338 | driver.perform(actions); 339 | } 340 | 341 | public void quit() 342 | { 343 | driver.quit(); 344 | } 345 | 346 | public void resetInputState() 347 | { 348 | driver.resetInputState(); 349 | } 350 | 351 | public void setErrorHandler(ErrorHandler handler) 352 | { 353 | driver.setErrorHandler(handler); 354 | } 355 | 356 | public void setFileDetector(FileDetector detector) 357 | { 358 | driver.setFileDetector(detector); 359 | } 360 | 361 | public void setLogLevel(Level level) 362 | { 363 | driver.setLogLevel(level); 364 | } 365 | 366 | public TargetLocator switchTo() 367 | { 368 | return driver.switchTo(); 369 | } 370 | 371 | /** 372 | * Attempts to find an element by accessibility id. 373 | * 374 | * @param using The accessibility id of the element to find 375 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 376 | * @return The element that was found. Raises an exception otherwise. 377 | */ 378 | public T findElementByAccessibilityId(String using, String elementName) 379 | { 380 | return findElementByGeneric(using, elementName, "accessibility_id", driver::findElementByAccessibilityId); 381 | } 382 | 383 | /** 384 | * Attempts to find an element by accessibility id. 385 | * 386 | * @param using The accessibility id of the element to find 387 | * @return The element that was found. Raises an exception otherwise. 388 | */ 389 | public T findElementByAccessibilityId(String using) 390 | { 391 | return findElementByAccessibilityId(using, null); 392 | } 393 | 394 | /** 395 | * Attempts to find all elements with the matching accessibility id. 396 | * 397 | * @param using The accessibility id of the elements to find. 398 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 399 | */ 400 | public List findElementsByAccessibilityId(String using) 401 | { 402 | return driver.findElementsByAccessibilityId(using); 403 | } 404 | 405 | /** 406 | * Attempts to find an element by class name. 407 | * 408 | * @param using The class name of the element to find 409 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 410 | * @return The element that was found. Raises an exception otherwise. 411 | */ 412 | public T findElementByClassName(String using, String elementName) 413 | { 414 | return findElementByGeneric(using, elementName, "class_name", driver::findElementByClassName); 415 | } 416 | 417 | /** 418 | * Attempts to find an element by class name. 419 | * 420 | * @param using The class name of the element to find 421 | * @return The element that was found. Raises an exception otherwise. 422 | */ 423 | public T findElementByClassName(String using) 424 | { 425 | return findElementByClassName(using, null); 426 | } 427 | 428 | /** 429 | * Attempts to find all elements with the matching class name. 430 | * 431 | * @param using The class name of the elements to find. 432 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 433 | */ 434 | public List findElementsByClassName(String using) 435 | { 436 | return driver.findElementsByClassName(using); 437 | } 438 | 439 | /** 440 | * Attempts to find an element by css selector. 441 | * 442 | * @param using The css selector of the element to find 443 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 444 | * @return The element that was found. Raises an exception otherwise. 445 | */ 446 | public T findElementByCssSelector(String using, String elementName) 447 | { 448 | return findElementByGeneric(using, elementName, "class_name", driver::findElementByCssSelector); 449 | } 450 | 451 | /** 452 | * Attempts to find an element by css selector. 453 | * 454 | * @param using The css selector of the element to find 455 | * @return The element that was found. Raises an exception otherwise. 456 | */ 457 | public T findElementByCssSelector(String using) 458 | { 459 | return findElementByCssSelector(using, null); 460 | } 461 | 462 | /** 463 | * Attempts to find all elements with the matching css selector. 464 | * 465 | * @param using The css selector of the elements to find. 466 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 467 | */ 468 | public List findElementsByCssSelector(String using) 469 | { 470 | return driver.findElementsByCssSelector(using); 471 | } 472 | 473 | /** 474 | * Attempts to find an element by id. 475 | * 476 | * @param using The id of the element to find 477 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 478 | * @return The element that was found. Raises an exception otherwise. 479 | */ 480 | public T findElementById(String using, String elementName) 481 | { 482 | return findElementByGeneric(using, elementName, "class_name", driver::findElementById); 483 | } 484 | 485 | /** 486 | * Attempts to find an element by id. 487 | * 488 | * @param using The id of the element to find 489 | * @return The element that was found. Raises an exception otherwise. 490 | */ 491 | public T findElementById(String using) 492 | { 493 | return findElementById(using, null); 494 | } 495 | 496 | /** 497 | * Attempts to find all elements with the matching id. 498 | * 499 | * @param using The id of the elements to find. 500 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 501 | */ 502 | public List findElementsById(String using) 503 | { 504 | return driver.findElementsById(using); 505 | } 506 | 507 | /** 508 | * Attempts to find an element by link text. 509 | * 510 | * @param using The link text of the element to find 511 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 512 | * @return The element that was found. Raises an exception otherwise. 513 | */ 514 | public T findElementByLinkText(String using, String elementName) 515 | { 516 | return findElementByGeneric(using, elementName, "class_name", driver::findElementByLinkText); 517 | } 518 | 519 | /** 520 | * Attempts to find an element by link text. 521 | * 522 | * @param using The link text of the element to find 523 | * @return The element that was found. Raises an exception otherwise. 524 | */ 525 | public T findElementByLinkText(String using) 526 | { 527 | return findElementByLinkText(using, null); 528 | } 529 | 530 | /** 531 | * Attempts to find all elements with the matching link text. 532 | * 533 | * @param using The link text of the elements to find. 534 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 535 | */ 536 | public List findElementsByLinkText(String using) 537 | { 538 | return driver.findElementsByLinkText(using); 539 | } 540 | 541 | /** 542 | * Attempts to find an element by name. 543 | * 544 | * @param using The name of the element to find 545 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 546 | * @return The element that was found. Raises an exception otherwise. 547 | */ 548 | public T findElementByName(String using, String elementName) 549 | { 550 | return findElementByGeneric(using, elementName, "name", driver::findElementByName); 551 | } 552 | 553 | /** 554 | * Attempts to find an element by name. 555 | * 556 | * @param using The name of the element to find 557 | * @return The element that was found. Raises an exception otherwise. 558 | */ 559 | public T findElementByName(String using) 560 | { 561 | return findElementByName(using, null); 562 | } 563 | 564 | /** 565 | * Attempts to find all elements with the matching name. 566 | * 567 | * @param using The name of the elements to find. 568 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 569 | */ 570 | public List findElementsByName(String using) 571 | { 572 | return driver.findElementsByName(using); 573 | } 574 | 575 | /** 576 | * Attempts to find an element by partial link text. 577 | * 578 | * @param using The partial link text of the element to find 579 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 580 | * @return The element that was found. Raises an exception otherwise. 581 | */ 582 | public T findElementByPartialLinkText(String using, String elementName) 583 | { 584 | return findElementByGeneric(using, elementName, "name", driver::findElementByPartialLinkText); 585 | } 586 | 587 | /** 588 | * Attempts to find an element by partial link text. 589 | * 590 | * @param using The partial link text of the element to find 591 | * @return The element that was found. Raises an exception otherwise. 592 | */ 593 | public T findElementByPartialLinkText(String using) 594 | { 595 | return findElementByPartialLinkText(using, null); 596 | } 597 | 598 | /** 599 | * Attempts to find all elements with the matching partial link text. 600 | * 601 | * @param using The partial link text of the elements to find. 602 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 603 | */ 604 | public List findElementsByPartialLinkText(String using) 605 | { 606 | return driver.findElementsByPartialLinkText(using); 607 | } 608 | 609 | /** 610 | * Attempts to find an element by tag name. 611 | * 612 | * @param using The tag name of the element to find 613 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 614 | * @return The element that was found. Raises an exception otherwise. 615 | */ 616 | public T findElementByTagName(String using, String elementName) 617 | { 618 | return findElementByGeneric(using, elementName, "name", driver::findElementByTagName); 619 | } 620 | 621 | /** 622 | * Attempts to find an element by tag name. 623 | * 624 | * @param using The tag name of the element to find 625 | * @return The element that was found. Raises an exception otherwise. 626 | */ 627 | public T findElementByTagName(String using) 628 | { 629 | return findElementByTagName(using, null); 630 | } 631 | 632 | /** 633 | * Attempts to find all elements with the matching tag name. 634 | * 635 | * @param using The tag name of the elements to find. 636 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 637 | */ 638 | public List findElementsByTagName(String using) 639 | { 640 | return driver.findElementsByTagName(using); 641 | } 642 | 643 | /** 644 | * Attempts to find an element by xpath. 645 | * 646 | * @param using The xpath of the element to find 647 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 648 | * @return The element that was found. Raises an exception otherwise. 649 | */ 650 | public T findElementByXPath(String using, String elementName) 651 | { 652 | return findElementByGeneric(using, elementName, "xpath", driver::findElementByXPath); 653 | } 654 | 655 | /** 656 | * Attempts to find an element by xpath. 657 | * 658 | * @param using The xpath of the element to find 659 | * @return The element that was found. Raises an exception otherwise. 660 | */ 661 | public T findElementByXPath(String using) 662 | { 663 | return findElementByXPath(using, null); 664 | } 665 | 666 | /** 667 | * Attempts to find all elements with the matching xpath. 668 | * 669 | * @param using The xpath of the elements to find. 670 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 671 | */ 672 | public List findElementsByXPath(String using) 673 | { 674 | return driver.findElementsByXPath(using); 675 | } 676 | 677 | /** 678 | * Finds an element by {@code elementName}. 679 | * 680 | * @param elementName The label name of the element to be classified. 681 | * @return An element associated with {@code elementName}. Throws NoSuchElementException otherwise. 682 | */ 683 | public MobileElement findByElementName(String elementName) 684 | { 685 | ClassifyResult r = classify(elementName); 686 | if (r.e == null) 687 | throw new NoSuchElementException(r.msg); 688 | 689 | return r.e; 690 | } 691 | 692 | /** 693 | * Shared {@code findElementBy} functionality. This serves as the base logic for most find by methods exposed to the end user. 694 | * 695 | * @param using The search term to use when looking for an element. 696 | * @param elementName The label name of the element to be classified. This is what the element will be stored under in the test.ai db. 697 | * @param shortcode The short identifier for the type of lookup being performed. This will be used to aut-generate an {@code elementName} if the user did not specify one. 698 | * @param fn The appium function to call with {@code using}, which will be used to fetch what Appium thinks is the target element. 699 | * @return The TestAiElement 700 | */ 701 | private T findElementByGeneric(String using, String elementName, String shortcode, Function fn) 702 | { 703 | if (elementName == null) 704 | elementName = String.format("element_name_by_%s_%s", shortcode, using.replace('.', '_')); 705 | 706 | elementName = elementName.replace(' ', '_'); 707 | 708 | try 709 | { 710 | T driverElement = fn.apply(using); 711 | if (driverElement != null) 712 | { 713 | ClassifyResult result = classify(elementName); 714 | updateElement(driverElement, result.key, elementName, true); 715 | } 716 | 717 | return driverElement; 718 | } 719 | catch (Throwable x) 720 | { 721 | log.info("Element '{}' was not found by Appium, trying with test.ai...", elementName); 722 | 723 | ClassifyResult result = classify(elementName); 724 | if (result.e != null) 725 | return (T) result.e; 726 | 727 | log.error("test.ai was also unable to find the element with name '{}'", elementName); 728 | 729 | throw x; 730 | } 731 | } 732 | 733 | /** 734 | * Updates the entry for an element as it is known to the test.ai servers. 735 | * 736 | * @param elem The element to update 737 | * @param key The key associated with this element 738 | * @param elementName The name associated with this element 739 | * @param trainIfNecessary Set {@code true} if the model on the server should also be trained with this element. 740 | */ 741 | private void updateElement(T elem, String key, String elementName, boolean trainIfNecessary) 742 | { 743 | Rectangle rect = ((MobileElement) elem).getRect(); 744 | HashMap form = CollectionUtils.keyValuesToHM("key", key, "api_key", apiKey, "run_id", runID, "x", Integer.toString(rect.x), "y", Integer.toString(rect.y), "width", 745 | Integer.toString(rect.width), "height", Integer.toString(rect.height), "multiplier", Double.toString(multiplier), "train_if_necessary", Boolean.toString(trainIfNecessary)); 746 | 747 | try (Response r = NetUtils.basicPOST(client, serverURL, "add_action", form)) 748 | { 749 | } 750 | catch (Throwable e) 751 | { 752 | e.printStackTrace(); 753 | } 754 | } 755 | 756 | /** 757 | * Perform additional classification on an element by querying the test.ai server. 758 | * 759 | * @param elementName The name of the element to run classification on. 760 | * @return The result of the classification. 761 | */ 762 | private ClassifyResult classify(String elementName) 763 | { 764 | if (testCaseName != null) 765 | return null; // TODO: add test case creation/interactive mode 766 | 767 | String pageSource = "", msg = "test.ai driver exception", key = null; 768 | try 769 | { 770 | pageSource = driver.getPageSource(); 771 | } 772 | catch (Throwable e) 773 | { 774 | 775 | } 776 | 777 | try 778 | { 779 | String screenshotBase64 = driver.getScreenshotAs(OutputType.BASE64); 780 | // Files.write(Paths.get("/tmp/scnshot.png"), Base64.getMimeDecoder().decode(screenshotBase64)); 781 | 782 | JsonObject r = JsonUtils.responseAsJson(NetUtils.basicPOST(client, serverURL, "classify", 783 | CollectionUtils.keyValuesToHM("screenshot", screenshotBase64, "source", pageSource, "api_key", apiKey, "label", elementName, "run_id", runID))); 784 | 785 | key = JsonUtils.stringFromJson(r, "key"); 786 | 787 | if (JsonUtils.booleanFromJson(r, "success")) 788 | { 789 | log.info("Successfully classified: {}", elementName); 790 | return new ClassifyResult(new TestAiElement(r.get("elem").getAsJsonObject(), driver, multiplier), key); 791 | } 792 | 793 | String rawMsg = JsonUtils.stringFromJson(r, "message"); 794 | if (rawMsg != null) 795 | { 796 | String cFailedBase = "Classification failed for element_name: "; 797 | 798 | if (rawMsg.contains("Please label") || rawMsg.contains("Did not find")) 799 | msg = String.format("%s%s - Please visit %s/label/%s to classify", cFailedBase, elementName, serverURL, elementName); 800 | else if (rawMsg.contains("frozen label")) 801 | msg = String.format("%s%s - However this element is frozen, so no new screenshot was uploaded. Please unfreeze the element if you want to add this screenshot to training", cFailedBase, 802 | elementName); 803 | else 804 | msg = String.format("%s: Unknown error, here was the API response: %s", msg, r); 805 | } 806 | } 807 | catch (Throwable e) 808 | { 809 | e.printStackTrace(); 810 | } 811 | 812 | log.warn(msg); 813 | return new ClassifyResult(null, key, msg); 814 | } 815 | 816 | /** 817 | * Simple container for encapsulating results of calls to {@code classify()}. 818 | * 819 | * @author Alexander Wu (alec@test.ai) 820 | * 821 | */ 822 | private static class ClassifyResult 823 | { 824 | /** 825 | * The TestAiElement created by the call to classify 826 | */ 827 | public TestAiElement e; 828 | 829 | /** 830 | * The key returned by the call to classify 831 | */ 832 | public String key; 833 | 834 | /** 835 | * The message associated with this result 836 | */ 837 | public String msg; 838 | 839 | /** 840 | * Constructor, creates a new ClassifyResult. 841 | * 842 | * @param e The TestAiElement to to use 843 | * @param key The key to use 844 | * @param msg The message to associate with this result 845 | */ 846 | ClassifyResult(TestAiElement e, String key, String msg) 847 | { 848 | this.e = e; 849 | this.key = key; 850 | this.msg = msg; 851 | } 852 | 853 | /** 854 | * Constructor, creates a new ClassifyResult, where the {@code msg} is set to the empty String by default. 855 | * 856 | * @param e 857 | * @param key 858 | */ 859 | ClassifyResult(TestAiElement e, String key) 860 | { 861 | this(e, key, ""); 862 | } 863 | } 864 | } 865 | --------------------------------------------------------------------------------