├── .gitignore ├── LICENSE.md ├── README.md ├── build.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src ├── main └── java │ └── ai │ └── test │ └── sdk │ ├── CollectionUtils.java │ ├── JsonUtils.java │ ├── MatchUtils.java │ ├── NetUtils.java │ ├── TestAiDriver.java │ ├── TestAiElement.java │ └── package-info.java └── test └── java └── ai └── test └── LibraryTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | build/ 3 | 4 | # OS Generated files 5 | .DS_Store 6 | ._* 7 | .Spotlight-V100 8 | .Trashes 9 | Icon? 10 | ehthumbs.db 11 | Thumbs.db 12 | 13 | # Eclipse 14 | .project 15 | .classpath 16 | .settings 17 | 18 | # Gradle 19 | .gradle/ 20 | /bin/ 21 | 22 | Scratch*.java -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 10 | 11 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 14 | 15 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 16 | 17 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 18 | 19 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 20 | 21 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 22 | 23 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 24 | 25 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 26 | 27 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 28 | 29 | 2. Grant of Copyright License. 30 | 31 | Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 32 | 33 | 3. Grant of Patent License. 34 | 35 | Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 36 | 37 | 4. Redistribution. 38 | 39 | You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 40 | 41 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 42 | You must cause any modified files to carry prominent notices stating that You changed the files; and 43 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 44 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 45 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 46 | 47 | 5. Submission of Contributions. 48 | 49 | Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 50 | 51 | 6. Trademarks. 52 | 53 | This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 54 | 55 | 7. Disclaimer of Warranty. 56 | 57 | Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 58 | 59 | 8. Limitation of Liability. 60 | 61 | In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 62 | 63 | 9. Accepting Warranty or Additional Liability. 64 | 65 | While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 66 | 67 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![test.ai sdk logo](https://testdotai.github.io/static-assets/shared/logo-sdk.png)](https://test.ai/sdk) 2 | 3 | [![JDK-11+](https://img.shields.io/badge/JDK-11%2B-blue)](https://adoptium.net) 4 | [![javadoc](https://javadoc.io/badge2/ai.test.sdk/test-ai-selenium/javadoc.svg)](https://javadoc.io/doc/ai.test.sdk/test-ai-selenium) 5 | [![Maven Central](https://img.shields.io/maven-central/v/ai.test.sdk/test-ai-selenium)](https://search.maven.org/artifact/ai.test.sdk/test-ai-selenium) 6 | [![Apache 2.0](https://img.shields.io/badge/Apache-2.0-blue)](https://www.apache.org/licenses/LICENSE-2.0) 7 | [![Discord](https://img.shields.io/discord/853669216880295946?&logo=discord)](https://sdk.test.ai/discord) 8 | [![Twitter](https://img.shields.io/twitter/follow/testdotai)](https://twitter.com/testdotai) 9 | 10 | The test.ai selenium SDK is a simple library that makes it easy to write robust cross-browser web tests backed by computer vision and artificial intelligence. 11 | 12 | test.ai integrates seamelessly with your existing tests, and will act as backup if your selectors break/fail by attempting to visually (computer vision) identify elements. 13 | 14 | The test.ai SDK is able to accomplish this by automatically ingesting your selenium elements (using both screenshots and element names) when you run your test cases with test.ai for the first time. 15 | 16 | The SDK is accompanied by a [web-based editor](https://sdk.test.ai/) which makes building visual test cases easy; you can draw boxes around your elements instead of using fragile CSS or XPath selectors. 17 | 18 | ## Install 19 | 20 | Add the following line(s) to the dependencies section in your 21 | 22 | **pom.xml (Maven)** 23 | ```xml 24 | 25 | ai.test.sdk 26 | test-ai-selenium 27 | 0.2.0 28 | 29 | ```` 30 | 31 | **build.gradle (Gradle)** 32 | ```groovy 33 | implementation 'ai.test.sdk:test-ai-selenium:0.2.0' 34 | ``` 35 | 36 | ## Tutorial 37 | We have a detailed step-by-step tutorial which will help you get set up with the SDK: https://github.com/testdotai/java-selenium-sdk-demo 38 | 39 | ## Resources 40 | * [Register/Login to your test.ai account](https://sdk.test.ai/login) 41 | * [API Docs](https://www.javadoc.io/doc/ai.test.sdk/test-ai-selenium) 42 | * [Another Tutorial](https://sdk.test.ai/tutorial) -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | id 'eclipse' 4 | id 'maven-publish' 5 | id 'signing' 6 | } 7 | 8 | 9 | description="${project.name} build script" 10 | group="ai.test.sdk" 11 | version="0.2.0" 12 | 13 | 14 | repositories { 15 | mavenLocal() 16 | mavenCentral() 17 | } 18 | 19 | dependencies { 20 | api 'com.google.code.gson:gson:2.9.0' 21 | api 'com.squareup.okhttp3:okhttp:4.9.3' 22 | api 'org.seleniumhq.selenium:selenium-java:3.141.59' 23 | api 'org.slf4j:slf4j-api:1.7.36' 24 | 25 | testImplementation 'org.junit.jupiter:junit-jupiter:5.7.2' 26 | } 27 | 28 | 29 | java { 30 | sourceCompatibility = JavaVersion.VERSION_11 31 | targetCompatibility = JavaVersion.VERSION_11 32 | 33 | withJavadocJar() 34 | withSourcesJar() 35 | } 36 | 37 | javadoc { 38 | source = sourceSets.main.allJava 39 | classpath = configurations.compileClasspath 40 | 41 | options { 42 | //setOutputLevel JavadocOutputLevel.VERBOSE 43 | setMemberLevel JavadocMemberLevel.PUBLIC 44 | setAuthor true 45 | 46 | jFlags '-Dhttp.agent=gradle-javadoc' 47 | 48 | links "https://docs.oracle.com/en/java/javase/11/docs/api/" 49 | links "https://www.selenium.dev/selenium/docs/api/java/" 50 | } 51 | } 52 | 53 | publishing { 54 | publications { 55 | mavenJava(MavenPublication) { 56 | from components.java 57 | 58 | pom { 59 | name = 'test-ai-selenium' 60 | description = 'Testing with selenium enhanced by Test.ai' 61 | inceptionYear = '2021' 62 | url = 'https://github.com/testdotai/java-selenium-sdk' 63 | licenses { 64 | license { 65 | name = 'Apache License, Version 2.0' 66 | url = 'https://www.apache.org/licenses/LICENSE-2.0' 67 | } 68 | } 69 | developers { 70 | developer { 71 | id = 'test.ai' 72 | name = 'Appdiff, Inc.' 73 | email = 'sdk@test.ai' 74 | } 75 | } 76 | scm { 77 | connection = 'scm:git:https://github.com/testdotai/java-selenium-sdk.git' 78 | developerConnection = 'scm:git:ssh://github.com/testdotai/java-selenium-sdk.git' 79 | url = 'https://github.com/testdotai/java-selenium-sdk' 80 | } 81 | } 82 | } 83 | } 84 | 85 | repositories { 86 | maven { 87 | url = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/" 88 | 89 | credentials { 90 | username project.hasProperty("ossrhUsername") ? project.properties["ossrhUsername"] : System.getenv('ossrhUsername') 91 | password project.hasProperty("ossrhPassword") ? project.properties["ossrhPassword"] : System.getenv('ossrhPassword') 92 | } 93 | } 94 | } 95 | } 96 | 97 | 98 | signing { 99 | sign publishing.publications.mavenJava 100 | } 101 | 102 | 103 | tasks.named('test') { 104 | useJUnitPlatform() 105 | } 106 | 107 | wrapper { 108 | gradleVersion = '7.4.1' 109 | } -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testdotai/java-selenium-sdk/74c3f2cd77699b6c3eb12ccc2b942411acc862ca/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'test-ai-selenium' -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/CollectionUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.util.HashMap; 4 | 5 | import com.google.gson.JsonObject; 6 | 7 | /** 8 | * Shared classes and methods enhancing collections functionality. 9 | * 10 | * @author Alexander Wu (alec@test.ai) 11 | * 12 | */ 13 | final class CollectionUtils 14 | { 15 | /** 16 | * Builds a {@code HashMap} out of a list of {@code String}s. Pass in values such that {@code [ k1, v1, k2, v2, k3, v3... ]}. 17 | * 18 | * @param sl The {@code String}s to use 19 | * @return A {@code HashMap} derived from the values in {@code sl} 20 | */ 21 | public static HashMap keyValuesToHM(String... sl) 22 | { 23 | HashMap m = new HashMap<>(); 24 | 25 | for (int i = 0; i < sl.length; i += 2) 26 | m.put(sl[i], sl[i + 1]); 27 | 28 | return m; 29 | } 30 | 31 | /** 32 | * Builds a new {@code JsonObject} from the list of Objects. Pass in values such that {@code [ k1, v1, k2, v2, k3, v3... ]}. 33 | * 34 | * @param ol The {@code Object}s to use 35 | * @return A {@code JsonObject} derived from the values in {@code ol} 36 | */ 37 | public static JsonObject keyValuesToJO(Object... ol) 38 | { 39 | JsonObject jo = new JsonObject(); 40 | 41 | for (int i = 0; i < ol.length; i += 2) 42 | { 43 | String k = (String) ol[i]; 44 | Object v = ol[i + 1]; 45 | 46 | if (v instanceof String) 47 | jo.addProperty(k, (String) v); 48 | else if (v instanceof Number) 49 | jo.addProperty(k, (Number) v); 50 | else if (v instanceof Boolean) 51 | jo.addProperty(k, (Boolean) v); 52 | else if (v instanceof Character) 53 | jo.addProperty(k, (Character) v); 54 | else 55 | throw new IllegalArgumentException(String.format("'%s' is not an acceptable type for JSON!", v)); 56 | } 57 | 58 | return jo; 59 | } 60 | 61 | /** 62 | * Simple Tuple implementation. A Tuple is an immutable two-pair of values. It may consist of any two Objects, which may or may not be in of the same type. 63 | * 64 | * @author Alexander Wu (alec@test.ai) 65 | * 66 | * @param The type of Object allowed for the first Object in the tuple. 67 | * @param The type of Object allowed for the second Object in the tuple. 68 | */ 69 | public static class Tuple 70 | { 71 | /** 72 | * The k value of the tuple 73 | */ 74 | public final K k; 75 | 76 | /** 77 | * The v value of the tuple 78 | */ 79 | public final V v; 80 | 81 | /** 82 | * Constructor, creates a new Tuple from the specified values. 83 | * 84 | * @param k The first entry in the Tuple. 85 | * @param v The second entry in the Tuple. 86 | */ 87 | public Tuple(K k, V v) 88 | { 89 | this.k = k; 90 | this.v = v; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.google.gson.JsonObject; 7 | import com.google.gson.JsonParser; 8 | 9 | import okhttp3.Response; 10 | 11 | /** 12 | * Shared utility methods for common tasks 13 | * 14 | * @author Alexander Wu (alec@test.ai) 15 | * 16 | */ 17 | final class JsonUtils 18 | { 19 | /** 20 | * The logger for this class 21 | */ 22 | private static Logger log = LoggerFactory.getLogger(JsonUtils.class); 23 | 24 | /** 25 | * Convenience method, extract the body of a {@code Response} as a {@code JsonObject}. 26 | * 27 | * @param r The Response object to use 28 | * @return The body of {@code r} as a {@code JsonObject}. 29 | */ 30 | public static JsonObject responseAsJson(Response r) 31 | { 32 | try 33 | { 34 | String body = r.body().string(); 35 | log.debug("Status: {} ----- Body: {}", r.code(), body); 36 | 37 | return JsonParser.parseString(body).getAsJsonObject(); 38 | } 39 | catch (Throwable e) 40 | { 41 | e.printStackTrace(); 42 | return null; 43 | } 44 | } 45 | 46 | /** 47 | * Convenience method, extract a String value associated with the specified key on a JsonObject. 48 | * 49 | * @param jo The JsonObject to extract a String from 50 | * @param key The key associated with the value to extract 51 | * @return The value associated with {@code key}, or the empty String if {@code key} was not in {@code jo}. 52 | */ 53 | public static String stringFromJson(JsonObject jo, String key) 54 | { 55 | return jo.has(key) ? jo.get(key).getAsString() : ""; 56 | } 57 | 58 | /** 59 | * Convenience method, extract a double value associated with the specified key on a JsonObject. 60 | * 61 | * @param jo The JsonObject to extract a double from 62 | * @param key The key associated with the value to extract 63 | * @return The value associated with {@code key}, or 0.0 if {@code key} was not in {@code jo}. 64 | */ 65 | public static double doubleFromJson(JsonObject jo, String key) 66 | { 67 | return jo.has(key) ? jo.get(key).getAsDouble() : 0; 68 | } 69 | 70 | /** 71 | * Convenience method, extract an int value associated with the specified key on a JsonObject. 72 | * 73 | * @param jo The JsonObject to extract an int from 74 | * @param key The key associated with the value to extract 75 | * @return The value associated with {@code key}, or 0 if {@code key} was not in {@code jo}. 76 | */ 77 | public static int intFromJson(JsonObject jo, String key) 78 | { 79 | return jo.has(key) ? jo.get(key).getAsInt() : 0; 80 | } 81 | 82 | /** 83 | * Convenience method, extract a boolean value associated with the specified key on a JsonObject. 84 | * 85 | * @param jo The JsonObject to extract a boolean from 86 | * @param key The key associated with the value to extract 87 | * @return The value associated with {@code key}, or false if {@code key} was not in {@code jo}. 88 | */ 89 | public static boolean booleanFromJson(JsonObject jo, String key) 90 | { 91 | return jo.has(key) ? jo.get(key).getAsBoolean() : false; 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/MatchUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.stream.Collectors; 9 | 10 | import org.openqa.selenium.NoSuchElementException; 11 | import org.openqa.selenium.Rectangle; 12 | import org.openqa.selenium.StaleElementReferenceException; 13 | import org.openqa.selenium.WebElement; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import com.google.gson.JsonObject; 18 | 19 | import ai.test.sdk.CollectionUtils.Tuple; 20 | 21 | /** 22 | * Static methods for matching bounding boxes to underlying Selenium elements. 23 | * 24 | * @author Alexander Wu (alec@test.ai) 25 | * 26 | */ 27 | class MatchUtils 28 | { 29 | /** 30 | * The logger for this class 31 | */ 32 | private static Logger log = LoggerFactory.getLogger(MatchUtils.class); 33 | 34 | /** 35 | * Matches a bounding box returned by the test.ai API to a selenium WebElement on the current page. 36 | * 37 | * @param boundingBox The json representing the element returned by the test.ai API. 38 | * @param driver The {@code TestAiDriver} to use 39 | * @return The best-matching, underlying {@code WebElement} which best fits the parameters specified by {@code boudingBox} 40 | */ 41 | public static WebElement matchBoundingBoxToSeleniumElement(JsonObject boundingBox, TestAiDriver driver) 42 | { 43 | HashMap newBox = new HashMap<>(); 44 | newBox.put("x", boundingBox.get("x").getAsDouble() / driver.multiplier); 45 | newBox.put("y", boundingBox.get("y").getAsDouble() / driver.multiplier); 46 | newBox.put("width", boundingBox.get("width").getAsDouble() / driver.multiplier); 47 | newBox.put("height", boundingBox.get("height").getAsDouble() / driver.multiplier); 48 | 49 | List elements = driver.driver.findElementsByXPath("//*"); 50 | List iouScores = new ArrayList<>(); 51 | 52 | for (WebElement e : elements) 53 | try 54 | { 55 | iouScores.add(iouBoxes(newBox, e.getRect())); 56 | } 57 | catch (StaleElementReferenceException x) 58 | { 59 | log.debug("Stale reference to element '{}', setting score of 0", e); 60 | iouScores.add(0.0); 61 | } 62 | 63 | List> composite = new ArrayList<>(); 64 | for (int i = 0; i < iouScores.size(); i++) 65 | composite.add(new Tuple<>(iouScores.get(i), elements.get(i))); 66 | 67 | Collections.sort(composite, (o1, o2) -> o2.k.compareTo(o1.k)); // sort the composite values in reverse (descending) order 68 | composite = composite.stream().filter(x -> x.k > 0).filter(x -> centerHit(newBox, x.v.getRect())).collect(Collectors.toList()); 69 | 70 | if (composite.size() == 0) 71 | throw new NoSuchElementException("Could not find any web element under the center of the bounding box"); 72 | 73 | for (Tuple t : composite) 74 | if (t.v.getTagName().equals("input") || t.v.getTagName().equals(("button")) && t.k > composite.get(0).k * 0.9) 75 | return t.v; 76 | 77 | return composite.get(0).v; 78 | } 79 | 80 | /** 81 | * Calculate the IOU score of two rectangles. This is derived from the overlap and areas of both rectangles. 82 | * 83 | * @param box1 The first box The first rectangle to check (the json returned from the test.ai API) 84 | * @param box2 The second box The second rectangle to check (the Rectangle from the selenium WebElement) 85 | * @return The IOU score of the two rectangles. Higher score means relative to other scores (obtained from comparisons between other pairs of rectangles) means better match. 86 | */ 87 | private static double iouBoxes(Map box1, Rectangle box2) 88 | { 89 | return iou(box1.get("x"), box1.get("y"), box1.get("width"), box1.get("height"), (double) box2.x, (double) box2.y, (double) box2.width, (double) box2.height); 90 | } 91 | 92 | /** 93 | * Calculate the IOU score of two rectangles. This is derived from the overlap and areas of both rectangles. 94 | * 95 | * @param x The x coordinate of the first box (upper left corner) 96 | * @param y The y coordinate of the first box (upper left corner) 97 | * @param w The width of the first box 98 | * @param h The height of the first box 99 | * @param xx The x coordinate of the second box (upper left corner) 100 | * @param yy The y coordinate of the second box (upper left corner) 101 | * @param ww The width of the second box 102 | * @param hh The height of the second box 103 | * @return The IOU value of both boxes. 104 | */ 105 | private static double iou(double x, double y, double w, double h, double xx, double yy, double ww, double hh) 106 | { 107 | double overlap = areaOverlap(x, y, w, h, xx, yy, ww, hh); 108 | return overlap / (area(w, h) + area(ww, hh) - overlap); 109 | } 110 | 111 | /** 112 | * Determines the amount of area overlap between two rectangles 113 | * 114 | * @param x The x coordinate of the first box (upper left corner) 115 | * @param y The y coordinate of the first box (upper left corner) 116 | * @param w The width of the first box 117 | * @param h The height of the first box 118 | * @param xx The x coordinate of the second box (upper left corner) 119 | * @param yy The y coordinate of the second box (upper left corner) 120 | * @param ww The width of the second box 121 | * @param hh The height of the second box 122 | * @return The amount of overlap, in square pixels. 123 | */ 124 | private static double areaOverlap(double x, double y, double w, double h, double xx, double yy, double ww, double hh) 125 | { 126 | double dx = Math.min(x + w, xx + ww) - Math.max(x, xx), dy = Math.min(y + h, yy + hh) - Math.max(y, yy); 127 | return dx >= 0 && dy >= 0 ? dx * dy : 0; 128 | } 129 | 130 | /** 131 | * Convenience function, calculates the area of a rectangle 132 | * 133 | * @param w The width of the rectangle 134 | * @param h The height of the rectangle 135 | * @return The area of the rectangle 136 | */ 137 | private static double area(double w, double h) 138 | { 139 | return w * h; 140 | } 141 | 142 | /** 143 | * Determines if center point of {@code box1} falls within the area of {@code box2} 144 | * 145 | * @param box1 The first rectangle to check (the json returned from the test.ai API) 146 | * @param box2 The second rectangle to check (the Rectangle from the selenium WebElement) 147 | * @return {@code true} if the center point of {@code box1} falls within the area of {@code box2} 148 | */ 149 | private static boolean centerHit(Map box1, Rectangle box2) 150 | { 151 | double centerX = box1.get("x") + box1.get("width") / 2, centerY = box1.get("y") + box1.get("height") / 2; 152 | return centerX > box2.x && centerX < box2.x + box2.width && centerY > box2.y && centerY < box2.y + box2.height; 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/NetUtils.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.io.IOException; 4 | import java.security.SecureRandom; 5 | import java.security.cert.CertificateException; 6 | import java.security.cert.X509Certificate; 7 | import java.time.Duration; 8 | import java.util.HashMap; 9 | 10 | import javax.net.ssl.HostnameVerifier; 11 | import javax.net.ssl.SSLContext; 12 | import javax.net.ssl.SSLSession; 13 | import javax.net.ssl.TrustManager; 14 | import javax.net.ssl.X509TrustManager; 15 | 16 | import com.google.gson.JsonObject; 17 | 18 | import okhttp3.FormBody; 19 | import okhttp3.HttpUrl; 20 | import okhttp3.MediaType; 21 | import okhttp3.OkHttpClient; 22 | import okhttp3.Request; 23 | import okhttp3.RequestBody; 24 | import okhttp3.Response; 25 | 26 | /** 27 | * Shared network/http-related utilities and functionality 28 | * 29 | * @author Alexander Wu (alec@test.ai) 30 | * 31 | */ 32 | final class NetUtils 33 | { 34 | /** 35 | * The {@code MediaType} representing the json MIME type. 36 | */ 37 | private static final MediaType JSON = MediaType.get("application/json; charset=utf-8"); 38 | 39 | /** 40 | * Performs a simple POST to the specified url with the provided client and {@code RequestBody}. 41 | * 42 | * @param client The OkHttp client to use 43 | * @param baseURL The base URL to target 44 | * @param endpoint The endpoint on the baseURL to target. 45 | * @param b The request body to POST. 46 | * @return The response from the server, in the form of a {@code Response} object 47 | * @throws IOException Network error 48 | */ 49 | private static Response basicPOST(OkHttpClient client, HttpUrl baseURL, String endpoint, RequestBody b) throws IOException 50 | { 51 | return client.newCall(new Request.Builder().url(baseURL.newBuilder().addPathSegment(endpoint).build()).post(b).build()).execute(); 52 | } 53 | 54 | /** 55 | * Performs a simple POST to the specified url with the provided client and json data. 56 | * 57 | * @param client The OkHttp client to use 58 | * @param baseURL The base URL to target 59 | * @param endpoint The endpoint on the baseURL to target. 60 | * @param jo The JsonObject to put in the request body 61 | * @return The response from the server, in the form of a {@code Response} object 62 | * @throws IOException Network error 63 | */ 64 | public static Response basicPOST(OkHttpClient client, HttpUrl baseURL, String endpoint, JsonObject jo) throws IOException 65 | { 66 | return basicPOST(client, baseURL, endpoint, RequestBody.create(jo.toString(), JSON)); 67 | } 68 | 69 | /** 70 | * Performs a simple form POST to the specified url with the provided client and form data. 71 | * 72 | * @param client The OkHttp client to use 73 | * @param baseURL The base URL to target 74 | * @param endpoint The endpoint on the baseURL to target. 75 | * @param form The form data to POST 76 | * @return The response from the server, in the form of a {@code Response} object 77 | * @throws IOException Network error 78 | */ 79 | public static Response basicPOST(OkHttpClient client, HttpUrl baseURL, String endpoint, HashMap form) throws IOException 80 | { 81 | FormBody.Builder fb = new FormBody.Builder(); 82 | form.forEach(fb::add); 83 | 84 | return basicPOST(client, baseURL, endpoint, fb.build()); 85 | } 86 | 87 | /** 88 | * Convenience method, creates a new OkHttpBuilder with timeouts configured. 89 | * 90 | * @return A OkHttpClient builder with reasonable timeouts configured. 91 | */ 92 | static OkHttpClient.Builder basicClient() 93 | { 94 | Duration d = Duration.ofSeconds(60); 95 | return new OkHttpClient.Builder().connectTimeout(d).writeTimeout(d).readTimeout(d).callTimeout(d); 96 | } 97 | 98 | /** 99 | * Creates a new {@code OkHttpClient} which ignores expired/invalid ssl certificates. Normally, OkHttp will raise an exception if it encounters bad certificates. 100 | * 101 | * @return A new {@code OkHttpClient} which ignores expired/invalid ssl certificates. 102 | */ 103 | public static OkHttpClient unsafeClient() 104 | { 105 | try 106 | { 107 | TrustManager tl[] = { new TrustAllX509Manager() }; 108 | 109 | SSLContext sslContext = SSLContext.getInstance("SSL"); 110 | sslContext.init(null, tl, new SecureRandom()); 111 | 112 | return basicClient().sslSocketFactory(sslContext.getSocketFactory(), (X509TrustManager) tl[0]).hostnameVerifier(new TrustAllHostnameVerifier()).build(); 113 | } 114 | catch (Throwable e) // highly unlikely, shut up compiler 115 | { 116 | return null; 117 | } 118 | } 119 | 120 | /** 121 | * A dummy {@code HostnameVerifier} which doesn't actually do any hostname checking. 122 | * 123 | * @author Alexander Wu (alec@test.ai) 124 | * 125 | */ 126 | private static class TrustAllHostnameVerifier implements HostnameVerifier 127 | { 128 | @Override 129 | public boolean verify(String hostname, SSLSession session) 130 | { 131 | return true; 132 | } 133 | } 134 | 135 | /** 136 | * A dummy {@code X509TrustManager} which doesn't actually do any certificate verification. 137 | * 138 | * @author Alexander Wu (alec@test.ai) 139 | * 140 | */ 141 | private static class TrustAllX509Manager implements X509TrustManager 142 | { 143 | @Override 144 | public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException 145 | { 146 | } 147 | 148 | @Override 149 | public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException 150 | { 151 | } 152 | 153 | @Override 154 | public X509Certificate[] getAcceptedIssuers() 155 | { 156 | return new X509Certificate[0]; 157 | } 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/TestAiDriver.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import java.io.IOException; 4 | import java.util.Collection; 5 | import java.util.List; 6 | import java.util.Objects; 7 | import java.util.Set; 8 | import java.util.UUID; 9 | import java.util.concurrent.TimeUnit; 10 | import java.util.function.Function; 11 | import java.util.logging.Level; 12 | 13 | import javax.imageio.ImageIO; 14 | 15 | import org.openqa.selenium.By; 16 | import org.openqa.selenium.Capabilities; 17 | import org.openqa.selenium.NoSuchElementException; 18 | import org.openqa.selenium.OutputType; 19 | import org.openqa.selenium.Rectangle; 20 | import org.openqa.selenium.WebElement; 21 | import org.openqa.selenium.interactions.Keyboard; 22 | import org.openqa.selenium.interactions.Mouse; 23 | import org.openqa.selenium.interactions.Sequence; 24 | import org.openqa.selenium.remote.CommandExecutor; 25 | import org.openqa.selenium.remote.ErrorHandler; 26 | import org.openqa.selenium.remote.FileDetector; 27 | import org.openqa.selenium.remote.RemoteWebDriver; 28 | import org.openqa.selenium.remote.SessionId; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import com.google.gson.JsonObject; 33 | 34 | import okhttp3.HttpUrl; 35 | import okhttp3.OkHttpClient; 36 | import okhttp3.Response; 37 | 38 | /** 39 | * A convenient wrapper around {@code RemoteWebDriver} which calls out to Test.ai to improve the accuracy of identified elements. 40 | * 41 | * @author Alexander Wu (alec@test.ai) 42 | */ 43 | @SuppressWarnings("deprecation") 44 | public class TestAiDriver extends RemoteWebDriver 45 | { 46 | /** 47 | * The current version of the SDK 48 | */ 49 | private static String SDK_VERSION = "0.2.0"; 50 | 51 | /** 52 | * The logger for this class 53 | */ 54 | private static Logger log = LoggerFactory.getLogger(TestAiDriver.class); 55 | 56 | /** 57 | * The client to use for making http requests 58 | */ 59 | private OkHttpClient client; 60 | 61 | /** 62 | * The driver used by the user that we're wrapping. 63 | */ 64 | RemoteWebDriver driver; 65 | 66 | /** 67 | * The user's fluffy dragon API key 68 | */ 69 | private String apiKey; 70 | 71 | /** 72 | * The base URL of the target server (e.g. {@code https://sdk.test.ai}) 73 | */ 74 | private HttpUrl serverURL; 75 | 76 | /** 77 | * The test case name. Used in live/interactive mode. 78 | */ 79 | private String testCaseName; 80 | 81 | /** 82 | * Indicates whether Test.ai should be used to improve the accuracy of returned elements 83 | */ 84 | // private boolean train; 85 | 86 | /** 87 | * The run id. This should be randomly generated each run. 88 | */ 89 | private String runID = UUID.randomUUID().toString(); 90 | 91 | /** 92 | * The UUID of the last screenshot in live/interactive mode. 93 | */ 94 | // private String lastTestCaseScreenshotUUID; 95 | 96 | /** 97 | * The screen density multiplier 98 | */ 99 | double multiplier; 100 | 101 | /** 102 | * Constructor, creates a new TestAiDriver. 103 | * 104 | * @param driver The selenium driver to wrap 105 | * @param apiKey Your API key, acquired from sdk.test.ai. 106 | * @param serverURL The server URL. Set {@code null} to use the default of sdk.test.ai. 107 | * @param testCaseName The test case name to use for interactive mode. Setting this to something other than {@code null} enables interactive mode. 108 | * @param train Set `true` to enable training for each encountered element. 109 | * @throws IOException If there was an initialization error. 110 | */ 111 | public TestAiDriver(RemoteWebDriver driver, String apiKey, String serverURL, String testCaseName, boolean train) throws IOException 112 | { 113 | this.driver = driver; 114 | this.apiKey = apiKey; 115 | this.testCaseName = testCaseName; 116 | // this.train = train; 117 | 118 | if (testCaseName == null) 119 | { 120 | StackTraceElement[] sl = Thread.currentThread().getStackTrace(); 121 | if (sl.length > 0) 122 | { 123 | StackTraceElement bottom = sl[sl.length - 1]; 124 | this.testCaseName = String.format("%s.%s", bottom.getClassName(), bottom.getMethodName()); 125 | 126 | log.info("No test case name was specified, defaulting to {}", this.testCaseName); 127 | } 128 | else 129 | this.testCaseName = "My first test case"; 130 | } 131 | 132 | this.serverURL = HttpUrl.parse(serverURL != null ? serverURL : Objects.requireNonNullElse(System.getenv("TESTAI_FLUFFY_DRAGON_URL"), "https://sdk.test.ai")); 133 | client = this.serverURL.equals(HttpUrl.parse("https://sdk.dev.test.ai")) ? NetUtils.unsafeClient() : NetUtils.basicClient().build(); 134 | multiplier = 1.0 * ImageIO.read(driver.getScreenshotAs(OutputType.FILE)).getWidth() / driver.manage().window().getSize().width; 135 | 136 | log.debug("The screen multiplier is {}", multiplier); 137 | 138 | try 139 | { 140 | JsonObject payload = CollectionUtils.keyValuesToJO("api_key", apiKey, "os", 141 | String.format("%s-%s-%s", System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch")), "sdk_version", SDK_VERSION, "language", 142 | String.format("java-%s", System.getProperty("java.version")), "test_case_uuid", runID); 143 | log.debug("Checking in with: {}", payload.toString()); 144 | 145 | JsonObject r = JsonUtils.responseAsJson(NetUtils.basicPOST(client, this.serverURL, "sdk_checkin", payload)); 146 | if (!JsonUtils.booleanFromJson(r, "success")) 147 | log.debug("Error during checkin, server said: {}", r); 148 | } 149 | catch (Throwable e) 150 | { 151 | log.debug("Checkin failed catastrophically: {}", e.getMessage()); 152 | } 153 | } 154 | 155 | /** 156 | * Constructor, creates a new TestAiDriver with the default server url (sdk.test.ai), non-interactive mode, and with training enabled. 157 | * 158 | * @param driver The {@code RemoteWebDriver} to wrap 159 | * @param apiKey Your API key, acquired from sdk.test.ai. 160 | * @throws IOException If there was an initialization error. 161 | */ 162 | public TestAiDriver(RemoteWebDriver driver, String apiKey) throws IOException 163 | { 164 | this(driver, apiKey, null, null, true); 165 | } 166 | 167 | /** 168 | * Convenience method, implicitly wait for the specified amount of time. 169 | * 170 | * @param waitTime The number of seconds to implicitly wait. 171 | * @return This {@code TestAiDriver}, for chaining convenience. 172 | */ 173 | public TestAiDriver implicitlyWait(long waitTime) 174 | { 175 | driver.manage().timeouts().implicitlyWait(waitTime, TimeUnit.SECONDS); 176 | return this; 177 | } 178 | 179 | @Override 180 | public Object executeAsyncScript(String script, Object... args) 181 | { 182 | return driver.executeAsyncScript(script, args); 183 | } 184 | 185 | @Override 186 | public Object executeScript(String script, Object... args) 187 | { 188 | return driver.executeScript(script, args); 189 | } 190 | 191 | /** 192 | * Opens a web browser and directs it to {@code url}. 193 | * 194 | * @param url The URL to launch the browser to. 195 | */ 196 | @Override 197 | public void get(String url) 198 | { 199 | driver.get(url); 200 | } 201 | 202 | @Override 203 | public WebElement findElement(By locator) 204 | { 205 | return driver.findElement(locator); 206 | } 207 | 208 | @Override 209 | public List findElements(By locator) 210 | { 211 | return driver.findElements(locator); 212 | } 213 | 214 | @Override 215 | public Capabilities getCapabilities() 216 | { 217 | return driver.getCapabilities(); 218 | } 219 | 220 | @Override 221 | public CommandExecutor getCommandExecutor() 222 | { 223 | return driver.getCommandExecutor(); 224 | } 225 | 226 | @Override 227 | public String getCurrentUrl() 228 | { 229 | return driver.getCurrentUrl(); 230 | } 231 | 232 | @Override 233 | public ErrorHandler getErrorHandler() 234 | { 235 | return driver.getErrorHandler(); 236 | } 237 | 238 | @Override 239 | public FileDetector getFileDetector() 240 | { 241 | return driver.getFileDetector(); 242 | } 243 | 244 | @Override 245 | public Keyboard getKeyboard() 246 | { 247 | return driver.getKeyboard(); 248 | } 249 | 250 | @Override 251 | public Mouse getMouse() 252 | { 253 | return driver.getMouse(); 254 | } 255 | 256 | @Override 257 | public String getPageSource() 258 | { 259 | return driver.getPageSource(); 260 | } 261 | 262 | @Override 263 | public X getScreenshotAs(OutputType outputType) 264 | { 265 | return driver.getScreenshotAs(outputType); 266 | } 267 | 268 | @Override 269 | public SessionId getSessionId() 270 | { 271 | return driver.getSessionId(); 272 | } 273 | 274 | @Override 275 | public String getTitle() 276 | { 277 | return driver.getTitle(); 278 | } 279 | 280 | @Override 281 | public String getWindowHandle() 282 | { 283 | return driver.getWindowHandle(); 284 | } 285 | 286 | @Override 287 | public Set getWindowHandles() 288 | { 289 | return driver.getWindowHandles(); 290 | } 291 | 292 | @Override 293 | public Options manage() 294 | { 295 | return driver.manage(); 296 | } 297 | 298 | @Override 299 | public Navigation navigate() 300 | { 301 | return driver.navigate(); 302 | } 303 | 304 | @Override 305 | public void perform(Collection actions) 306 | { 307 | driver.perform(actions); 308 | } 309 | 310 | @Override 311 | public void quit() 312 | { 313 | driver.quit(); 314 | } 315 | 316 | @Override 317 | public void resetInputState() 318 | { 319 | driver.resetInputState(); 320 | } 321 | 322 | @Override 323 | public void setErrorHandler(ErrorHandler handler) 324 | { 325 | driver.setErrorHandler(handler); 326 | } 327 | 328 | @Override 329 | public void setFileDetector(FileDetector detector) 330 | { 331 | driver.setFileDetector(detector); 332 | } 333 | 334 | @Override 335 | public void setLogLevel(Level level) 336 | { 337 | driver.setLogLevel(level); 338 | } 339 | 340 | @Override 341 | public TargetLocator switchTo() 342 | { 343 | return driver.switchTo(); 344 | } 345 | 346 | @Override 347 | public String toString() 348 | { 349 | return driver.toString(); 350 | } 351 | 352 | /** 353 | * Attempts to find an element by class name. 354 | * 355 | * @param using The class name of the element to find 356 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 357 | * @return The element that was found. Raises an exception otherwise. 358 | */ 359 | public WebElement findElementByClassName(String using, String elementName) 360 | { 361 | return findElementByGeneric(using, elementName, "class_name", driver::findElementByClassName); 362 | } 363 | 364 | /** 365 | * Attempts to find an element by class name. 366 | * 367 | * @param using The class name of the element to find 368 | * @return The element that was found. Raises an exception otherwise. 369 | */ 370 | @Override 371 | public WebElement findElementByClassName(String using) 372 | { 373 | return findElementByClassName(using, null); 374 | } 375 | 376 | /** 377 | * Attempts to find all elements with the matching class name. 378 | * 379 | * @param using The class name of the elements to find. 380 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 381 | */ 382 | @Override 383 | public List findElementsByClassName(String using) 384 | { 385 | return driver.findElementsByClassName(using); 386 | } 387 | 388 | /** 389 | * Attempts to find an element by css selector. 390 | * 391 | * @param using The css selector of the element to find 392 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 393 | * @return The element that was found. Raises an exception otherwise. 394 | */ 395 | public WebElement findElementByCssSelector(String using, String elementName) 396 | { 397 | return findElementByGeneric(using, elementName, "class_name", driver::findElementByCssSelector); 398 | } 399 | 400 | /** 401 | * Attempts to find an element by css selector. 402 | * 403 | * @param using The css selector of the element to find 404 | * @return The element that was found. Raises an exception otherwise. 405 | */ 406 | @Override 407 | public WebElement findElementByCssSelector(String using) 408 | { 409 | return findElementByCssSelector(using, null); 410 | } 411 | 412 | /** 413 | * Attempts to find all elements with the matching css selector. 414 | * 415 | * @param using The css selector of the elements to find. 416 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 417 | */ 418 | @Override 419 | public List findElementsByCssSelector(String using) 420 | { 421 | return driver.findElementsByCssSelector(using); 422 | } 423 | 424 | /** 425 | * Attempts to find an element by id. 426 | * 427 | * @param using The id of the element to find 428 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 429 | * @return The element that was found. Raises an exception otherwise. 430 | */ 431 | public WebElement findElementById(String using, String elementName) 432 | { 433 | return findElementByGeneric(using, elementName, "class_name", driver::findElementById); 434 | } 435 | 436 | /** 437 | * Attempts to find an element by id. 438 | * 439 | * @param using The id of the element to find 440 | * @return The element that was found. Raises an exception otherwise. 441 | */ 442 | @Override 443 | public WebElement findElementById(String using) 444 | { 445 | return findElementById(using, null); 446 | } 447 | 448 | /** 449 | * Attempts to find all elements with the matching id. 450 | * 451 | * @param using The id of the elements to find. 452 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 453 | */ 454 | @Override 455 | public List findElementsById(String using) 456 | { 457 | return driver.findElementsById(using); 458 | } 459 | 460 | /** 461 | * Attempts to find an element by link text. 462 | * 463 | * @param using The link text of the element to find 464 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 465 | * @return The element that was found. Raises an exception otherwise. 466 | */ 467 | public WebElement findElementByLinkText(String using, String elementName) 468 | { 469 | return findElementByGeneric(using, elementName, "class_name", driver::findElementByLinkText); 470 | } 471 | 472 | /** 473 | * Attempts to find an element by link text. 474 | * 475 | * @param using The link text of the element to find 476 | * @return The element that was found. Raises an exception otherwise. 477 | */ 478 | @Override 479 | public WebElement findElementByLinkText(String using) 480 | { 481 | return findElementByLinkText(using, null); 482 | } 483 | 484 | /** 485 | * Attempts to find all elements with the matching link text. 486 | * 487 | * @param using The link text of the elements to find. 488 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 489 | */ 490 | @Override 491 | public List findElementsByLinkText(String using) 492 | { 493 | return driver.findElementsByLinkText(using); 494 | } 495 | 496 | /** 497 | * Attempts to find an element by name. 498 | * 499 | * @param using The name of the element to find 500 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 501 | * @return The element that was found. Raises an exception otherwise. 502 | */ 503 | public WebElement findElementByName(String using, String elementName) 504 | { 505 | return findElementByGeneric(using, elementName, "name", driver::findElementByName); 506 | } 507 | 508 | /** 509 | * Attempts to find an element by name. 510 | * 511 | * @param using The name of the element to find 512 | * @return The element that was found. Raises an exception otherwise. 513 | */ 514 | @Override 515 | public WebElement findElementByName(String using) 516 | { 517 | return findElementByName(using, null); 518 | } 519 | 520 | /** 521 | * Attempts to find all elements with the matching name. 522 | * 523 | * @param using The name of the elements to find. 524 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 525 | */ 526 | @Override 527 | public List findElementsByName(String using) 528 | { 529 | return driver.findElementsByName(using); 530 | } 531 | 532 | /** 533 | * Attempts to find an element by partial link text. 534 | * 535 | * @param using The partial link text of the element to find 536 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 537 | * @return The element that was found. Raises an exception otherwise. 538 | */ 539 | public WebElement findElementByPartialLinkText(String using, String elementName) 540 | { 541 | return findElementByGeneric(using, elementName, "name", driver::findElementByPartialLinkText); 542 | } 543 | 544 | /** 545 | * Attempts to find an element by partial link text. 546 | * 547 | * @param using The partial link text of the element to find 548 | * @return The element that was found. Raises an exception otherwise. 549 | */ 550 | @Override 551 | public WebElement findElementByPartialLinkText(String using) 552 | { 553 | return findElementByPartialLinkText(using, null); 554 | } 555 | 556 | /** 557 | * Attempts to find all elements with the matching partial link text. 558 | * 559 | * @param using The partial link text of the elements to find. 560 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 561 | */ 562 | @Override 563 | public List findElementsByPartialLinkText(String using) 564 | { 565 | return driver.findElementsByPartialLinkText(using); 566 | } 567 | 568 | /** 569 | * Attempts to find an element by tag name. 570 | * 571 | * @param using The tag name of the element to find 572 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 573 | * @return The element that was found. Raises an exception otherwise. 574 | */ 575 | public WebElement findElementByTagName(String using, String elementName) 576 | { 577 | return findElementByGeneric(using, elementName, "name", driver::findElementByTagName); 578 | } 579 | 580 | /** 581 | * Attempts to find an element by tag name. 582 | * 583 | * @param using The tag name of the element to find 584 | * @return The element that was found. Raises an exception otherwise. 585 | */ 586 | @Override 587 | public WebElement findElementByTagName(String using) 588 | { 589 | return findElementByTagName(using, null); 590 | } 591 | 592 | /** 593 | * Attempts to find all elements with the matching tag name. 594 | * 595 | * @param using The tag name of the elements to find. 596 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 597 | */ 598 | @Override 599 | public List findElementsByTagName(String using) 600 | { 601 | return driver.findElementsByTagName(using); 602 | } 603 | 604 | /** 605 | * Attempts to find an element by xpath. 606 | * 607 | * @param using The xpath of the element to find 608 | * @param elementName The label name of the element to be classified. Optional, set {@code null} to auto generate an element name. 609 | * @return The element that was found. Raises an exception otherwise. 610 | */ 611 | public WebElement findElementByXPath(String using, String elementName) 612 | { 613 | return findElementByGeneric(using, elementName, "xpath", driver::findElementByXPath); 614 | } 615 | 616 | /** 617 | * Attempts to find an element by xpath. 618 | * 619 | * @param using The xpath of the element to find 620 | * @return The element that was found. Raises an exception otherwise. 621 | */ 622 | @Override 623 | public WebElement findElementByXPath(String using) 624 | { 625 | return findElementByXPath(using, null); 626 | } 627 | 628 | /** 629 | * Attempts to find all elements with the matching xpath. 630 | * 631 | * @param using The xpath of the elements to find. 632 | * @return A {@code List} with any elements that were found, or an empty {@code List} if no matches were found. 633 | */ 634 | @Override 635 | public List findElementsByXPath(String using) 636 | { 637 | return driver.findElementsByXPath(using); 638 | } 639 | 640 | /** 641 | * Finds an element by {@code elementName}. Please use {@link #findElementByElementName(String)} instead. 642 | * 643 | * @param elementName The label name of the element to be classified. 644 | * @return An element associated with {@code elementName}. Throws NoSuchElementException otherwise. 645 | */ 646 | @Deprecated 647 | public WebElement findByElementName(String elementName) 648 | { 649 | return findElementByElementName(elementName); 650 | } 651 | 652 | /** 653 | * Finds an element by {@code elementName}. 654 | * 655 | * @param elementName The label name of the element to be classified. 656 | * @return An element associated with {@code elementName}. Throws NoSuchElementException otherwise. 657 | */ 658 | public WebElement findElementByElementName(String elementName) 659 | { 660 | ClassifyResult r = classify(elementName); 661 | if (r.e == null) 662 | throw new NoSuchElementException(r.msg); 663 | 664 | return r.e; 665 | } 666 | 667 | /** 668 | * Shared {@code findElementBy} functionality. This serves as the base logic for most find by methods exposed to the end user. 669 | * 670 | * @param using The search term to use when looking for an element. 671 | * @param elementName The label name of the element to be classified. This is what the element will be stored under in the test.ai db. 672 | * @param shortcode The short identifier for the type of lookup being performed. This will be used to aut-generate an {@code elementName} if the user did not specify one. 673 | * @param fn The selenium function to call with {@code using}, which will be used to fetch what selenium thinks is the target element. 674 | * @return The TestAiElement 675 | */ 676 | private WebElement findElementByGeneric(String using, String elementName, String shortcode, Function fn) 677 | { 678 | if (elementName == null) 679 | elementName = String.format("element_name_by_%s_%s", shortcode, using.replace('.', '_')); 680 | 681 | elementName = elementName.replace(' ', '_'); 682 | 683 | try 684 | { 685 | WebElement driverElement = fn.apply(using); 686 | if (driverElement != null) 687 | { 688 | ClassifyResult result = classify(elementName); 689 | updateElement(driverElement, result.key, elementName, true); 690 | } 691 | 692 | return driverElement; 693 | } 694 | catch (Throwable x) 695 | { 696 | log.info("Element '{}' was not found by Selenium, trying with test.ai...", elementName); 697 | 698 | ClassifyResult result = classify(elementName); 699 | if (result.e != null) 700 | return result.e; 701 | 702 | log.error("test.ai was also unable to find the element with name '{}'", elementName); 703 | 704 | throw x; 705 | } 706 | } 707 | 708 | /** 709 | * Updates the entry for an element as it is known to the test.ai servers. 710 | * 711 | * @param elem The element to update 712 | * @param key The key associated with this element 713 | * @param elementName The name associated with this element 714 | * @param trainIfNecessary Set {@code true} if the model on the server should also be trained with this element. 715 | */ 716 | private void updateElement(WebElement elem, String key, String elementName, boolean trainIfNecessary) 717 | { 718 | Rectangle rect = elem.getRect(); 719 | JsonObject form = CollectionUtils.keyValuesToJO("key", key, "api_key", apiKey, "label", elementName, "run_id", runID, "x", rect.x * multiplier, "y", rect.y * multiplier, "width", 720 | rect.width * multiplier, "height", rect.height * multiplier, "multiplier", multiplier, "train_if_necessary", trainIfNecessary, "test_case_uuid", testCaseName); 721 | 722 | try (Response r = NetUtils.basicPOST(client, serverURL, "add_action", form)) 723 | { 724 | log.debug("Updated element {}, response from the server was '{}'", elementName, r.body().string()); 725 | } 726 | catch (Throwable e) 727 | { 728 | e.printStackTrace(); 729 | } 730 | } 731 | 732 | /** 733 | * Perform additional classification on an element by querying the test.ai server. 734 | * 735 | * @param elementName The name of the element to run classification on. 736 | * @return The result of the classification. 737 | */ 738 | private ClassifyResult classify(String elementName) 739 | { 740 | // if (testCaseName != null) 741 | // return null; // TODO: add test case creation/interactive mode 742 | 743 | String pageSource = "", msg = "test.ai driver exception", key = null; 744 | try 745 | { 746 | pageSource = driver.getPageSource(); 747 | } 748 | catch (Throwable e) 749 | { 750 | 751 | } 752 | 753 | try 754 | { 755 | String screenshotBase64 = driver.getScreenshotAs(OutputType.BASE64); 756 | // Files.write(Paths.get("/tmp/scnshot.png"), Base64.getMimeDecoder().decode(screenshotBase64)); 757 | 758 | JsonObject r = JsonUtils.responseAsJson(NetUtils.basicPOST(client, serverURL, "classify", 759 | CollectionUtils.keyValuesToHM("screenshot", screenshotBase64, "source", pageSource, "api_key", apiKey, "label", elementName, "run_id", runID))); 760 | 761 | key = JsonUtils.stringFromJson(r, "key"); 762 | 763 | if (JsonUtils.booleanFromJson(r, "success")) 764 | { 765 | log.info("Successfully classified: {}", elementName); 766 | return new ClassifyResult(new TestAiElement(r.get("elem").getAsJsonObject(), this), key); 767 | } 768 | 769 | String rawMsg = JsonUtils.stringFromJson(r, "message"); 770 | 771 | if (rawMsg != null) 772 | { 773 | String cFailedBase = "Classification failed for element_name: "; 774 | 775 | if (rawMsg.contains("Please label") || rawMsg.contains("Did not find")) 776 | msg = String.format("%s%s - Please visit %s/label/%s to classify", cFailedBase, elementName, serverURL, elementName); 777 | else if (rawMsg.contains("frozen label")) 778 | msg = String.format("%s%s - However this element is frozen, so no new screenshot was uploaded. Please unfreeze the element if you want to add this screenshot to training", cFailedBase, 779 | elementName); 780 | else 781 | msg = String.format("%s: Unknown error, here was the API response: %s", msg, r); 782 | } 783 | } 784 | catch (Throwable e) 785 | { 786 | e.printStackTrace(); 787 | } 788 | 789 | log.warn(msg); 790 | return new ClassifyResult(null, key, msg); 791 | } 792 | 793 | /** 794 | * Simple container for encapsulating results of calls to {@code classify()}. 795 | * 796 | * @author Alexander Wu (alec@test.ai) 797 | * 798 | */ 799 | private static class ClassifyResult 800 | { 801 | /** 802 | * The TestAiElement created by the call to classify 803 | */ 804 | public TestAiElement e; 805 | 806 | /** 807 | * The key returned by the call to classify 808 | */ 809 | public String key; 810 | 811 | /** 812 | * The message associated with this result 813 | */ 814 | public String msg; 815 | 816 | /** 817 | * Constructor, creates a new ClassifyResult. 818 | * 819 | * @param e The TestAiElement to to use 820 | * @param key The key to use 821 | * @param msg The message to associate with this result 822 | */ 823 | ClassifyResult(TestAiElement e, String key, String msg) 824 | { 825 | this.e = e; 826 | this.key = key; 827 | this.msg = msg; 828 | } 829 | 830 | /** 831 | * Constructor, creates a new ClassifyResult, where the {@code msg} is set to the empty String by default. 832 | * 833 | * @param e 834 | * @param key 835 | */ 836 | ClassifyResult(TestAiElement e, String key) 837 | { 838 | this(e, key, ""); 839 | } 840 | } 841 | } 842 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/TestAiElement.java: -------------------------------------------------------------------------------- 1 | package ai.test.sdk; 2 | 3 | import com.google.gson.JsonObject; 4 | 5 | import java.util.List; 6 | 7 | import org.openqa.selenium.By; 8 | import org.openqa.selenium.Dimension; 9 | import org.openqa.selenium.Point; 10 | import org.openqa.selenium.Rectangle; 11 | import org.openqa.selenium.WebElement; 12 | import org.openqa.selenium.remote.RemoteWebDriver; 13 | import org.openqa.selenium.remote.RemoteWebElement; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | /** 18 | * An enhanced RemoteWebElement which uses the results of the Test.ai classifier for improved accuracy. 19 | * 20 | * @author Alexander Wu (alec@test.ai) 21 | * 22 | */ 23 | public class TestAiElement extends RemoteWebElement 24 | { 25 | /** 26 | * The logger for this class 27 | */ 28 | private static Logger log = LoggerFactory.getLogger(TestAiElement.class); 29 | 30 | /** 31 | * The webdriver the user is using. We wrap this for when the user calls methods that interact with selenium. 32 | */ 33 | private RemoteWebDriver driver; 34 | 35 | /** 36 | * The underlying {@code WebElement} used for performing actions in the browser. 37 | */ 38 | private WebElement realElement; 39 | 40 | /** 41 | * The text in this element, as determined by test.ai's classifier 42 | */ 43 | private String text; 44 | 45 | /** 46 | * The size of this element, in pixels 47 | */ 48 | private Dimension size; 49 | 50 | /** 51 | * The location of this element, in pixels (offset from the upper left corner of the screen) 52 | */ 53 | private Point location; 54 | 55 | /** 56 | * The rectangle that can be drawn around this element. Basically combines size and location. 57 | */ 58 | private Rectangle rectangle; 59 | 60 | /** 61 | * The tag name of this element, as determined by test.ai's classifier 62 | */ 63 | private String tagName; 64 | 65 | /** 66 | * Constructor, creates a new TestAiElement 67 | * 68 | * @param elem The element data returned by the FD API, as JSON 69 | * @param driver The {@code TestAiDriver} to associate with this {@code TestAiElement}. 70 | */ 71 | TestAiElement(JsonObject elem, TestAiDriver driver) 72 | { 73 | log.debug("Creating new TestAiElement w/ {}", elem); 74 | 75 | this.driver = driver.driver; 76 | this.realElement = MatchUtils.matchBoundingBoxToSeleniumElement(elem, driver); 77 | 78 | text = JsonUtils.stringFromJson(elem, "text"); 79 | size = new Dimension(JsonUtils.intFromJson(elem, "width") / (int) driver.multiplier, JsonUtils.intFromJson(elem, "height") / (int) driver.multiplier); 80 | 81 | location = new Point(JsonUtils.intFromJson(elem, "x") / (int) driver.multiplier, JsonUtils.intFromJson(elem, "y") / (int) driver.multiplier); 82 | 83 | // this.property = property //TODO: not referenced/implemented on python side?? 84 | rectangle = new Rectangle(location, size); 85 | tagName = JsonUtils.stringFromJson(elem, "class"); 86 | 87 | } 88 | 89 | @Override 90 | public String getText() 91 | { 92 | return text; 93 | } 94 | 95 | @Override 96 | public Dimension getSize() 97 | { 98 | return size; 99 | } 100 | 101 | @Override 102 | public Point getLocation() 103 | { 104 | return location; 105 | } 106 | 107 | @Override 108 | public Rectangle getRect() 109 | { 110 | return rectangle; 111 | } 112 | 113 | @Override 114 | public String getTagName() 115 | { 116 | return tagName; 117 | } 118 | 119 | @Override 120 | public void clear() 121 | { 122 | realElement.clear(); 123 | } 124 | 125 | @Override 126 | public WebElement findElement(By by) 127 | { 128 | return driver.findElement(by); 129 | } 130 | 131 | @Override 132 | public List findElements(By by) 133 | { 134 | return driver.findElements(by); 135 | } 136 | 137 | @Override 138 | public String getAttribute(String name) 139 | { 140 | return realElement.getAttribute(name); 141 | } 142 | 143 | @Override 144 | public String getCssValue(String propertyName) 145 | { 146 | return realElement.getCssValue(propertyName); 147 | } 148 | 149 | @Override 150 | public boolean isDisplayed() 151 | { 152 | return realElement.isDisplayed(); 153 | } 154 | 155 | @Override 156 | public boolean isEnabled() 157 | { 158 | return realElement.isEnabled(); 159 | } 160 | 161 | @Override 162 | public boolean isSelected() 163 | { 164 | return realElement.isSelected(); 165 | } 166 | 167 | @Override 168 | public void click() 169 | { 170 | realElement.click(); 171 | } 172 | 173 | @Override 174 | public void sendKeys(CharSequence... keysToSend) 175 | { 176 | realElement.sendKeys(keysToSend); 177 | } 178 | 179 | @Override 180 | public void submit() 181 | { 182 | realElement.submit(); 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/main/java/ai/test/sdk/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Contains the main classes for the Test.ai (Fluffy Dragon) SDK. These classes provide simple wrappers around existing selenium functionality to seamlessly incorporate Test.ai's powerful element 3 | * classification technology. 4 | */ 5 | 6 | package ai.test.sdk; -------------------------------------------------------------------------------- /src/test/java/ai/test/LibraryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This Java source file was generated by the Gradle 'init' task. 3 | */ 4 | package ai.test; 5 | 6 | import org.junit.jupiter.api.Test; 7 | //import static org.junit.jupiter.api.Assertions.*; 8 | 9 | class LibraryTest { 10 | @Test void someLibraryMethodReturnsTrue() { 11 | // Library classUnderTest = new Library(); 12 | // assertTrue(classUnderTest.someLibraryMethod(), "someLibraryMethod should return 'true'"); 13 | } 14 | } 15 | --------------------------------------------------------------------------------