├── Rakefile ├── .gitignore ├── lib ├── sikuli │ ├── version.rb │ ├── exception.rb │ ├── screen.rb │ ├── typeable.rb │ ├── app.rb │ ├── key_code.rb │ ├── platform.rb │ ├── config.rb │ ├── region.rb │ ├── searchable.rb │ └── clickable.rb └── sikuli.rb ├── Gemfile ├── spec ├── support │ └── images │ │ ├── apple.png │ │ ├── green_apple.png │ │ ├── smiley_face.png │ │ └── test_area.jpg ├── sikuli │ ├── app_spec.rb │ ├── screen_spec.rb │ ├── region_spec.rb │ ├── typeable_spec.rb │ ├── config_spec.rb │ ├── clickable_spec.rb │ └── searchable_spec.rb └── spec_helper.rb ├── sikuli.gemspec └── README.md /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.gem 3 | .bundle 4 | Gemfile.lock 5 | pkg/* 6 | -------------------------------------------------------------------------------- /lib/sikuli/version.rb: -------------------------------------------------------------------------------- 1 | module Sikuli 2 | VERSION = "0.3.0" 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | # Specify your gem's dependencies in sikuli_ruby.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /spec/support/images/apple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaslemley/sikuli_ruby/HEAD/spec/support/images/apple.png -------------------------------------------------------------------------------- /spec/support/images/green_apple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaslemley/sikuli_ruby/HEAD/spec/support/images/green_apple.png -------------------------------------------------------------------------------- /spec/support/images/smiley_face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaslemley/sikuli_ruby/HEAD/spec/support/images/smiley_face.png -------------------------------------------------------------------------------- /spec/support/images/test_area.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaslemley/sikuli_ruby/HEAD/spec/support/images/test_area.jpg -------------------------------------------------------------------------------- /spec/sikuli/app_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::App do 4 | subject { Sikuli::App.new('Finder') } 5 | 6 | its(:window) { should be_an_instance_of Sikuli::Region } 7 | end 8 | -------------------------------------------------------------------------------- /lib/sikuli.rb: -------------------------------------------------------------------------------- 1 | require "sikuli/platform" 2 | require Sikuli::Platform.sikuli_script_path 3 | require "sikuli/version" 4 | 5 | require "sikuli/app" 6 | require "sikuli/exception" 7 | require "sikuli/region" 8 | require "sikuli/screen" 9 | require "sikuli/key_code" 10 | require "sikuli/config" 11 | -------------------------------------------------------------------------------- /spec/sikuli/screen_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::Screen, 'when initialized' do 4 | subject { Sikuli::Screen.new } 5 | 6 | its(:x) { should == 0 } 7 | its(:y) { should == 0 } 8 | its(:width) { should > 0 } # screen's resolution 9 | its(:width) { should be_instance_of Fixnum } 10 | its(:height) { should > 0 } # screen's resolution 11 | its(:height) { should be_instance_of Fixnum } 12 | end 13 | -------------------------------------------------------------------------------- /lib/sikuli/exception.rb: -------------------------------------------------------------------------------- 1 | # Exception classes for Sikuli image searching and matching 2 | # 3 | module Sikuli 4 | 5 | # Thrown when Sikuli is unable to find a match within the region for the 6 | # file given. 7 | # 8 | class ImageNotFound < StandardError; end 9 | 10 | # Thrown when a filename is given that is not found on disk in the image 11 | # path. Image path can be configured using Sikuli::Config.image_path 12 | # 13 | class FileDoesNotExist < StandardError; end 14 | end -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'java' 2 | 3 | require 'rspec' 4 | require 'sikuli' 5 | 6 | Sikuli::Config.logging = false 7 | 8 | def setup_test_area 9 | Sikuli::Config.image_path = "#{Dir.pwd}/spec/support/images/" 10 | screen = Sikuli::Screen.new 11 | app = Sikuli::App.new('Preview') 12 | app.focus() 13 | center = screen.find("#{Dir.pwd}/spec/support/images/smiley_face.png") 14 | 15 | test_area = Sikuli::Region.new(center.x - 212, center.y - 212, 503, 503) 16 | test_area.highlight 17 | 18 | test_area 19 | end 20 | -------------------------------------------------------------------------------- /lib/sikuli/screen.rb: -------------------------------------------------------------------------------- 1 | # A Screen object defines a special type of Sikuli::Region that represents 2 | # the entire screen. 3 | # 4 | # TODO: Test the Screen object with multiple monitors attached. 5 | # 6 | module Sikuli 7 | class Screen < Region 8 | 9 | # Public: creates a new Screen object 10 | # 11 | # Examples 12 | # 13 | # screen = Sikuli::Screen.new 14 | # 15 | # Returns the newly initialized Screen object 16 | def initialize 17 | @java_obj = org.sikuli.script::Screen.new() 18 | end 19 | end 20 | end -------------------------------------------------------------------------------- /spec/sikuli/region_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::Region, 'when initialized with a rectangle' do 4 | subject { Sikuli::Region.new(10, 15, 250, 400) } 5 | its(:x) { should == 10 } 6 | its(:y) { should == 15 } 7 | its(:width) { should == 250 } 8 | its(:height) { should == 400 } 9 | end 10 | 11 | describe Sikuli::Region, 'when initialzed with a sikuli region' do 12 | subject { Sikuli::Region.new org.sikuli.script::Region.new(11, 16, 251, 401) } 13 | its(:x) { should == 11 } 14 | its(:y) { should == 16 } 15 | its(:width) { should == 251 } 16 | its(:height) { should == 401 } 17 | end -------------------------------------------------------------------------------- /lib/sikuli/typeable.rb: -------------------------------------------------------------------------------- 1 | # Defines interactions with the keyboard. Implemented in the Region class. 2 | # 3 | module Sikuli 4 | module Typeable 5 | 6 | # Public: Types text as if it was being typed on the keyboard with an 7 | # optional key modifier 8 | # 9 | # text - String representing text to be typed on keyboard 10 | # modifier - (optional) Sikilu constant (defined in key_code.rb) 11 | # representing key to hold while typing text 12 | # 13 | # Examples 14 | # 15 | # region.type("Hello World") 16 | # region.type("s", Sikuli::KEY_CMD) # saves a file 17 | # 18 | # Returns nothing 19 | def type(text, modifier = 0) 20 | @java_obj.type(nil, text, modifier) 21 | end 22 | end 23 | end -------------------------------------------------------------------------------- /spec/sikuli/typeable_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::Region, "#Typeable" do 4 | before(:all) do 5 | app = Sikuli::App.new("TextEdit") 6 | app.focus 7 | @region = app.window 8 | end 9 | 10 | context "modifying text input with" do 11 | it "apple key" do 12 | @region.type("n", Sikuli::KEY_CMD) 13 | end 14 | 15 | it "shift key" do 16 | @region.type("this should be lower case") 17 | @region.type("this should be upper case", Sikuli::KEY_SHIFT) 18 | end 19 | end 20 | 21 | context "unicode characters" do 22 | it("backspace") { @region.type(Sikuli::KEY_BACKSPACE * 50) } 23 | it("return") { @region.type(Sikuli::KEY_RETURN) } 24 | it("left arrow") { @region.type(Sikuli::LEFT_ARROW) } 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/sikuli/app.rb: -------------------------------------------------------------------------------- 1 | # An App object represents a running app on the system. 2 | # 3 | module Sikuli 4 | class App 5 | 6 | # Public: creates a new App instance 7 | # 8 | # app_name - String name of the app 9 | # 10 | # Examples 11 | # 12 | # App.new("TextEdit") 13 | # 14 | # Returns the newly initialized App 15 | def initialize(app_name) 16 | @java_obj = org.sikuli.script::App.new(app_name) 17 | end 18 | 19 | # Public: brings the App to focus 20 | # 21 | # Returns nothing 22 | def focus 23 | @java_obj.focus() 24 | end 25 | 26 | # Public: the Region instance representing the app's window 27 | # 28 | # Returns the newly initialized Region 29 | def window 30 | Region.new(@java_obj.window()) 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/sikuli/key_code.rb: -------------------------------------------------------------------------------- 1 | require 'java' 2 | java_import 'org.sikuli.script.Key' 3 | java_import 'org.sikuli.script.KeyModifier' 4 | 5 | # These constants represent keyboard codes for interacting with the keyboard. 6 | # Keyboard interaction is defined in the Sikuli::Typeable module. 7 | # 8 | module Sikuli 9 | 10 | # Command Key 11 | KEY_CMD = KeyModifier::META 12 | 13 | # Shift Key 14 | KEY_SHIFT = KeyModifier::SHIFT 15 | 16 | # Control Key 17 | KEY_CTRL = KeyModifier::CTRL 18 | 19 | # Alt Key 20 | KEY_ALT = KeyModifier::ALT 21 | 22 | # Backspace Key 23 | KEY_BACKSPACE = Key::BACKSPACE 24 | 25 | # Return Key 26 | KEY_RETURN = Key::ENTER 27 | 28 | # Left Arrow Key 29 | LEFT_ARROW = Key::LEFT 30 | 31 | # Right Arrow Key 32 | RIGHT_ARROW = Key::RIGHT 33 | 34 | # Up Arrow Key 35 | UP_ARROW = Key::UP 36 | 37 | # Down Arrow Key 38 | DOWN_ARROW = Key::DOWN 39 | end -------------------------------------------------------------------------------- /sikuli.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "sikuli/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.add_development_dependency "rspec" 7 | s.name = "sikuli" 8 | s.version = Sikuli::VERSION 9 | s.authors = ["Chas Lemley"] 10 | s.email = ["chas.lemley@gmail.com"] 11 | s.homepage = "https://github.com/chaslemley/sikuli_ruby" 12 | s.summary = %q{Ruby wrapper for Sikuli GUI automation library} 13 | s.description = %q{Sikuli allows you to interact with your application's user interface using image based search to automate user actions.} 14 | 15 | s.rubyforge_project = "sikuli" 16 | 17 | s.files = `git ls-files`.split("\n") 18 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 19 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 20 | s.require_paths = ["lib"] 21 | end 22 | -------------------------------------------------------------------------------- /lib/sikuli/platform.rb: -------------------------------------------------------------------------------- 1 | # Detect the platform we're running on so we can tweak behaviour 2 | # in various places. 3 | 4 | module Sikuli 5 | class Platform 6 | 7 | WINDOWS = RbConfig::CONFIG['host_os'] =~ /mswin/ 8 | LINUX = RbConfig::CONFIG['host_os'] =~ /linux/ 9 | MINGW = RbConfig::CONFIG['host_os'] =~ /mingw/ 10 | OS_X = RbConfig::CONFIG['host_os'] =~ /darwin/ 11 | 12 | def self.sikuli_script_path 13 | if OS_X 14 | path = "/Applications/Sikuli-IDE.app/Contents/Resources/Java/sikuli-script.jar" 15 | else 16 | raise LoadError, no_sikuli_home_err_msg if ENV['SIKULI_HOME'].nil? 17 | path = "#{ENV['SIKULI_HOME']}/sikuli-script.jar" 18 | end 19 | unless File.exist?(path) 20 | raise LoadError, "Failed to load '#{path}'\nIs Sikuli installed?" 21 | end 22 | path 23 | end 24 | 25 | private 26 | 27 | def self.no_sikuli_home_err_msg 28 | err = "Failed to load 'sikuli-script.jar' from the Sikuli home directory" 29 | return err + "\nMake sure %SIKULI_HOME% is set!" if WINDOWS 30 | return err + "\nMake sure $SIKULI_HOME is set!" if LINUX 31 | err + "\nMake sure %SIKULI_HOME% or $SIKULI_HOME is set!" 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /spec/sikuli/config_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::Config do 4 | it "should retain a list of directories to search in for images" do 5 | Sikuli::Config.image_path = "" 6 | java.lang.System.getProperty("SIKULI_IMAGE_PATH").should == "" 7 | end 8 | 9 | it "should allow a path to be added" do 10 | Sikuli::Config.run do |config| 11 | config.image_path = '/Users/images/' 12 | end 13 | 14 | java.lang.System.getProperty("SIKULI_IMAGE_PATH").should == '/Users/images/' 15 | Sikuli::Config.image_path.should == '/Users/images/' 16 | end 17 | 18 | it "should allow logging to be turned on" do 19 | Sikuli::Config.run do |config| 20 | config.logging = true 21 | end 22 | 23 | org.sikuli.script::Settings.InfoLogs.should be_true 24 | org.sikuli.script::Settings.ActionLogs.should be_true 25 | org.sikuli.script::Settings.DebugLogs.should be_true 26 | end 27 | 28 | it "should allow logging to be turned off" do 29 | Sikuli::Config.run do |config| 30 | config.logging = false 31 | end 32 | 33 | org.sikuli.script::Settings.InfoLogs.should be_false 34 | org.sikuli.script::Settings.ActionLogs.should be_false 35 | org.sikuli.script::Settings.DebugLogs.should be_false 36 | end 37 | 38 | it "should allow us to turn off highlighting on find" do 39 | Sikuli::Config.highlight_on_find = false 40 | Sikuli::Config.highlight_on_find.should be_false 41 | end 42 | 43 | it "should allow us to turn on highlighting on find" do 44 | Sikuli::Config.highlight_on_find = true 45 | Sikuli::Config.highlight_on_find.should be_true 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /spec/sikuli/clickable_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::Region, "#Clickable" do 4 | before(:all) do 5 | @region = setup_test_area 6 | end 7 | 8 | it "should perform a click at 10, 10" do 9 | lambda { @region.click(12, 12) }.should_not raise_error 10 | end 11 | 12 | it "should perform a double click 10, 1040" do 13 | lambda { @region.double_click(12, 491) }.should_not raise_error 14 | end 15 | 16 | it "should perform a double click on an image" do 17 | lambda { @region.double_click("smiley_face.png") }.should_not raise_error 18 | end 19 | 20 | it "should perform a drag and drop" do 21 | lambda { @region.drag_drop(12, 12, 491, 491) }.should_not raise_error 22 | end 23 | 24 | it "should perform a click on an image" do 25 | lambda { @region.click("smiley_face.png") }.should_not raise_error 26 | end 27 | 28 | it "should not perform a click on an image that is outside of the region" do 29 | lambda { @region.click("apple.png") }.should 30 | raise_error(Sikuli::ImageNotFound, "The image 'apple.png' did not match in this region.") 31 | end 32 | 33 | it "should perform a click and hold on an image" do 34 | lambda { @region.click_and_hold(2, "green_apple.png") }.should_not raise_error 35 | end 36 | 37 | it "should perform a click and hold on a point" do 38 | lambda { @region.click_and_hold(2, 100, 100) }.should_not raise_error 39 | end 40 | 41 | it "should simulate a mouse scroll wheel up event" do 42 | lambda { @region.wheel_up(10) }.should_not raise_error 43 | end 44 | 45 | it "should simulate a mouse scroll wheel down event" do 46 | lambda { @region.wheel_down(10) }.should_not raise_error 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Note 2 | ==== 3 | 4 | This project is no longer maintained. I recommend using this project instead: https://github.com/andreanastacio/rukuli 5 | 6 | Sikuli Ruby 7 | =========== 8 | 9 | [Sikuli](http://sikuli.org/) allows you to interact with your application's user interface using image based search to automate user actions. 10 | 11 | Requirements 12 | ------------ 13 | 14 | * [JRuby](http://jruby.org/download) or `rvm install jruby` 15 | * [Sikuli X 1.0rc3](http://sikuli.org/) 16 | 17 | Compatibility 18 | ------------- 19 | It's recommended to use JRuby ~> 1.6.0 and run it in 1.9 mode to get unicode characters working as expected. 20 | 21 | ``` 22 | export JRUBY_OPTS=--1.9 23 | ``` 24 | 25 | On Windows or Linux make sure to set SIKULI_HOME to the Sikuli installation directory and to add the Sikuli installation directory and Sikuli libs directory to the include path. 26 | 27 | ``` 28 | export SIKULI_HOME="~/bin/Sikuli-X-1.0rc3/Sikuli-IDE/" 29 | PATH="${PATH}:~/bin/Sikuli-X-1.0rc3/Sikuli-IDE/:~/bin/Sikuli-X-1.0rc3/Sikuli-IDE/libs/" 30 | ``` 31 | 32 | Installation 33 | ------------ 34 | 35 | gem install sikuli 36 | 37 | Usage 38 | ----- 39 | 40 | require 'java' 41 | require 'sikuli' 42 | 43 | Sikuli::Config.run do |config| 44 | config.image_path = "#{Dir.pwd}/images/" 45 | config.logging = false 46 | end 47 | 48 | screen = Sikuli::Screen.new 49 | screen.click(10, 10) # should open your apple menu 50 | 51 | app = Sikuli::App.new("iPhone Simulator") 52 | app.window.click('ui_element.png') if app.window.find('ui_element.png') 53 | 54 | Running the test suite 55 | ---------------------- 56 | 57 | 1. You need to open `test_area.jpg` in **Preview** from `spec/support/images/` directory 58 | before running tests. 59 | 2. You also need to open the **TextEdit** app 60 | 61 | Examples 62 | -------- 63 | 64 | * [Cucumber Suite](https://github.com/chaslemley/cucumber_sikuli) 65 | -------------------------------------------------------------------------------- /lib/sikuli/config.rb: -------------------------------------------------------------------------------- 1 | # Config variables for the Sikuli driver 2 | # 3 | module Sikuli 4 | class Config 5 | class << self 6 | 7 | # Public: the Boolean representing whether or not to perform a 1 second 8 | # highlight when an image is matched through Searchable#find, 9 | # Searchable#find_all. Defaults to false. 10 | attr_accessor :highlight_on_find 11 | 12 | # Public: the absolute file path where Sikuli will look for images when 13 | # a just a filename is passed to a search or click method 14 | # 15 | # Returns the String representation of the path 16 | def image_path 17 | java.lang.System.getProperty("SIKULI_IMAGE_PATH") 18 | end 19 | 20 | # Public: the setter for the absolute file path where Sikuli will search 21 | # for images with given a filename as an image 22 | # 23 | # Examples 24 | # 25 | # Sikuli::Config.image_path = "/Users/clemley/sikuli/images/" 26 | # 27 | # Returns nothing 28 | def image_path=(path) 29 | java.lang.System.setProperty("SIKULI_IMAGE_PATH", path) 30 | end 31 | 32 | # Public: turns stdout logging on and off for the Sikuli java classes. 33 | # Defaults to true. 34 | # 35 | # Examples 36 | # 37 | # Sikuli::Config.logging = false 38 | # 39 | # Returns nothing 40 | def logging=(boolean) 41 | return unless [TrueClass, FalseClass].include? boolean.class 42 | org.sikuli.script::Settings.InfoLogs = boolean 43 | org.sikuli.script::Settings.ActionLogs = boolean 44 | org.sikuli.script::Settings.DebugLogs = boolean 45 | end 46 | 47 | # Public: convienence method for grouping the setting of config 48 | # variables 49 | # 50 | # Examples 51 | # 52 | # Sikuli::Config.run do |config| 53 | # config.logging = true 54 | # config.image_path = "/User/clemley/images" 55 | # config.highlight_on_find = true 56 | # end 57 | # 58 | # Returns nothing 59 | def run(*args) 60 | if block_given? 61 | yield self 62 | end 63 | end 64 | 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /spec/sikuli/searchable_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sikuli::Region, "#Searchable" do 4 | before(:all) do 5 | @region = setup_test_area 6 | end 7 | 8 | it "should find an image within the region" do 9 | @region.find("smiley_face.png").should_not be_nil 10 | end 11 | 12 | it "should return a region containing the found image" do 13 | @region.find("smiley_face.png").should be_an_instance_of Sikuli::Region 14 | end 15 | 16 | it "should raise an error if a file can not be found" do 17 | lambda { @region.find("no_photo.png") }.should 18 | raise_error(Sikuli::FileDoesNotExist, "The file 'no_photo.png' does not exist.") 19 | end 20 | 21 | it "should not find an image that is not in the region" do 22 | lambda { @region.find("apple.png") }.should 23 | raise_error(Sikuli::ImageNotFound, "The image 'apple.png' did not match in this region.") 24 | end 25 | 26 | it "should return true if the image is found" do 27 | @region.find!("smiley_face.png").should be_an_instance_of Sikuli::Region 28 | end 29 | 30 | it "should return nil if the image is not found" do 31 | @region.find!("apple.png").should be_nil 32 | end 33 | 34 | it "should raise an exception if the file does not exist" do 35 | lambda { @region.find!("no_photo.png") }.should 36 | raise_error(Sikuli::FileDoesNotExist, "The file 'no_photo.png' does not exist.") 37 | end 38 | 39 | context "#wait" do 40 | it "should raise an error if no match is found after a given time" do 41 | lambda { @region.wait('apple.png') }.should 42 | raise_error(Sikuli::ImageNotFound, "The image 'apple.png' did not match in this region.") 43 | end 44 | 45 | it "should return a Region object when a match is found" do 46 | match = @region.wait('green_apple.png', 5) 47 | match.should be_an_instance_of Sikuli::Region 48 | end 49 | end 50 | 51 | context "#find_all" do 52 | before(:all) do 53 | @matches = @region.find_all("green_apple.png") 54 | end 55 | 56 | it "should raise an error if no matches are found" do 57 | lambda { @region.find_all("apple.png") }.should 58 | raise_error(Sikuli::ImageNotFound, "The image 'apple.png' did not match in this region.") 59 | end 60 | 61 | it "should return an array" do 62 | @matches.should be_an_instance_of Array 63 | end 64 | 65 | it "should contain 4 matches" do 66 | @matches.length.should == 4 67 | end 68 | 69 | it "should contain regions" do 70 | @matches.each do |m| 71 | m.should be_an_instance_of Sikuli::Region 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/sikuli/region.rb: -------------------------------------------------------------------------------- 1 | # A Region represents a rectangle on screen. Regions are the main point of 2 | # interaction for Sikuli actions. Regions can receive actions from the mouse, 3 | # keyboard, and image search. 4 | # 5 | require "sikuli/clickable" 6 | require "sikuli/typeable" 7 | require "sikuli/searchable" 8 | 9 | module Sikuli 10 | class Region 11 | include Clickable 12 | include Typeable 13 | include Searchable 14 | 15 | # Public: creates a new Region object 16 | # 17 | # args - Array representing x (left bound), y (top), width, height 18 | # 4 Fixnums left, top, width, height 19 | # An instance of an org.sikuli.script::Region 20 | # 21 | # Examples 22 | # 23 | # Region.new([10, 10, 200, 300]) 24 | # Region.new(10, 10, 200, 300) 25 | # Region.new(another_region) 26 | # 27 | # Returns the newly initialized object 28 | def initialize(*args) 29 | @java_obj = org.sikuli.script::Region.new(*args) 30 | end 31 | 32 | # Public: highlight the region with a ~ 5 pixel red border 33 | # 34 | # seconds - Fixnum length of time to show border 35 | # 36 | # Returns nothing 37 | def highlight(seconds = 1) 38 | @java_obj.highlight(seconds) 39 | end 40 | 41 | # Public: the x component of the top, left corner of the Region 42 | def x 43 | @java_obj.x() 44 | end 45 | 46 | # Public: the y component of the top, left corner of the Region 47 | def y 48 | @java_obj.y() 49 | end 50 | 51 | # Public: the width in pixels of the Region 52 | def width 53 | @java_obj.w() 54 | end 55 | 56 | # Public: the height in pixels of the Region 57 | def height 58 | @java_obj.h() 59 | end 60 | 61 | # Public: provide access to all region methods provided by the SikuliScript API 62 | # See http://sikuli.org/doc/java/edu/mit/csail/uid/Region.html 63 | def method_missing method_name, *args, &block 64 | @java_obj.send method_name, *args, &block 65 | end 66 | 67 | private 68 | 69 | # Private: interpret a java NativeException and raises a more descriptive 70 | # exception 71 | # 72 | # exception - The original java exception thrown by the sikuli java_obj 73 | # filename - A string representing the filename to include in the 74 | # exception message 75 | # 76 | # Returns nothing 77 | def raise_exception(exception, filename) 78 | if exception.message == "org.sikuli.script.FindFailed: File null not exists" 79 | raise Sikuli::FileDoesNotExist, "The file '#{filename}' does not exist." 80 | else 81 | raise Sikuli::ImageNotFound, "The image '#{filename}' did not match in this region." 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/sikuli/searchable.rb: -------------------------------------------------------------------------------- 1 | # The Sikuli::Searchable module is the heart of Sikuli. It defines the 2 | # wrapper around Sikuli's on screen image searching and matching capability 3 | # It is implemented by the Region class. 4 | # 5 | module Sikuli 6 | module Searchable 7 | 8 | # Public: search for an image within a Region 9 | # 10 | # filename - A String representation of the filename to match against 11 | # similarity - A Float between 0 and 1 representing the threshold for 12 | # matching an image. Passing 1 corresponds to a 100% pixel for pixel 13 | # match. Defaults to 0.9 (90% match) 14 | # 15 | # Examples 16 | # 17 | # region.find('needle.png') 18 | # region.find('needle.png', 0.5) 19 | # 20 | # Returns an instance of Region representing the best match 21 | # 22 | # Throws Sikuli::FileNotFound if the file could not be found on the system 23 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 24 | def find(filename, similarity = 0.9) 25 | begin 26 | pattern = build_pattern(filename, similarity) 27 | match = Region.new(@java_obj.find(pattern)) 28 | match.highlight if Sikuli::Config.highlight_on_find 29 | match 30 | rescue NativeException => e 31 | raise_exception e, filename 32 | end 33 | end 34 | 35 | # Public: search for an image within a region (does not raise ImageNotFound exceptions) 36 | # 37 | # filename - A String representation of the filename to match against 38 | # similarity - A Float between 0 and 1 representing the threshold for 39 | # matching an image. Passing 1 corresponds to a 100% pixel for pixel 40 | # match. Defaults to 0.9 (90% match) 41 | # 42 | # Examples 43 | # 44 | # region.find!('needle.png') 45 | # region.find!('needle.png', 0.5) 46 | # 47 | # Returns the match or nil if no match is found 48 | def find!(filename, similarity = 0.9) 49 | begin 50 | find(filename, similarity) 51 | rescue Sikuli::ImageNotFound => e 52 | nil 53 | end 54 | end 55 | 56 | # Public: search for an image within a Region and return all matches 57 | # 58 | # TODO: Sort return results so they are always returned in the same order 59 | # (top left to bottom right) 60 | # 61 | # filename - A String representation of the filename to match against 62 | # similarity - A Float between 0 and 1 representing the threshold for 63 | # matching an image. Passing 1 corresponds to a 100% pixel for pixel 64 | # match. Defaults to 0.9 (90% match) 65 | # 66 | # Examples 67 | # 68 | # region.find_all('needle.png') 69 | # region.find_all('needle.png', 0.5) 70 | # 71 | # Returns an array of Region objects that match the given file and 72 | # threshold 73 | # 74 | # Throws Sikuli::FileNotFound if the file could not be found on the system 75 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 76 | def find_all(filename, similarity = 0.9) 77 | begin 78 | pattern = build_pattern(filename, similarity) 79 | matches = @java_obj.findAll(pattern) 80 | regions = matches.collect do |r| 81 | match = Region.new(r) 82 | match.highlight if Sikuli::Config.highlight_on_find 83 | match 84 | end 85 | regions 86 | rescue NativeException => e 87 | raise_exception e, filename 88 | end 89 | end 90 | 91 | # Public: wait for a match to appear within a region 92 | # 93 | # filename - A String representation of the filename to match against 94 | # time - A Fixnum representing the amount of time to wait defaults 95 | # to 2 seconds 96 | # similarity - A Float between 0 and 1 representing the threshold for 97 | # matching an image. Passing 1 corresponds to a 100% pixel for pixel 98 | # match. Defaults to 0.9 (90% match) 99 | # 100 | # Examples 101 | # 102 | # region.wait('needle.png') # wait for needle.png to appear for up to 1 second 103 | # region.wait('needle.png', 10) # wait for needle.png to appear for 10 seconds 104 | # 105 | # Returns nothing 106 | # 107 | # Throws Sikuli::FileNotFound if the file could not be found on the system 108 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 109 | def wait(filename, time = 2, similarity = 0.9) 110 | begin 111 | pattern = build_pattern(filename, similarity) 112 | match = Region.new(@java_obj.wait(pattern, time)) 113 | match.highlight if Sikuli::Config.highlight_on_find 114 | match 115 | rescue NativeException => e 116 | raise_exception e, filename 117 | end 118 | end 119 | 120 | private 121 | 122 | # Private: builds a java Pattern to check 123 | # 124 | # filename - A String representation of the filename to match against 125 | # similarity - A Float between 0 and 1 representing the threshold for 126 | # matching an image. Passing 1 corresponds to a 100% pixel for pixel 127 | # match. Defaults to 0.9 (90% match) 128 | # 129 | # Returns a org.sikuli.script::Pattern object to match against 130 | def build_pattern(filename, similarity) 131 | org.sikuli.script::Pattern.new(filename).similar(similarity) 132 | end 133 | end 134 | end -------------------------------------------------------------------------------- /lib/sikuli/clickable.rb: -------------------------------------------------------------------------------- 1 | # The Clickable module defines interaction with the mouse. It is included in 2 | # the Region class. 3 | # 4 | module Sikuli 5 | module Clickable 6 | 7 | # Public: Performs a single click on an image match or point (x, y) 8 | # 9 | # args - String representing filename of image to find and click 10 | # args - Fixnum, Fixnum representing x and y coordinates within 11 | # a Region (0,0) is the top left 12 | # 13 | # Examples 14 | # 15 | # region.click('smile.png') 16 | # region.click(123, 432) 17 | # 18 | # Returns nothing 19 | def click(*args) 20 | case args.length 21 | when 1 then click_image(args[0]) 22 | when 2 then click_point(args[0], args[1]) 23 | else raise ArgumentError 24 | end 25 | end 26 | 27 | # Public: Performs a double click on an image match or point (x, y) 28 | # 29 | # args - String representing filename of image to find and click 30 | # args - Fixnum, Fixnum representing x and y coordinates within 31 | # a Region (0,0) is the top left 32 | # 33 | # Examples 34 | # 35 | # region.double_click('smile.png') 36 | # region.double_click(123, 432) 37 | # 38 | # Returns nothing 39 | def double_click(*args) 40 | case args.length 41 | when 1 then click_image(args[0], true) 42 | when 2 then click_point(args[0], args[1], true) 43 | else raise ArgumentError 44 | end 45 | end 46 | 47 | # Public: Performs a click and hold on an image match or point (x, y) 48 | # 49 | # args - String representing filename of image to find and click 50 | # args - Fixnum, Fixnum representing x and y coordinates within 51 | # a Region (0,0) is the top left 52 | # seconds - Fixnum representing the number of seconds to hold down 53 | # before releasing 54 | # 55 | # Examples 56 | # 57 | # region.click_and_hold('smile.png', 2) 58 | # region.click_and_hold(123, 432, 2) 59 | # 60 | # Returns nothing 61 | def click_and_hold(seconds = 1, *args) 62 | case args.length 63 | when 1 then click_image_and_hold(args[0], seconds) 64 | when 2 then click_point_and_hold(args[0], args[1], seconds) 65 | else raise ArgumentError 66 | end 67 | end 68 | 69 | # Public: Performs a mouse down, drag, and mouse up 70 | # 71 | # start_x - Fixnum representing the x of the mouse down 72 | # start_y - Fixnum representing the y of the mouse down 73 | # end_x - Fixnum representing the x of the mouse up 74 | # end_y - Fixnum representing the y of the mouse up 75 | # 76 | # Examples 77 | # 78 | # region.drag_drop(20, 12, 23, 44) 79 | # 80 | # Returns nothing 81 | def drag_drop(start_x, start_y, end_x, end_y) 82 | @java_obj.dragDrop( 83 | org.sikuli.script::Location.new(start_x, start_y).offset(x(), y()), 84 | org.sikuli.script::Location.new(end_x, end_y).offset(x(), y()), 85 | 0 86 | ) 87 | end 88 | 89 | # Public: Simulates turning of the mouse wheel up 90 | # 91 | # steps - Fixnum representing the number of steps to turn the mouse wheel 92 | # 93 | # Examples 94 | # 95 | # region.wheel_up(10) 96 | # 97 | # Returns nothing 98 | def wheel_up(steps = 1) 99 | @java_obj.wheel(-1, steps) 100 | end 101 | 102 | # Public: Simulates turning of the mouse wheel down 103 | # 104 | # steps - Fixnum representing the number of steps to turn the mouse wheel 105 | # 106 | # Examples 107 | # 108 | # region.wheel_down(10) 109 | # 110 | # Returns nothing 111 | def wheel_down(steps = 1) 112 | @java_obj.wheel(1, steps) 113 | end 114 | 115 | private 116 | 117 | # Private: turns the mouse wheel 118 | # 119 | # direction - Fixnum represeting direction to turn wheel 120 | # steps - the number of steps to turn the mouse wheel 121 | # 122 | # Returns nothing 123 | def wheel(direction, steps) 124 | @java_obj.wheel(direction, steps) 125 | end 126 | 127 | # Private: clicks on a matched Region based on an image based search 128 | # 129 | # filename - A String representation of the filename of the region to 130 | # match against 131 | # seconds - The length in seconds to hold the mouse 132 | # 133 | # Returns nothing 134 | # 135 | # Throws Sikuli::FileNotFound if the file could not be found on the system 136 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 137 | def click_image_and_hold(filename, seconds) 138 | begin 139 | pattern = org.sikuli.script::Pattern.new(filename).similar(0.9) 140 | @java_obj.hover(pattern) 141 | @java_obj.mouseDown(java.awt.event.InputEvent::BUTTON1_MASK) 142 | sleep(seconds.to_i) 143 | @java_obj.mouseUp(0) 144 | rescue NativeException => e 145 | raise_exception e, filename 146 | end 147 | end 148 | 149 | # Private: clicks on a point within the region 150 | # 151 | # filename - A String representation of the filename of the region to 152 | # match against 153 | # 154 | # Returns nothing 155 | # 156 | # Throws Sikuli::FileNotFound if the file could not be found on the system 157 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 158 | def click_point_and_hold(x, y, seconds) 159 | begin 160 | location = org.sikuli.script::Location.new(x, y).offset(x(), y()) 161 | @java_obj.hover(location) 162 | @java_obj.mouseDown(java.awt.event.InputEvent::BUTTON1_MASK) 163 | sleep(seconds.to_i) 164 | @java_obj.mouseUp(0) 165 | rescue NativeException => e 166 | raise_exception e, filename 167 | end 168 | end 169 | 170 | # Private: clicks on a matched Region based on an image based search 171 | # 172 | # filename - A String representation of the filename of the region to 173 | # match against 174 | # is_double - (optional) Boolean determining if should be a double click 175 | # 176 | # Returns nothing 177 | # 178 | # Throws Sikuli::FileNotFound if the file could not be found on the system 179 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 180 | def click_image(filename, is_double = false, and_hold = false) 181 | begin 182 | if is_double 183 | @java_obj.doubleClick(filename, 0) 184 | else 185 | @java_obj.click(filename, 0) 186 | end 187 | rescue NativeException => e 188 | raise_exception e, filename 189 | end 190 | end 191 | 192 | # Private: clicks on a point relative to a Region's top left corner 193 | # 194 | # x - a Fixnum representing the x component of the point to click 195 | # y - a Fixnum representing the y component of the point to click 196 | # is_double - (optional) Boolean determining if should be a double click 197 | # 198 | # Returns nothing 199 | # 200 | # Throws Sikuli::FileNotFound if the file could not be found on the system 201 | # Throws Sikuli::ImageNotMatched if no matches are found within the region 202 | def click_point(x, y, is_double = false) 203 | if is_double 204 | @java_obj.doubleClick(org.sikuli.script::Location.new(x, y).offset(x(), y()), 0) 205 | else 206 | @java_obj.click(org.sikuli.script::Location.new(x, y).offset(x(), y()), 0) 207 | end 208 | end 209 | end 210 | end 211 | --------------------------------------------------------------------------------