├── Rakefile ├── .gitignore ├── lib ├── salesforce_bulk │ ├── version.rb │ ├── connection.rb │ └── job.rb └── salesforce_bulk.rb ├── Gemfile ├── salesforce_bulk.gemspec └── README.rdoc /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | -------------------------------------------------------------------------------- /lib/salesforce_bulk/version.rb: -------------------------------------------------------------------------------- 1 | module SalesforceBulk 2 | VERSION = "1.0.0" 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | # Specify your gem's dependencies in salesforce_bulk.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /salesforce_bulk.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "salesforce_bulk/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "salesforce_bulk" 7 | s.version = SalesforceBulk::VERSION 8 | s.authors = ["Jorge Valdivia"] 9 | s.email = ["jorge@valdivia.me"] 10 | s.homepage = "https://github.com/jorgevaldivia/salesforce_bulk" 11 | s.summary = %q{Ruby support for the Salesforce Bulk API} 12 | s.description = %q{This gem provides a super simple interface for the Salesforce Bulk API. It provides support for insert, update, upsert, delete, and query.} 13 | 14 | s.rubyforge_project = "salesforce_bulk" 15 | 16 | s.files = `git ls-files`.split("\n") 17 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 18 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 19 | s.require_paths = ["lib"] 20 | 21 | # specify any dependencies here; for example: 22 | # s.add_development_dependency "rspec" 23 | # s.add_runtime_dependency "rest-client" 24 | 25 | s.add_dependency "xml-simple" 26 | 27 | end 28 | -------------------------------------------------------------------------------- /lib/salesforce_bulk.rb: -------------------------------------------------------------------------------- 1 | require 'net/https' 2 | require 'xmlsimple' 3 | require 'csv' 4 | require "salesforce_bulk/version" 5 | require 'salesforce_bulk/job' 6 | require 'salesforce_bulk/connection' 7 | 8 | module SalesforceBulk 9 | # Your code goes here... 10 | class Api 11 | 12 | @@SALESFORCE_API_VERSION = '24.0' 13 | 14 | def initialize(username, password, in_sandbox=false) 15 | @connection = SalesforceBulk::Connection.new(username, password, @@SALESFORCE_API_VERSION, in_sandbox) 16 | end 17 | 18 | def upsert(sobject, records, external_field, wait=false) 19 | self.do_operation('upsert', sobject, records, external_field, wait) 20 | end 21 | 22 | def update(sobject, records, wait=false) 23 | self.do_operation('update', sobject, records, nil, wait) 24 | end 25 | 26 | def create(sobject, records, wait=false) 27 | self.do_operation('insert', sobject, records, nil, wait) 28 | end 29 | 30 | def delete(sobject, records, wait=false) 31 | self.do_operation('delete', sobject, records, nil, wait) 32 | end 33 | 34 | def query(sobject, query) 35 | self.do_operation('query', sobject, query, nil) 36 | end 37 | 38 | def do_operation(operation, sobject, records, external_field, wait=false) 39 | job = SalesforceBulk::Job.new(operation, sobject, records, external_field, @connection) 40 | 41 | # TODO: put this in one function 42 | job_id = job.create_job() 43 | if(operation == "query") 44 | batch_id = job.add_query() 45 | else 46 | batch_id = job.add_batch() 47 | end 48 | job.close_job() 49 | 50 | if wait or operation == 'query' 51 | while true 52 | state = job.check_batch_status() 53 | if state != "Queued" && state != "InProgress" 54 | break 55 | end 56 | sleep(2) # wait x seconds and check again 57 | end 58 | 59 | if state == 'Completed' 60 | job.get_batch_result() 61 | job 62 | else 63 | job.result.message = "There is an error in your job. The response returned a state of #{state}. Please check your query/parameters and try again." 64 | job.result.success = false 65 | return job 66 | 67 | end 68 | else 69 | return job 70 | end 71 | 72 | end 73 | 74 | def parse_batch_result result 75 | begin 76 | CSV.parse(result, :headers => true) 77 | rescue 78 | result 79 | end 80 | end 81 | 82 | end # End class 83 | end # End module 84 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = salesforce-bulk 2 | 3 | ==Overview 4 | 5 | Salesforce bulk is a simple ruby gem for connecting to and using the Salesforce Bulk API (http://www.salesforce.com/us/developer/docs/api_asynch/index.htm). There are already some gems out there that provide connectivity to the Salesforce SOAP and Rest APIs, if your needs are simple, I recommend using those, specifically raygao's asf-rest-adapter (https://github.com/raygao/asf-rest-adapter) or databasedotcom (https://rubygems.org/gems/databasedotcom). 6 | 7 | ==Installation 8 | 9 | sudo gem install salesforce_bulk 10 | 11 | ==How to use 12 | 13 | Using this gem is simple and straight forward. 14 | 15 | To initialize: 16 | 17 | require 'salesforce_bulk' 18 | salesforce = SalesforceBulk::Api.new("YOUR_SALESFORCE_USERNAME", "YOUR_SALESFORCE_PASSWORD+YOUR_SALESFORCE_TOKEN") 19 | 20 | To use sandbox: 21 | salesforce = SalesforceBulk::Api.new("YOUR_SALESFORCE_SANDBOX_USERNAME", "YOUR_SALESFORCE_PASSWORD+YOUR_SALESFORCE_SANDBOX_TOKEN", true) 22 | 23 | Note: the second parameter is a combination of your Salesforce token and password. So if your password is xxxx and your token is yyyy, the second parameter will be xxxxyyyy 24 | 25 | Sample operations: 26 | 27 | # Insert/Create 28 | new_account = Hash["name" => "Test Account", "type" => "Other"] # Add as many fields per record as needed. 29 | records_to_insert = Array.new 30 | records_to_insert.push(new_account) # You can add as many records as you want here, just keep in mind that Salesforce has governor limits. 31 | result = salesforce.create("Account", records_to_insert) 32 | puts "result is: #{result.inspect}" 33 | 34 | # Update 35 | updated_account = Hash["name" => "Test Account -- Updated", "id" => "a00A0001009zA2m"] # Nearly identical to an insert, but we need to pass the salesforce id. 36 | records_to_update = Array.new 37 | records_to_update.push(updated_account) 38 | salesforce.update("Account", records_to_update) 39 | 40 | # Upsert 41 | upserted_account = Hash["name" => "Test Account -- Upserted", "External_Field_Name" => "123456"] # Fields to be updated. External field must be included 42 | records_to_upsert = Array.new 43 | records_to_upsert.push(upserted_account) 44 | salesforce.upsert("Account", records_to_upsert, "External_Field_Name") # Note that upsert accepts an extra parameter for the external field name 45 | 46 | OR 47 | 48 | salesforce.upsert("Account", records_to_upsert, "External_Field_Name", true) # last parameter indicates whether to wait until the batch finishes 49 | 50 | # Delete 51 | deleted_account = Hash["id" => "a00A0001009zA2m"] # We only specify the id of the records to delete 52 | records_to_delete = Array.new 53 | records_to_delete.push(deleted_account) 54 | salesforce.delete("Account", records_to_delete) 55 | 56 | # Query 57 | res = salesforce.query("Account", "select id, name, createddate from Account limit 3") # We just need to pass the sobject name and the query string 58 | puts res.result.records.inspect 59 | 60 | Result reporting: 61 | 62 | new_account = Hash["type" => "Other"] # Missing required field "name." 63 | records_to_insert = Array.new 64 | records_to_insert.push(new_account) # You can add as many records as you want here, just keep in mind that Salesforce has governor limits. 65 | result = salesforce.create("Account", records_to_insert, true) # Result reporting can only be used if wait is set to true 66 | puts result.success? # false 67 | puts result.has_errors? # true 68 | puts result.result.errors # An indexed hash detailing the errors 69 | 70 | == Copyright 71 | 72 | Copyright (c) 2012 Jorge Valdivia. 73 | 74 | ===end 75 | 76 | -------------------------------------------------------------------------------- /lib/salesforce_bulk/connection.rb: -------------------------------------------------------------------------------- 1 | module SalesforceBulk 2 | 3 | class Connection 4 | 5 | @@XML_HEADER = '' 6 | @@API_VERSION = nil 7 | @@LOGIN_HOST = 'login.salesforce.com' 8 | @@INSTANCE_HOST = nil # Gets set in login 9 | 10 | def initialize(username, password, api_version, in_sandbox) 11 | @username = username 12 | @password = password 13 | @session_id = nil 14 | @server_url = nil 15 | @instance = nil 16 | @@API_VERSION = api_version 17 | @@LOGIN_PATH = "/services/Soap/u/#{@@API_VERSION}" 18 | @@PATH_PREFIX = "/services/async/#{@@API_VERSION}/" 19 | @@LOGIN_HOST = 'test.salesforce.com' if in_sandbox 20 | 21 | login 22 | end 23 | 24 | #private 25 | 26 | def login 27 | 28 | xml = '' 29 | xml += "" 32 | xml += " " 33 | xml += " " 34 | xml += " #{@username}" 35 | xml += " #{@password}" 36 | xml += " " 37 | xml += " " 38 | xml += "" 39 | 40 | headers = Hash['Content-Type' => 'text/xml; charset=utf-8', 'SOAPAction' => 'login'] 41 | 42 | response = post_xml(@@LOGIN_HOST, @@LOGIN_PATH, xml, headers) 43 | # response_parsed = XmlSimple.xml_in(response) 44 | response_parsed = parse_response response 45 | 46 | @session_id = response_parsed['Body'][0]['loginResponse'][0]['result'][0]['sessionId'][0] 47 | @server_url = response_parsed['Body'][0]['loginResponse'][0]['result'][0]['serverUrl'][0] 48 | @instance = parse_instance 49 | 50 | @@INSTANCE_HOST = "#{@instance}.salesforce.com" 51 | end 52 | 53 | def post_xml(host, path, xml, headers) 54 | 55 | host = host || @@INSTANCE_HOST 56 | 57 | if host != @@LOGIN_HOST # Not login, need to add session id to header 58 | headers['X-SFDC-Session'] = @session_id; 59 | path = "#{@@PATH_PREFIX}#{path}" 60 | end 61 | 62 | https(host).post(path, xml, headers).body 63 | end 64 | 65 | def get_request(host, path, headers) 66 | host = host || @@INSTANCE_HOST 67 | path = "#{@@PATH_PREFIX}#{path}" 68 | 69 | if host != @@LOGIN_HOST # Not login, need to add session id to header 70 | headers['X-SFDC-Session'] = @session_id; 71 | end 72 | 73 | https(host).get(path, headers).body 74 | end 75 | 76 | def https(host) 77 | req = Net::HTTP.new(host, 443) 78 | req.use_ssl = true 79 | req.verify_mode = OpenSSL::SSL::VERIFY_NONE 80 | req 81 | end 82 | 83 | def parse_instance 84 | unless @server_url =~ /https:\/\/([a-z]{2,2}[0-9]{1,2})(-api)?/ 85 | @server_url =~ /https:\/\/[^.]*\.([a-z]{2,2}[0-9]{1,2}).my.salesforce.com/ 86 | end 87 | if $~.nil? 88 | unless @instance = @server_url.split(".salesforce.com")[0].split("://")[1] 89 | raise "Unable to parse instance from serverUrl: #{@server_url}" 90 | end 91 | else 92 | @instance = $~.captures[0] 93 | end 94 | @instance 95 | end 96 | 97 | def parse_response(response) 98 | response_parsed = XmlSimple.xml_in(response) 99 | 100 | if response.downcase.include?("faultstring") || response.downcase.include?("exceptionmessage") 101 | begin 102 | 103 | if response.downcase.include?("faultstring") 104 | error_message = response_parsed["Body"][0]["Fault"][0]["faultstring"][0] 105 | elsif response.downcase.include?("exceptionmessage") 106 | error_message = response_parsed["exceptionMessage"][0] 107 | end 108 | 109 | rescue 110 | raise "An unknown error has occured within the salesforce_bulk gem. This is most likely caused by bad request, but I am unable to parse the correct error message. Here is a dump of the response for your convenience. #{response}" 111 | end 112 | 113 | raise error_message 114 | end 115 | 116 | response_parsed 117 | end 118 | 119 | end 120 | 121 | end 122 | -------------------------------------------------------------------------------- /lib/salesforce_bulk/job.rb: -------------------------------------------------------------------------------- 1 | module SalesforceBulk 2 | 3 | class Job 4 | 5 | attr :result 6 | 7 | def initialize(operation, sobject, records, external_field, connection) 8 | 9 | @@operation = operation 10 | @@sobject = sobject 11 | @@external_field = external_field 12 | @@records = records 13 | @@connection = connection 14 | @@XML_HEADER = '' 15 | 16 | # @result = {"errors" => [], "success" => nil, "records" => [], "raw" => nil, "message" => 'The job has been queued.'} 17 | @result = JobResult.new 18 | 19 | end 20 | 21 | def create_job() 22 | xml = "#{@@XML_HEADER}" 23 | xml += "#{@@operation}" 24 | xml += "#{@@sobject}" 25 | if !@@external_field.nil? # This only happens on upsert 26 | xml += "#{@@external_field}" 27 | end 28 | xml += "CSV" 29 | xml += "" 30 | 31 | path = "job" 32 | headers = Hash['Content-Type' => 'application/xml; charset=utf-8'] 33 | 34 | response = @@connection.post_xml(nil, path, xml, headers) 35 | response_parsed = @@connection.parse_response response 36 | 37 | @@job_id = response_parsed['id'][0] 38 | end 39 | 40 | def close_job() 41 | xml = "#{@@XML_HEADER}" 42 | xml += "Closed" 43 | xml += "" 44 | 45 | path = "job/#{@@job_id}" 46 | headers = Hash['Content-Type' => 'application/xml; charset=utf-8'] 47 | 48 | response = @@connection.post_xml(nil, path, xml, headers) 49 | response_parsed = XmlSimple.xml_in(response) 50 | 51 | #job_id = response_parsed['id'][0] 52 | end 53 | 54 | def add_query 55 | path = "job/#{@@job_id}/batch/" 56 | headers = Hash["Content-Type" => "text/csv; charset=UTF-8"] 57 | 58 | response = @@connection.post_xml(nil, path, @@records, headers) 59 | response_parsed = XmlSimple.xml_in(response) 60 | 61 | @@batch_id = response_parsed['id'][0] 62 | end 63 | 64 | def add_batch() 65 | keys = @@records.first.keys 66 | 67 | output_csv = keys.to_csv 68 | 69 | @@records.each do |r| 70 | fields = Array.new 71 | keys.each do |k| 72 | fields.push(r[k]) 73 | end 74 | 75 | row_csv = fields.to_csv 76 | output_csv += row_csv 77 | end 78 | 79 | path = "job/#{@@job_id}/batch/" 80 | headers = Hash["Content-Type" => "text/csv; charset=UTF-8"] 81 | 82 | response = @@connection.post_xml(nil, path, output_csv, headers) 83 | response_parsed = XmlSimple.xml_in(response) 84 | 85 | @@batch_id = response_parsed['id'][0] 86 | end 87 | 88 | def check_batch_status() 89 | path = "job/#{@@job_id}/batch/#{@@batch_id}" 90 | headers = Hash.new 91 | 92 | response = @@connection.get_request(nil, path, headers) 93 | response_parsed = XmlSimple.xml_in(response) 94 | 95 | begin 96 | #puts "check: #{response_parsed.inspect}\n" 97 | response_parsed['state'][0] 98 | rescue Exception => e 99 | #puts "check: #{response_parsed.inspect}\n" 100 | 101 | nil 102 | end 103 | end 104 | 105 | def get_batch_result() 106 | path = "job/#{@@job_id}/batch/#{@@batch_id}/result" 107 | headers = Hash["Content-Type" => "text/xml; charset=UTF-8"] 108 | 109 | response = @@connection.get_request(nil, path, headers) 110 | 111 | if(@@operation == "query") # The query op requires us to do another request to get the results 112 | response_parsed = XmlSimple.xml_in(response) 113 | result_id = response_parsed["result"][0] 114 | 115 | path = "job/#{@@job_id}/batch/#{@@batch_id}/result/#{result_id}" 116 | headers = Hash.new 117 | headers = Hash["Content-Type" => "text/xml; charset=UTF-8"] 118 | 119 | response = @@connection.get_request(nil, path, headers) 120 | 121 | end 122 | 123 | parse_results response 124 | 125 | response = response.lines.to_a[1..-1].join 126 | # csvRows = CSV.parse(response, :headers => true) 127 | end 128 | 129 | def parse_results response 130 | @result.success = true 131 | @result.raw = response.lines.to_a[1..-1].join 132 | csvRows = CSV.parse(response, :headers => true) 133 | 134 | csvRows.each_with_index do |row, index| 135 | if @@operation != "query" 136 | row["Created"] = row["Created"] == "true" ? true : false 137 | row["Success"] = row["Success"] == "true" ? true : false 138 | end 139 | 140 | @result.records.push row 141 | if row["Success"] == false 142 | @result.success = false 143 | @result.errors.push({"#{index}" => row["Error"]}) if row["Error"] 144 | end 145 | end 146 | 147 | @result.message = "The job has been closed." 148 | 149 | end 150 | 151 | end 152 | 153 | class JobResult 154 | attr_writer :errors, :success, :records, :raw, :message 155 | attr_reader :errors, :success, :records, :raw, :message 156 | 157 | def initialize 158 | @errors = [] 159 | @success = nil 160 | @records = [] 161 | @raw = nil 162 | @message = 'The job has been queued.' 163 | end 164 | 165 | def success? 166 | @success 167 | end 168 | 169 | def has_errors? 170 | @errors.count > 0 171 | end 172 | end 173 | 174 | end 175 | --------------------------------------------------------------------------------