├── .github └── workflows │ └── ruby.yml ├── .gitignore ├── Gemfile ├── README.rdoc ├── Rakefile ├── lib ├── shopify-api-throttle.rb └── shopify-api-throttle │ ├── shopify_api │ ├── base.rb │ ├── limits.rb │ └── throttle.rb │ └── version.rb ├── shopify-api-throttle.gemspec └── spec ├── boot.rb ├── credits_spec.rb ├── query_spec.rb └── shopify_api.yml.example /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake 6 | # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby 7 | 8 | name: Ruby 9 | 10 | on: 11 | push: 12 | branches: [ master ] 13 | pull_request: 14 | branches: [ master ] 15 | 16 | jobs: 17 | test: 18 | 19 | runs-on: ubuntu-latest 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | - name: Set up Ruby 24 | # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, 25 | # change this to (see https://github.com/ruby/setup-ruby#versioning): 26 | uses: ruby/setup-ruby@v1 27 | # uses: ruby/setup-ruby@ec106b438a1ff6ff109590de34ddc62c540232e0 28 | with: 29 | ruby-version: 2.6 30 | - name: Install dependencies 31 | run: bundle install 32 | - name: Run tests 33 | env: # Or as an environment variable 34 | SHOPIFY_API_YAML: ${{ secrets.SHOPIFY_API_YAML }} 35 | run: bundle exec rspec 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | spec/shopify_api.yml 6 | 7 | # Ignore all logfiles and tempfiles. 8 | /log/*.log 9 | /tmp 10 | /.idea 11 | 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | # Specify your gem's dependencies in shopify-api-throttle.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = shopify-api-throttle 2 | 3 | This is a gem for dealing with the API limits placed on the Shopify API. 4 | 5 | == Problem 6 | 7 | Currently Shopify a leaky bucket limiter when using the API, if you go over that limit it will throw an exception. 8 | 9 | == Solution 10 | 11 | By wrapping all the calls in a throttling method it is possible to wait until the API limit is reset or retry failing calls. 12 | 13 | == Installation 14 | gem "shopify_api" 15 | gem "shopify-api-throttle" 16 | 17 | == Usage 18 | 19 | Background workers will want to ensure that all of their calls are completed, even if that results in some delays in processing. 20 | To handle this a new method has been added that executes a block. 21 | 22 | ShopifyAPI.throttle { shopifyObject.save! } 23 | 24 | Remember scoping issues also apply: 25 | 26 | collection = nil 27 | ShopifyAPI.throttle { collection = ShopifyAPI::CustomCollection.find(123) } 28 | 29 | Or much easier 30 | 31 | collection = ShopifyAPI.throttle { ShopifyAPI::CustomCollection.find(123) } 32 | 33 | The block will be retried up to a number times, and will sleep for an increasing number of seconds between retries. 34 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler::GemHelper.install_tasks 3 | -------------------------------------------------------------------------------- /lib/shopify-api-throttle.rb: -------------------------------------------------------------------------------- 1 | $:.unshift File.dirname(__FILE__) 2 | 3 | module ShopifyAPI 4 | module Throttle 5 | # Connection hack 6 | require 'shopify_api' 7 | #require 'shopify-api-limits/shopify_api/base' 8 | 9 | require 'shopify-api-throttle/shopify_api/limits' 10 | require 'shopify-api-throttle/shopify_api/throttle' 11 | 12 | def self.included(klass) 13 | klass.send(:extend, ClassMethods) 14 | end 15 | 16 | class Error < StandardError; end 17 | class GlobalError < Error; end 18 | class ShopError < Error; end 19 | 20 | end 21 | end 22 | 23 | ShopifyAPI.send(:include, ShopifyAPI::Throttle) 24 | -------------------------------------------------------------------------------- /lib/shopify-api-throttle/shopify_api/base.rb: -------------------------------------------------------------------------------- 1 | require 'active_resource' 2 | ## 3 | # Redefines #find_every to automatically compose resultsets from multiple ShopifyAPI queries due to API limit of 250 records / request. 4 | # Seemlessly stitches all requests to #all, #find(:all), etc, as if there were no LIMIT. 5 | # @see http://wiki.shopify.com/Retrieving_more_than_250_Products%2C_Orders_etc. 6 | # 7 | module ShopifyAPI 8 | class Base < ActiveResource::Base 9 | SHOPIFY_API_MAX_LIMIT = 250 10 | 11 | class << self 12 | # get reference to unbound class-method #find_every 13 | find_every = self.instance_method(:find_every) 14 | 15 | define_method(:find_every) do |options| 16 | options[:params] ||= {} 17 | 18 | # Determine number of ShopifyAPI requests to stitch together all records of this query. 19 | limit = options[:params][:limit] 20 | 21 | # Bail out to default functionality unless limit == false 22 | return find_every.bind(self).call(options) unless limit == false 23 | 24 | total = count(options).to_f 25 | options[:params].update(:limit => SHOPIFY_API_MAX_LIMIT) 26 | pages = (total/SHOPIFY_API_MAX_LIMIT).ceil 27 | 28 | # raise Limits::Error if not enough credits to retrieve entire recordset 29 | raise ShopifyAPI::Limits::Error.new if ShopifyAPI.credit_maxed? 30 | 31 | # Iterate from 1 -> max-pages and call the original #find_every, capturing the responses into one list 32 | 33 | rs = [] 34 | 1.upto(pages) {|page| 35 | options[:params].update(page: page) 36 | rs.concat find_every.bind(self).call(options) 37 | } 38 | rs 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/shopify-api-throttle/shopify_api/limits.rb: -------------------------------------------------------------------------------- 1 | module ShopifyAPI 2 | module Throttle 3 | module ClassMethods 4 | 5 | RETRY_AFTER_HEADER = 'retry-after' 6 | 7 | RETRY_AFTER = 10 8 | 9 | CREDIT_LIMIT_HEADER_PARAM = 'X-Shopify-Shop-Api-Call-Limit' 10 | 11 | ## 12 | # Have I reached my API call limit? 13 | # @return {Boolean} 14 | # 15 | def credit_below?(required = 1) 16 | credit_left < required 17 | end 18 | 19 | ## 20 | # @return {HTTPResponse} 21 | # 22 | def response 23 | Base.connection.response || { CREDIT_LIMIT_HEADER_PARAM => '10/40', RETRY_AFTER_HEADER => 0 } 24 | end 25 | 26 | ## 27 | # How many seconds until we can retry 28 | # @return {Integer} 29 | # 30 | def retry_after 31 | @retry_after = response[RETRY_AFTER_HEADER].to_i 32 | @retry_after = @retry_after == 0 ? RETRY_AFTER : @retry_after 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/shopify-api-throttle/shopify_api/throttle.rb: -------------------------------------------------------------------------------- 1 | module ShopifyAPI 2 | module Throttle 3 | module ClassMethods 4 | THROTTLE_RETRY_AFTER = 10 5 | THROTTLE_RETRY_MAX = 5 6 | THROTTLE_MIN_CREDIT = 10 7 | 8 | def throttle(&block) 9 | retried ||= 0 10 | begin 11 | if credit_below_safe?(THROTTLE_MIN_CREDIT) 12 | sleep_for = [[THROTTLE_MIN_CREDIT - ShopifyAPI.credit_left, THROTTLE_RETRY_AFTER].min, 1].max 13 | puts "Credit Maxed: #{ShopifyAPI.credit_left}/#{ShopifyAPI.credit_limit}, sleeping for #{sleep_for} seconds" 14 | sleep sleep_for 15 | 16 | if $shopify_store && $shopify_store.respond_to?(:on_throttled) # Use this to call back into your application when throttled 17 | $shopify_store.on_throttled 18 | end 19 | end 20 | 21 | if $shopify_store && $shopify_store.respond_to?(:on_request) # Use this to call back into your application when a request happens 22 | $shopify_store.on_request 23 | end 24 | 25 | yield 26 | rescue ActiveResource::ResourceNotFound, ActiveResource::BadRequest, ActiveResource::UnauthorizedAccess, 27 | ActiveResource::ForbiddenAccess, ActiveResource::MethodNotAllowed, ActiveResource::ResourceGone, 28 | ActiveResource::ResourceConflict, ActiveResource::ResourceInvalid 29 | raise 30 | rescue ActiveResource::ConnectionError, ActiveResource::ServerError, 31 | ActiveResource::ClientError, Timeout::Error, OpenSSL::SSL::SSLError, Errno::ECONNRESET => ex 32 | if retried <= THROTTLE_RETRY_MAX 33 | retry_after = ((ex.respond_to?(:response) && ex.response && ex.response['Retry-After']) || THROTTLE_RETRY_AFTER).to_i 34 | puts "Throttle Retry: #{ShopifyAPI.credit_left}/#{ShopifyAPI.credit_limit}, sleeping for #{retry_after} seconds" 35 | sleep retry_after 36 | retried += 1 37 | retry 38 | else 39 | raise 40 | end 41 | rescue => ex 42 | if ex.message =~ /Connection timed out/ 43 | sleep THROTTLE_RETRY_AFTER 44 | retried += 1 45 | retry 46 | else 47 | puts "Exception Raised: #{ex.class}" 48 | raise 49 | end 50 | end 51 | end 52 | 53 | def credit_below_safe?(value) 54 | ShopifyAPI.credit_below?(value) 55 | rescue ShopifyAPI::Limits::LimitUnavailable 56 | false 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/shopify-api-throttle/version.rb: -------------------------------------------------------------------------------- 1 | module ShopifyAPI 2 | module Throttle 3 | VERSION = '0.0.10' 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /shopify-api-throttle.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "shopify-api-throttle/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "shopify-api-throttle" 7 | s.version = ShopifyAPI::Throttle::VERSION 8 | s.platform = Gem::Platform::RUBY 9 | s.authors = ["Brad Rees"] 10 | s.email = ["brad@powertoolsapp.com"] 11 | s.homepage = "" 12 | s.summary = %q{This gem throttles API calls to keep within the limits of the ShopifyAPI gem} 13 | s.description = %q{This gem throttles API calls to keep within the limits of the ShopifyAPI gem} 14 | 15 | #s.rubyforge_project = "shopify-api-throttle" 16 | 17 | s.add_dependency "shopify_api", '< 10.0.0' 18 | s.add_development_dependency "rspec", '>=2.6.0' 19 | 20 | s.files = `git ls-files`.split("\n") 21 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 22 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 23 | s.require_paths = ["lib"] 24 | end 25 | -------------------------------------------------------------------------------- /spec/boot.rb: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler.require(:default, :development) 3 | 4 | require 'yaml' 5 | 6 | # Load shopify config 7 | config = ENV.fetch('SHOPIFY_API_YAML', nil) ? YAML.load(ENV.fetch('SHOPIFY_API_YAML')) : YAML.load_file(File.join(File.dirname(__FILE__), "shopify_api.yml")) 8 | 9 | begin 10 | ShopifyAPI::ApiVersion.version_lookup_mode = :raise_on_unknown 11 | ShopifyAPI::ApiVersion.fetch_known_versions 12 | ShopifyAPI::Meta.admin_versions.each do |v| 13 | ShopifyAPI::Base.api_version = v.handle 14 | end 15 | rescue StandardError => e 16 | puts e 17 | end 18 | 19 | ShopifyAPI::Base.site = config["site"] 20 | -------------------------------------------------------------------------------- /spec/credits_spec.rb: -------------------------------------------------------------------------------- 1 | 2 | require './spec/boot' 3 | 4 | describe "Throttle" do 5 | it "Can fetch local limits" do 6 | count = ShopifyAPI.credit_used 7 | limit = ShopifyAPI.credit_limit 8 | 9 | expect(count < limit).to be true 10 | expect(count > 0).to be true 11 | expect(ShopifyAPI.credit_maxed?).to be false 12 | expect(ShopifyAPI.credit_left > 0).to be true 13 | end 14 | 15 | it "Can execute up to local max" do 16 | until ShopifyAPI.credit_maxed? 17 | ShopifyAPI::Shop.current 18 | puts "avail: #{ShopifyAPI.credit_left}, maxed: #{ShopifyAPI.credit_maxed?}" 19 | end 20 | expect(ShopifyAPI.credit_maxed?).to be true 21 | expect(ShopifyAPI.credit_left == 0).to be true 22 | 23 | puts "Response:" 24 | ShopifyAPI.response.each{|header,value| puts "#{header}: #{value}" } 25 | 26 | puts "Retry after: #{ShopifyAPI.retry_after}" 27 | expect(ShopifyAPI.retry_after > 0).to be true 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /spec/query_spec.rb: -------------------------------------------------------------------------------- 1 | require './spec/boot' 2 | 3 | describe "Query" do 4 | it "Can retrieve an all query with empty params" do 5 | rs = ShopifyAPI.throttle { ShopifyAPI::Product.all } 6 | expect(rs.length == ShopifyAPI.throttle { ShopifyAPI::Product.count } || rs.length == 50).to be true 7 | end 8 | 9 | 10 | it "Can respect limit param when provided" do 11 | rs = ShopifyAPI.throttle { ShopifyAPI::Product.all(:params => {:limit => 1}) } 12 | puts "limit rs: #{rs.length}" 13 | expect(rs.length == 1).to be true 14 | end 15 | 16 | #it "Can aggregate result set by using :limit => false" do 17 | # rs = ShopifyAPI.throttle { ShopifyAPI::Product.all(:params => {:limit => false}) } 18 | # puts "len: #{rs.length}" 19 | # (rs.length == ShopifyAPI.throttle { ShopifyAPI::Product.count }).should be_true 20 | #end 21 | 22 | 23 | end -------------------------------------------------------------------------------- /spec/shopify_api.yml.example: -------------------------------------------------------------------------------- 1 | site: --------------------------------------------------------------------------------