├── .gitignore ├── .npmignore ├── .nvmrc ├── .prettierrc ├── .releaserc.json ├── .travis.yml ├── LICENSE.md ├── README.md ├── __mocks__ └── gatsby-source-filesystem.js ├── __tests__ ├── gatsby-node.js ├── testUtils.js └── utils.js ├── constants.js ├── fixtures ├── page1.json ├── page2.json └── page3.json ├── gatsby-node.js ├── generateFixtures.js ├── index.js ├── jest.config.js ├── package.json ├── testUtils.js ├── test_sites ├── gatsby_v3 │ ├── .gitignore │ ├── README.md │ ├── gatsby-config.js │ ├── package.json │ ├── src │ │ ├── images │ │ │ └── icon.png │ │ └── pages │ │ │ ├── 404.js │ │ │ └── index.js │ └── yarn.lock └── gatsby_v4 │ ├── .gitignore │ ├── README.md │ ├── gatsby-config.js │ ├── package.json │ ├── src │ ├── images │ │ └── icon.png │ └── pages │ │ ├── 404.js │ │ └── index.js │ └── yarn.lock ├── testing └── jest.setup.js ├── utils.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .DS_Store 4 | .idea 5 | .env 6 | .env.* 7 | .yalc 8 | yalc.lock 9 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .DS_Store 4 | .idea 5 | .env 6 | .yalc 7 | yalc.lock 8 | 9 | /__mocks__ 10 | /__tests__ 11 | /fixtures 12 | /test_sites 13 | /testing 14 | /.npmignore 15 | /.nvmrc 16 | /.prettierrc 17 | /.releaserc.json 18 | /.travis.yml 19 | /generateFixtures.js 20 | /jest.config.js 21 | /testUtils.js 22 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 14.18.2 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "es5" 5 | } 6 | -------------------------------------------------------------------------------- /.releaserc.json: -------------------------------------------------------------------------------- 1 | { 2 | "branches": [ 3 | "+([0-9])?(.{+([0-9]),x}).x", 4 | "master", 5 | { "name": "alpha", "prerelease": true }, 6 | { "name": "beta", "prerelease": true } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | cache: yarn 4 | 5 | jobs: 6 | include: 7 | - stage: release 8 | deploy: 9 | provider: script 10 | on: 11 | all_branches: true 12 | skip_cleanup: true 13 | script: 14 | - npx semantic-release 15 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Dylan On 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gatsby-source-etsy 🛍 2 | 3 | **Note: This plugin is no longer maintained.** 4 | 5 | [![Current npm package version](https://img.shields.io/npm/v/gatsby-source-etsy)](https://www.npmjs.com/package/gatsby-source-etsy) 6 | 7 | Downloads listing info and images from your Etsy shop! 8 | 9 | ## Installation 10 | 11 | ### Sites on Gatsby v3+ 12 | 13 | Install the package from npm: 14 | 15 | `npm i gatsby-source-etsy` 16 | 17 | Install peer dependencies: 18 | 19 | `npm i gatsby-source-filesystem` 20 | 21 | ### Sites on Gatsby v2 22 | 23 | Install version 1 from npm: 24 | 25 | `npm i gatsby-source-etsy@release-1.x` 26 | 27 | ### Sites on Gatsby v1 28 | 29 | Gatsby v1 is not supported. 30 | 31 | ## Configuration 32 | 33 | Next, add the plugin to your `gatsby-config.js` file: 34 | 35 | ```javascript 36 | module.exports = { 37 | plugins: [ 38 | { 39 | resolve: 'gatsby-source-etsy', 40 | options: { 41 | api_key: 'your api key here', 42 | shop_id: 'your shop id here', 43 | // The following properties are optional - Most of them narrow the results returned from Etsy. 44 | // 45 | // You don't have to use them, and in fact, you probably shouldn't! 46 | // You're probably here because you need to source *all* your listings. 47 | language: 'en', 48 | translate_keywords: true, 49 | keywords: 'coffee', 50 | sort_on: 'created', 51 | sort_order: 'up', 52 | min_price: 0.01, 53 | max_price: 999.99, 54 | color: '#333333', 55 | color_accuracy: 0, 56 | tags: 'diy,coffee,brewing', 57 | taxonomy_id: 18, 58 | include_private: true, 59 | }, 60 | }, 61 | ], 62 | } 63 | ``` 64 | 65 | This plugin supports the options specified in Etsy's documentation under [findAllShopListingsActive](https://www.etsy.com/developers/documentation/reference/listing#method_findallshoplistingsactive). 66 | 67 | For information on the `language` and `translate_keywords` properties, please see [Searching Listings](https://www.etsy.com/developers/documentation/reference/listing#section_searching_listings). 68 | 69 | ## Example GraphQL queries 70 | 71 | Listing info: 72 | 73 | ```graphql 74 | { 75 | allEtsyListing(sort: { fields: featured_rank, order: ASC }, limit: 4) { 76 | nodes { 77 | currency_code 78 | title 79 | listing_id 80 | price 81 | url 82 | } 83 | } 84 | } 85 | ``` 86 | 87 | Query transformed/optimized images for a listing (e.g. for use with `gatsby-image` - see below): 88 | 89 | ```graphql 90 | { 91 | allEtsyListing(sort: { fields: featured_rank, order: ASC }, limit: 4) { 92 | nodes { 93 | childrenEtsyListingImage { 94 | rank 95 | childFile { 96 | childImageSharp { 97 | fluid { 98 | base64 99 | tracedSVG 100 | aspectRatio 101 | src 102 | srcSet 103 | srcWebp 104 | srcSetWebp 105 | originalName 106 | originalImg 107 | presentationHeight 108 | presentationWidth 109 | sizes 110 | } 111 | } 112 | } 113 | } 114 | } 115 | } 116 | } 117 | ``` 118 | 119 | ## Queryable entities 120 | 121 | - allEtsyListing 122 | - allEtsyListingImage 123 | - etsyListing 124 | - childrenEtsyListingImage 125 | - etsyListingImage 126 | - childFile 127 | 128 | ## Usage with `gatsby-image` 129 | 130 | Install the necessary packages: 131 | 132 | `npm install gatsby-image gatsby-plugin-sharp gatsby-transformer-sharp` 133 | 134 | Query: 135 | 136 | ```graphql 137 | { 138 | etsyListing { 139 | childrenEtsyListingImage { 140 | childFile { 141 | childImageSharp { 142 | fluid { 143 | ...GatsbyImageSharpFluid 144 | } 145 | } 146 | } 147 | } 148 | } 149 | } 150 | ``` 151 | 152 | See [`gatsby-image`](https://www.gatsbyjs.org/packages/gatsby-image/) for more. 153 | 154 | ## Contributing 155 | 156 | Did something break, or is there additional functionality you'd like to add to this plugin? Consider contributing to this project! 157 | 158 | Feel free to open an issue to discuss what's happening first, or dive right in and open a PR. 159 | 160 | ### Developing this plugin locally 161 | 162 | You can use `yalc` to test changes you make to this plugin against a local Gatsby site: 163 | 164 | ```bash 165 | # Install yalc globally on your system 166 | yarn global add yalc 167 | 168 | # Publish the package to your local repository 169 | # (Run this from this repo's root directory) 170 | yalc publish 171 | 172 | # Use the package from your local repository instead of one from npm 173 | # (Run this from your Gatsby site's root directory) 174 | yalc add gatsby-source-etsy 175 | ``` 176 | 177 | For up-to-date information and troubleshooting, see `yalc`'s [documentation](https://github.com/wclr/yalc). 178 | -------------------------------------------------------------------------------- /__mocks__/gatsby-source-filesystem.js: -------------------------------------------------------------------------------- 1 | const createRemoteFileNode = jest.fn(async () => { 2 | return { 3 | id: 'fileNode1', 4 | } 5 | }) 6 | 7 | module.exports = { 8 | createRemoteFileNode, 9 | } 10 | -------------------------------------------------------------------------------- /__tests__/gatsby-node.js: -------------------------------------------------------------------------------- 1 | const nock = require('nock') 2 | const utilsMock = require('../utils') 3 | const { 4 | ETSY_BASE_URL, 5 | ETSY_FETCH_CONFIG, 6 | ETSY_PAGE_LIMIT, 7 | } = require('../constants') 8 | const { sourceNodes } = require('../gatsby-node') 9 | const page1 = require('../fixtures/page1.json') 10 | const page2 = require('../fixtures/page2.json') 11 | const page3 = require('../fixtures/page3.json') 12 | 13 | jest.mock('../utils', () => { 14 | const originalModule = jest.requireActual('../utils') 15 | return { 16 | ...originalModule, 17 | createThrottledFetch: jest.fn(originalModule.createThrottledFetch), 18 | } 19 | }) 20 | 21 | const createNode = jest.fn() 22 | const createParentChildLink = jest.fn() 23 | const touchNode = jest.fn() 24 | const actions = { 25 | createNode, 26 | createParentChildLink, 27 | touchNode, 28 | } 29 | const cache = { 30 | // Simulate empty cache 31 | get: jest.fn(async () => undefined), 32 | set: jest.fn(), 33 | } 34 | const createContentDigest = jest.fn(() => 'mockContentDigest') 35 | const createNodeId = jest.fn() 36 | const getNode = jest.fn(nodeId => { 37 | if (typeof nodeId !== 'string') { 38 | return 39 | } 40 | return { 41 | id: nodeId, 42 | } 43 | }) 44 | const reporter = { 45 | info: jest.fn(), 46 | } 47 | const store = {} 48 | const api_key = 'mockApiKey' 49 | const shop_id = 'mockShopId' 50 | const listingId = 'id1' 51 | const listingImageId = 'imageId1' 52 | const listingImageUrl = 'mockImageUrl' 53 | const listingsEndpoint = `/shops/${shop_id}/listings/active` 54 | 55 | let nockScope = nock(ETSY_BASE_URL) 56 | 57 | afterEach(() => { 58 | nock.cleanAll() 59 | }) 60 | 61 | describe('networking', () => { 62 | beforeEach(() => { 63 | nockScope 64 | .filteringPath(/\/listings\/.*\/images/, `/listings/${listingId}/images`) 65 | .persist() 66 | .get(listingsEndpoint) 67 | .query({ 68 | api_key: api_key, 69 | limit: ETSY_PAGE_LIMIT, 70 | offset: 0, 71 | }) 72 | .reply(200, page1) 73 | .get(listingsEndpoint) 74 | .query({ 75 | api_key: api_key, 76 | limit: ETSY_PAGE_LIMIT, 77 | offset: 1 * ETSY_PAGE_LIMIT, 78 | }) 79 | .reply(200, page2) 80 | .get(listingsEndpoint) 81 | .query({ 82 | api_key: api_key, 83 | limit: ETSY_PAGE_LIMIT, 84 | offset: 2 * ETSY_PAGE_LIMIT, 85 | }) 86 | .reply(200, page3) 87 | .get(`/listings/${listingId}/images`) 88 | .query({ 89 | api_key: api_key, 90 | }) 91 | .reply(200, { 92 | results: [ 93 | { 94 | listing_image_id: listingImageId, 95 | url_fullxfull: listingImageUrl, 96 | }, 97 | ], 98 | }) 99 | }) 100 | 101 | it('fetches all available listings and images', async () => { 102 | await sourceNodes( 103 | { 104 | actions, 105 | cache, 106 | createContentDigest, 107 | createNodeId, 108 | getNode, 109 | reporter, 110 | store, 111 | }, 112 | { 113 | api_key, 114 | shop_id, 115 | } 116 | ) 117 | expect(nockScope.isDone()).toBe(true) 118 | }) 119 | }) 120 | 121 | describe('processing', () => { 122 | beforeEach(() => { 123 | nockScope 124 | .persist() 125 | .get(listingsEndpoint) 126 | .query({ 127 | api_key: api_key, 128 | limit: ETSY_PAGE_LIMIT, 129 | offset: 0, 130 | }) 131 | .reply(200, { 132 | results: [ 133 | { 134 | listing_id: listingId, 135 | last_modified_tsz: 1570240827981, 136 | }, 137 | ], 138 | }) 139 | .get(listingsEndpoint) 140 | .query({ 141 | api_key: api_key, 142 | limit: ETSY_PAGE_LIMIT, 143 | offset: 1 * ETSY_PAGE_LIMIT, 144 | }) 145 | .reply(200, { 146 | results: [], 147 | }) 148 | .get(`/listings/${listingId}/images`) 149 | .query(true) 150 | .reply(200, { 151 | results: [ 152 | { 153 | listing_image_id: listingImageId, 154 | url_fullxfull: listingImageUrl, 155 | }, 156 | ], 157 | }) 158 | }) 159 | 160 | describe('when the listing is not cached', () => { 161 | it('creates a listing node and an image node', async () => { 162 | // Run test 163 | await sourceNodes( 164 | { 165 | actions, 166 | cache, 167 | createContentDigest, 168 | createNodeId, 169 | getNode, 170 | reporter, 171 | store, 172 | }, 173 | { 174 | api_key, 175 | shop_id, 176 | } 177 | ) 178 | expect(utilsMock.createThrottledFetch).toBeCalledWith(ETSY_FETCH_CONFIG) 179 | expect(cache.get).toBeCalledWith('cached-gsetsy_listing_id1') 180 | expect(touchNode).not.toBeCalled() 181 | expect(reporter.info).toBeCalledWith( 182 | 'gatsby-source-etsy: cached listing node not found, downloading gsetsy_listing_id1' 183 | ) 184 | expect(createNode).toBeCalledTimes(2) 185 | expect(createNode.mock.calls[0][0]).toEqual({ 186 | id: 'gsetsy_listing_id1', 187 | parent: null, 188 | internal: { 189 | type: 'EtsyListing', 190 | contentDigest: 'mockContentDigest', 191 | }, 192 | listing_id: 'id1', 193 | last_modified_tsz: 1570240827981, 194 | }) 195 | expect(createNode.mock.calls[1][0]).toEqual({ 196 | id: 'gsetsy_listing_id1_image_imageId1', 197 | parent: 'gsetsy_listing_id1', 198 | internal: { 199 | type: 'EtsyListingImage', 200 | contentDigest: 'mockContentDigest', 201 | }, 202 | listing_image_id: listingImageId, 203 | url_fullxfull: listingImageUrl, 204 | }) 205 | expect(createParentChildLink).toBeCalledTimes(2) 206 | expect(cache.set).toBeCalledWith('cached-gsetsy_listing_id1', { 207 | cachedListingNodeId: 'gsetsy_listing_id1', 208 | cachedImageNodeIds: ['gsetsy_listing_id1_image_imageId1'], 209 | }) 210 | }) 211 | }) 212 | 213 | describe('when the listing is cached', () => { 214 | const cacheWithListing = { 215 | // Simulate nothing being found in cache 216 | get: jest.fn(async () => { 217 | return { 218 | cachedListingNodeId: 'cached-gsetsy_listing_id1', 219 | cachedImageNodeIds: ['gsetsy_listing_id1_image_imageId1'], 220 | } 221 | }), 222 | } 223 | const mockListingNode = { 224 | id: 'gsetsy_listing_id1', 225 | listing_id: `id1`, 226 | last_modified_tsz: 1570240827981, 227 | } 228 | const mockListingImageNode = { 229 | id: 'gsetsy_listing_id1_image_imageId1', 230 | listing_id: `id1`, 231 | last_modified_tsz: 1570240827981, 232 | } 233 | const mockGetNode = nodeId => { 234 | const cachedNodes = { 235 | 'cached-gsetsy_listing_id1': mockListingNode, 236 | gsetsy_listing_id1_image_imageId1: mockListingImageNode, 237 | } 238 | return cachedNodes[nodeId] 239 | } 240 | it('uses existing nodes instead of creating nodes', async () => { 241 | // Run test 242 | await sourceNodes( 243 | { 244 | actions, 245 | cache: cacheWithListing, 246 | createContentDigest, 247 | createNodeId, 248 | getNode: mockGetNode, 249 | reporter, 250 | store, 251 | }, 252 | { 253 | api_key, 254 | shop_id, 255 | } 256 | ) 257 | expect(reporter.info).toBeCalledWith( 258 | `gatsby-source-etsy: using cached version of listing node gsetsy_listing_id1` 259 | ) 260 | expect(touchNode).toBeCalledTimes(2) 261 | expect(touchNode.mock.calls[0][0]).toEqual(mockListingNode) 262 | expect(touchNode.mock.calls[1][0]).toEqual(mockListingImageNode) 263 | expect(createNode).not.toBeCalled() 264 | }) 265 | }) 266 | }) 267 | -------------------------------------------------------------------------------- /__tests__/testUtils.js: -------------------------------------------------------------------------------- 1 | const Joi = require('joi') 2 | const { generateFakeEtsyPages } = require('../testUtils') 3 | 4 | describe('generateFakeEtsyPages', () => { 5 | it('returns the default number of pages and results', () => { 6 | const pages = generateFakeEtsyPages() 7 | expect(pages.length).toBe(2) 8 | const [firstPage] = pages 9 | expect(firstPage.results.length).toBe(100) 10 | }) 11 | 12 | it('returns the right number of pages', () => { 13 | const results = 25 14 | const perPage = 10 15 | const pages = generateFakeEtsyPages(results, perPage) 16 | expect(pages.length).toBe(4) 17 | }) 18 | 19 | it('returns the right count', () => { 20 | const results = 25 21 | const perPage = 10 22 | const pages = generateFakeEtsyPages(results, perPage) 23 | pages.forEach(page => { 24 | expect(page.count).toBe(results) 25 | }) 26 | }) 27 | 28 | it('returns the right pagination info', () => { 29 | const results = 25 30 | const perPage = 10 31 | const pages = generateFakeEtsyPages(results, perPage) 32 | pages.forEach((page, i) => { 33 | const { pagination, params } = page 34 | expect(pagination.effective_limit).toBe(perPage) 35 | expect(pagination.effective_offset).toBe(perPage * i) 36 | expect(pagination.effective_page).toBe(i + 1) 37 | expect(params.limit).toBe(String(perPage)) 38 | expect(params.offset).toBe(String(perPage * i)) 39 | if (i === pages.length - 1) { 40 | expect(pagination.next_offset).toBe(null) 41 | expect(pagination.next_page).toBe(null) 42 | } else { 43 | expect(pagination.next_offset).toBe(perPage * (i + 1)) 44 | expect(pagination.next_page).toBe(i + 2) 45 | } 46 | }) 47 | }) 48 | 49 | it('returns data in the right schema', async () => { 50 | const schema = Joi.object({ 51 | count: Joi.number() 52 | .integer() 53 | .min(0), 54 | pagination: { 55 | effective_limit: Joi.number() 56 | .integer() 57 | .min(0), 58 | effective_offset: Joi.number() 59 | .integer() 60 | .min(0), 61 | next_offset: Joi.number() 62 | .integer() 63 | .min(0) 64 | .allow(null), 65 | effective_page: Joi.number() 66 | .integer() 67 | .min(0), 68 | next_page: Joi.number() 69 | .integer() 70 | .min(0) 71 | .allow(null), 72 | }, 73 | params: Joi.object({ 74 | limit: Joi.string(), 75 | offset: Joi.string(), 76 | page: Joi.number() 77 | .integer() 78 | .positive() 79 | .allow(null), 80 | shop_id: Joi.string(), 81 | keywords: null, 82 | sort_on: Joi.string(), 83 | sort_order: Joi.string(), 84 | min_price: null, 85 | max_price: null, 86 | color: null, 87 | color_accuracy: 0, 88 | tags: null, 89 | taxonomy_id: null, 90 | translate_keywords: Joi.string(), 91 | include_private: 0, 92 | }), 93 | type: Joi.string(), 94 | results: Joi.array().items( 95 | Joi.object({ 96 | listing_id: Joi.number() 97 | .integer() 98 | .positive(), 99 | state: Joi.string(), 100 | user_id: Joi.number() 101 | .integer() 102 | .positive(), 103 | category_id: null, 104 | title: Joi.string(), 105 | description: Joi.string(), 106 | creation_tsz: Joi.date().timestamp('unix'), 107 | ending_tsz: Joi.date().timestamp('unix'), 108 | original_creation_tsz: Joi.date().timestamp('unix'), 109 | last_modified_tsz: Joi.date().timestamp('unix'), 110 | price: Joi.string(), 111 | currency_code: Joi.string(), 112 | quantity: Joi.number() 113 | .integer() 114 | .min(0), 115 | sku: Joi.array(), 116 | tags: Joi.array().items(Joi.string()), 117 | materials: Joi.array(), 118 | shop_section_id: Joi.number() 119 | .integer() 120 | .positive(), 121 | featured_rank: Joi.number() 122 | .allow(null) 123 | .integer() 124 | .positive(), 125 | state_tsz: Joi.date().timestamp('unix'), 126 | url: Joi.string().uri(), 127 | views: Joi.number() 128 | .integer() 129 | .min(0), 130 | num_favorers: Joi.number() 131 | .integer() 132 | .min(0), 133 | shipping_template_id: Joi.number() 134 | .integer() 135 | .positive(), 136 | processing_min: Joi.number() 137 | .integer() 138 | .allow(null), 139 | processing_max: Joi.number() 140 | .integer() 141 | .allow(null), 142 | who_made: Joi.string(), 143 | is_supply: Joi.string(), 144 | when_made: Joi.string(), 145 | item_weight: null, 146 | item_weight_unit: Joi.string(), 147 | item_length: null, 148 | item_width: null, 149 | item_height: null, 150 | item_dimensions_unit: Joi.string(), 151 | is_private: Joi.boolean(), 152 | recipient: null, 153 | occasion: null, 154 | style: null, 155 | non_taxable: Joi.boolean(), 156 | is_customizable: Joi.boolean(), 157 | is_digital: Joi.boolean(), 158 | file_data: '', 159 | should_auto_renew: Joi.boolean(), 160 | language: Joi.string(), 161 | has_variations: Joi.boolean(), 162 | taxonomy_id: Joi.number() 163 | .integer() 164 | .positive(), 165 | taxonomy_path: Joi.array().items(Joi.string()), 166 | used_manufacturer: Joi.boolean(), 167 | is_vintage: Joi.boolean(), 168 | }) 169 | ), 170 | }) 171 | const results = 10 172 | const perPage = 2 173 | const pages = generateFakeEtsyPages(results, perPage) 174 | const validations = pages.map(page => { 175 | return schema.validateAsync(page) 176 | }) 177 | await Promise.all(validations) 178 | }) 179 | }) 180 | -------------------------------------------------------------------------------- /__tests__/utils.js: -------------------------------------------------------------------------------- 1 | const nock = require('nock') 2 | const { 3 | ETSY_BASE_URL, 4 | ETSY_FETCH_CONFIG, 5 | ETSY_PAGE_LIMIT, 6 | } = require('../constants') 7 | const { createThrottledFetch, getListingsRecursively } = require('../utils') 8 | const page1 = require('../fixtures/page1') 9 | const page2 = require('../fixtures/page2') 10 | const page3 = require('../fixtures/page3') 11 | 12 | describe('getListingsRecursively', () => { 13 | let nockScope = nock(ETSY_BASE_URL) 14 | const api_key = 'mockApiKey' 15 | const shop_id = 'mockShopId' 16 | const listingsEndpoint = `/shops/${shop_id}/listings/active` 17 | 18 | afterEach(() => { 19 | nock.cleanAll() 20 | }) 21 | 22 | describe('with default config', () => { 23 | beforeEach(() => { 24 | nockScope 25 | .persist() 26 | .get(listingsEndpoint) 27 | .query({ 28 | api_key, 29 | limit: ETSY_PAGE_LIMIT, 30 | offset: 0, 31 | }) 32 | .reply(200, page1) 33 | .get(listingsEndpoint) 34 | .query({ 35 | api_key, 36 | limit: ETSY_PAGE_LIMIT, 37 | offset: 1 * ETSY_PAGE_LIMIT, 38 | }) 39 | .reply(200, page2) 40 | .get(listingsEndpoint) 41 | .query({ 42 | api_key, 43 | limit: ETSY_PAGE_LIMIT, 44 | offset: 2 * ETSY_PAGE_LIMIT, 45 | }) 46 | .reply(200, page3) 47 | }) 48 | 49 | it('fetches all the listings', async () => { 50 | const etsyFetch = createThrottledFetch(ETSY_FETCH_CONFIG) 51 | const listings = await getListingsRecursively(shop_id, api_key, etsyFetch) 52 | expect(nockScope.isDone()).toBe(true) 53 | expect(listings.length).toBe(101) 54 | }) 55 | }) 56 | 57 | describe('with custom config', () => { 58 | const customConfig = { 59 | keywords: 'testKeyword', 60 | sort_on: 'created', 61 | sort_order: 'up', 62 | min_price: 0.01, 63 | max_price: 999.99, 64 | color: '#333333', 65 | color_accuracy: 0, 66 | tags: 'one,two,three', 67 | taxonomy_id: 18, 68 | translate_keywords: true, 69 | include_private: true, 70 | } 71 | beforeEach(() => { 72 | nockScope 73 | .get(listingsEndpoint) 74 | .query({ 75 | api_key, 76 | limit: ETSY_PAGE_LIMIT, 77 | offset: 0, 78 | ...customConfig, 79 | }) 80 | .reply(200, page1) 81 | .get(listingsEndpoint) 82 | .query({ 83 | api_key, 84 | limit: ETSY_PAGE_LIMIT, 85 | offset: 1 * ETSY_PAGE_LIMIT, 86 | ...customConfig, 87 | }) 88 | .reply(200, page2) 89 | .get(listingsEndpoint) 90 | .query({ 91 | api_key, 92 | limit: ETSY_PAGE_LIMIT, 93 | offset: 2 * ETSY_PAGE_LIMIT, 94 | ...customConfig, 95 | }) 96 | .reply(200, { results: [] }) 97 | }) 98 | 99 | it('adds query params to the request', async () => { 100 | const etsyFetch = createThrottledFetch(ETSY_FETCH_CONFIG) 101 | await getListingsRecursively(shop_id, api_key, etsyFetch, customConfig) 102 | expect(nockScope.isDone()).toBe(true) 103 | }) 104 | }) 105 | 106 | describe('with invalid custom config', () => { 107 | beforeEach(() => { 108 | nockScope 109 | .get(listingsEndpoint) 110 | .query({ 111 | api_key, 112 | limit: ETSY_PAGE_LIMIT, 113 | offset: 0, 114 | }) 115 | .reply(200, page1) 116 | .get(listingsEndpoint) 117 | .query({ 118 | api_key, 119 | limit: ETSY_PAGE_LIMIT, 120 | offset: 1 * ETSY_PAGE_LIMIT, 121 | }) 122 | .reply(200, page2) 123 | .get(listingsEndpoint) 124 | .query({ 125 | api_key, 126 | limit: ETSY_PAGE_LIMIT, 127 | offset: 2 * ETSY_PAGE_LIMIT, 128 | }) 129 | .reply(200, { results: [] }) 130 | }) 131 | 132 | it('skips or overrides specific query params', async () => { 133 | const etsyFetch = createThrottledFetch(ETSY_FETCH_CONFIG) 134 | await getListingsRecursively(shop_id, api_key, etsyFetch, { 135 | shop_id: 'wrongShopId', 136 | api_key: 'wrongApiKey', 137 | limit: 12, 138 | offset: 999, 139 | page: 42, 140 | }) 141 | expect(nockScope.isDone()).toBe(true) 142 | }) 143 | }) 144 | }) 145 | -------------------------------------------------------------------------------- /constants.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ETSY_BASE_URL: 'https://openapi.etsy.com/v2', 3 | ETSY_PAGE_LIMIT: 100, 4 | ETSY_FETCH_CONFIG: { 5 | minTime: 150, 6 | maxConcurrent: 6, 7 | }, 8 | } 9 | -------------------------------------------------------------------------------- /fixtures/page2.json: -------------------------------------------------------------------------------- 1 | { 2 | "results": [ 3 | { 4 | "listing_id": 20408, 5 | "state": "National", 6 | "user_id": 95775, 7 | "category_id": null, 8 | "title": "perferendis neque aliquid", 9 | "description": "Facere molestias illum eaque quo. Quisquam blanditiis ad et repellendus consequatur qui dolor. Enim dolore aliquid voluptatum repellendus amet sed voluptas molestiae aut.", 10 | "creation_tsz": "2020-03-24T14:24:07.165Z", 11 | "ending_tsz": "2021-03-01T15:02:44.763Z", 12 | "original_creation_tsz": "2020-07-07T13:32:01.337Z", 13 | "last_modified_tsz": "2020-07-16T03:02:52.037Z", 14 | "price": "120.00", 15 | "currency_code": "GMD", 16 | "quantity": 43261, 17 | "sku": [], 18 | "tags": ["dolorem"], 19 | "materials": [], 20 | "shop_section_id": 52385, 21 | "featured_rank": 35572, 22 | "state_tsz": "2020-06-28T12:44:44.799Z", 23 | "url": "https://estrella.biz", 24 | "views": 43581, 25 | "num_favorers": 72972, 26 | "shipping_template_id": 70993, 27 | "processing_min": null, 28 | "processing_max": null, 29 | "who_made": "ad", 30 | "is_supply": "officiis", 31 | "when_made": "tempore", 32 | "item_weight": null, 33 | "item_weight_unit": "sed", 34 | "item_length": null, 35 | "item_width": null, 36 | "item_height": null, 37 | "item_dimensions_unit": "vel", 38 | "is_private": false, 39 | "recipient": null, 40 | "occasion": null, 41 | "style": null, 42 | "non_taxable": true, 43 | "is_customizable": true, 44 | "is_digital": false, 45 | "file_data": "", 46 | "should_auto_renew": false, 47 | "language": "nb_NO", 48 | "has_variations": true, 49 | "taxonomy_id": 4355, 50 | "taxonomy_path": ["qui"], 51 | "used_manufacturer": true, 52 | "is_vintage": false 53 | } 54 | ], 55 | "count": 101, 56 | "params": { 57 | "limit": "100", 58 | "offset": "100", 59 | "page": null, 60 | "shop_id": "c6007581-b18b-4907-96e6-14f4015227d1", 61 | "keywords": null, 62 | "sort_on": "created", 63 | "sort_order": "down", 64 | "min_price": null, 65 | "max_price": null, 66 | "color": null, 67 | "color_accuracy": 0, 68 | "tags": null, 69 | "taxonomy_id": null, 70 | "translate_keywords": "false", 71 | "include_private": 0 72 | }, 73 | "type": "Listing", 74 | "pagination": { 75 | "effective_limit": 100, 76 | "effective_offset": 100, 77 | "next_offset": 200, 78 | "effective_page": 2, 79 | "next_page": 3 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /fixtures/page3.json: -------------------------------------------------------------------------------- 1 | { 2 | "results": [], 3 | "count": 101, 4 | "params": { 5 | "limit": "100", 6 | "offset": "200", 7 | "page": null, 8 | "shop_id": "c6007581-b18b-4907-96e6-14f4015227d1", 9 | "keywords": null, 10 | "sort_on": "created", 11 | "sort_order": "down", 12 | "min_price": null, 13 | "max_price": null, 14 | "color": null, 15 | "color_accuracy": 0, 16 | "tags": null, 17 | "taxonomy_id": null, 18 | "translate_keywords": "false", 19 | "include_private": 0 20 | }, 21 | "type": "Listing", 22 | "pagination": { 23 | "effective_limit": 100, 24 | "effective_offset": 200, 25 | "next_offset": null, 26 | "effective_page": 3, 27 | "next_page": null 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /gatsby-node.js: -------------------------------------------------------------------------------- 1 | const { createRemoteFileNode } = require('gatsby-source-filesystem') 2 | const { createThrottledFetch, getListingsRecursively } = require('./utils') 3 | const { ETSY_BASE_URL, ETSY_FETCH_CONFIG } = require('./constants') 4 | 5 | exports.sourceNodes = async ( 6 | { 7 | actions, 8 | cache, 9 | createContentDigest, 10 | createNodeId, 11 | getNode, 12 | reporter, 13 | store, 14 | }, 15 | configOptions 16 | ) => { 17 | const etsyFetch = createThrottledFetch(ETSY_FETCH_CONFIG) 18 | 19 | const { createNode, createParentChildLink, touchNode } = actions 20 | const { api_key, shop_id, ...queryParams } = configOptions 21 | 22 | const listings = await getListingsRecursively( 23 | shop_id, 24 | api_key, 25 | etsyFetch, 26 | queryParams 27 | ) 28 | 29 | // * Process listings 30 | const listingProcessingJobs = listings.map(async listing => { 31 | const { listing_id } = listing 32 | const listingNodeId = `gsetsy_listing_${listing_id}` 33 | 34 | // * Check if there is a cached node for this listing 35 | const { cachedListingNodeId, cachedImageNodeIds } = 36 | (await cache.get(`cached-${listingNodeId}`)) || {} 37 | const cachedListingNode = getNode(cachedListingNodeId) 38 | if ( 39 | cachedListingNode && 40 | cachedListingNode.last_modified_tsz === listing.last_modified_tsz 41 | ) { 42 | reporter.info( 43 | `gatsby-source-etsy: using cached version of listing node ${cachedListingNode.id}` 44 | ) 45 | touchNode(cachedListingNode) 46 | cachedImageNodeIds.forEach(nodeId => touchNode(getNode(nodeId))) 47 | return 48 | } 49 | 50 | reporter.info( 51 | `gatsby-source-etsy: cached listing node not found, downloading ${listingNodeId}` 52 | ) 53 | 54 | // * Create a node for the listing 55 | await createNode({ 56 | id: listingNodeId, 57 | parent: null, 58 | internal: { 59 | type: 'EtsyListing', 60 | contentDigest: createContentDigest(listing), 61 | }, 62 | ...listing, 63 | }) 64 | 65 | // * Get images metadata for the listing 66 | const { results: images } = await etsyFetch( 67 | `${ETSY_BASE_URL}/listings/${listing_id}/images?api_key=${api_key}` 68 | ).then(res => res.json()) 69 | 70 | // * Process images 71 | const imageNodePromises = images.map(async image => { 72 | // * Create a node for each image 73 | const imageNodeId = `${listingNodeId}_image_${image.listing_image_id}` 74 | await createNode({ 75 | id: imageNodeId, 76 | parent: listingNodeId, 77 | internal: { 78 | type: 'EtsyListingImage', 79 | contentDigest: createContentDigest(image), 80 | }, 81 | ...image, 82 | }) 83 | const listingNode = getNode(listingNodeId) 84 | const imageNode = getNode(imageNodeId) 85 | createParentChildLink({ 86 | parent: listingNode, 87 | child: imageNode, 88 | }) 89 | // * Create a child node for each image file 90 | const url = image.url_fullxfull 91 | const fileNode = await createRemoteFileNode({ 92 | url, 93 | parentNodeId: imageNodeId, 94 | store, 95 | cache, 96 | createNode, 97 | createNodeId, 98 | }) 99 | createParentChildLink({ 100 | parent: imageNode, 101 | child: fileNode, 102 | }) 103 | return imageNode 104 | }) 105 | const imageNodes = await Promise.all(imageNodePromises) 106 | const imageNodeIds = imageNodes.map(node => node.id) 107 | 108 | // * Cache the listing node id and image node ids 109 | await cache.set(`cached-${listingNodeId}`, { 110 | cachedListingNodeId: listingNodeId, 111 | cachedImageNodeIds: imageNodeIds, 112 | }) 113 | }) 114 | return Promise.all(listingProcessingJobs) 115 | } 116 | -------------------------------------------------------------------------------- /generateFixtures.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const prettier = require('prettier') 3 | const { generateFakeEtsyPages } = require('./testUtils') 4 | 5 | function generateFixtures() { 6 | const pages = generateFakeEtsyPages(101, 100) 7 | 8 | pages.forEach((page, i) => { 9 | const dir = 'fixtures' 10 | if (!fs.existsSync(dir)) { 11 | fs.mkdirSync(dir) 12 | } 13 | const json = JSON.stringify(page) 14 | const content = prettier.format(json, { parser: 'json' }) 15 | fs.writeFileSync(`fixtures/page${i + 1}.json`, content) 16 | }) 17 | } 18 | 19 | generateFixtures() 20 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // no-op 2 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | clearMocks: true, 3 | setupFilesAfterEnv: ['./testing/jest.setup.js'], 4 | } 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-source-etsy", 3 | "version": "0.0.0-development", 4 | "description": "Gatsby.js plugin that sources an Etsy shop's featured listings.", 5 | "main": "index.js", 6 | "scripts": { 7 | "cz": "git-cz", 8 | "fixtures:generate": "node generateFixtures", 9 | "test": "jest --runInBand", 10 | "semantic-release": "semantic-release" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/dylanon/gatsby-source-etsy.git" 15 | }, 16 | "keywords": [ 17 | "gatsby", 18 | "gatsby-plugin", 19 | "gatsby-source", 20 | "etsy" 21 | ], 22 | "author": "Dylan On ", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/dylanon/gatsby-source-etsy/issues" 26 | }, 27 | "dependencies": { 28 | "bottleneck": "^2.19.5", 29 | "node-fetch": "^2.6.7" 30 | }, 31 | "peerDependencies": { 32 | "gatsby": "^3.0.0 || ^4.0.0", 33 | "gatsby-source-filesystem": "^3.0.0 || ^4.0.0" 34 | }, 35 | "engines": { 36 | "node": ">=14.15.0" 37 | }, 38 | "devDependencies": { 39 | "faker": "^5.5.3", 40 | "git-cz": "^3.3.0", 41 | "jest": "^24.9.0", 42 | "joi": "^17.6.0", 43 | "nock": "^13.2.4", 44 | "prettier": "1.18.2", 45 | "semantic-release": "^18.0.1" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /testUtils.js: -------------------------------------------------------------------------------- 1 | const faker = require('faker') 2 | 3 | function generateFakeEtsyPages(numberOfResults = 100, resultsPerPage = 100) { 4 | const pages = [] 5 | let workingPage = { results: [] } 6 | for (let i = 0; i < numberOfResults; i = i + 1) { 7 | workingPage.results.push({ 8 | listing_id: faker.datatype.number(), 9 | state: faker.random.word(), 10 | user_id: faker.datatype.number(), 11 | category_id: null, 12 | title: faker.lorem.words(), 13 | description: faker.lorem.sentences(), 14 | creation_tsz: faker.date.past(), 15 | ending_tsz: faker.date.future(), 16 | original_creation_tsz: faker.date.past(), 17 | last_modified_tsz: faker.date.past(), 18 | price: faker.commerce.price(), 19 | currency_code: faker.finance.currencyCode(), 20 | quantity: faker.datatype.number(), 21 | sku: [], 22 | tags: [faker.lorem.word()], 23 | materials: [], 24 | shop_section_id: faker.datatype.number(), 25 | featured_rank: faker.datatype.number(), 26 | state_tsz: faker.date.past(), 27 | url: faker.internet.url(), 28 | views: faker.datatype.number(), 29 | num_favorers: faker.datatype.number(), 30 | shipping_template_id: faker.datatype.number(), 31 | processing_min: null, 32 | processing_max: null, 33 | who_made: faker.lorem.word(), 34 | is_supply: faker.lorem.word(), 35 | when_made: faker.lorem.word(), 36 | item_weight: null, 37 | item_weight_unit: faker.lorem.word(), 38 | item_length: null, 39 | item_width: null, 40 | item_height: null, 41 | item_dimensions_unit: faker.lorem.word(), 42 | is_private: faker.datatype.boolean(), 43 | recipient: null, 44 | occasion: null, 45 | style: null, 46 | non_taxable: faker.datatype.boolean(), 47 | is_customizable: faker.datatype.boolean(), 48 | is_digital: faker.datatype.boolean(), 49 | file_data: '', 50 | should_auto_renew: faker.datatype.boolean(), 51 | language: faker.random.locale(), 52 | has_variations: faker.datatype.boolean(), 53 | taxonomy_id: faker.datatype.number(), 54 | taxonomy_path: [faker.lorem.word()], 55 | used_manufacturer: faker.datatype.boolean(), 56 | is_vintage: faker.datatype.boolean(), 57 | }) 58 | const resultNumber = i + 1 59 | if ( 60 | resultNumber % resultsPerPage === 0 || 61 | resultNumber === numberOfResults 62 | ) { 63 | const pageIndex = Math.floor(i / resultsPerPage) 64 | workingPage = { 65 | ...workingPage, 66 | count: numberOfResults, 67 | params: { 68 | limit: String(resultsPerPage), 69 | offset: String(resultsPerPage * pageIndex), 70 | page: null, 71 | shop_id: faker.datatype.uuid(), 72 | keywords: null, 73 | sort_on: 'created', 74 | sort_order: 'down', 75 | min_price: null, 76 | max_price: null, 77 | color: null, 78 | color_accuracy: 0, 79 | tags: null, 80 | taxonomy_id: null, 81 | translate_keywords: 'false', 82 | include_private: 0, 83 | }, 84 | type: 'Listing', 85 | pagination: { 86 | effective_limit: resultsPerPage, 87 | effective_offset: resultsPerPage * pageIndex, 88 | next_offset: resultsPerPage * (pageIndex + 1), 89 | effective_page: pageIndex + 1, 90 | next_page: pageIndex + 2, 91 | }, 92 | } 93 | pages.push(workingPage) 94 | workingPage = { results: [] } 95 | } 96 | } 97 | if (pages.length) { 98 | const lastPage = pages[pages.length - 1] 99 | pages.push({ 100 | ...lastPage, 101 | results: [], 102 | params: { 103 | ...lastPage.params, 104 | offset: String(Number(lastPage.params.offset) + resultsPerPage), 105 | }, 106 | pagination: { 107 | ...lastPage.pagination, 108 | effective_offset: lastPage.pagination.effective_offset + resultsPerPage, 109 | next_offset: null, 110 | effective_page: lastPage.pagination.effective_page + 1, 111 | next_page: null, 112 | }, 113 | }) 114 | } 115 | return pages 116 | } 117 | 118 | module.exports = { 119 | generateFakeEtsyPages, 120 | } 121 | -------------------------------------------------------------------------------- /test_sites/gatsby_v3/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .cache/ 3 | .env 4 | .env.development 5 | public 6 | -------------------------------------------------------------------------------- /test_sites/gatsby_v3/README.md: -------------------------------------------------------------------------------- 1 | # Gatsby v3 Test Site 2 | 3 | ## Developing 4 | 5 | In this repo's root directory: 6 | 7 | - Run `yalc publish` 8 | 9 | In this directory: 10 | 11 | - Create `.env.development` and populate with your secrets: 12 | 13 | ```bash 14 | GATSBY_ETSY_API_KEY=your-api-key 15 | GATSBY_ETSY_STORE_ID=your-store-id 16 | ``` 17 | 18 | - `yalc link gatsby-source-etsy` 19 | - `yarn develop` 20 | -------------------------------------------------------------------------------- /test_sites/gatsby_v3/gatsby-config.js: -------------------------------------------------------------------------------- 1 | require('dotenv').config({ 2 | path: `.env.${process.env.NODE_ENV}`, 3 | }) 4 | 5 | module.exports = { 6 | siteMetadata: { 7 | siteUrl: 'https://www.yourdomain.tld', 8 | title: 'Gatsby v3 Test Site', 9 | }, 10 | plugins: [ 11 | { 12 | resolve: `gatsby-source-etsy`, 13 | options: { 14 | api_key: process.env.GATSBY_ETSY_API_KEY, 15 | shop_id: process.env.GATSBY_ETSY_STORE_ID, 16 | }, 17 | }, 18 | ], 19 | } 20 | -------------------------------------------------------------------------------- /test_sites/gatsby_v3/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-v3-test-site", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "Gatsby v3 Test Site", 6 | "author": "Dylan On", 7 | "keywords": [ 8 | "gatsby" 9 | ], 10 | "scripts": { 11 | "develop": "gatsby develop", 12 | "start": "gatsby develop", 13 | "build": "gatsby build", 14 | "serve": "gatsby serve", 15 | "clean": "gatsby clean" 16 | }, 17 | "dependencies": { 18 | "gatsby": "^3.0.0", 19 | "gatsby-source-etsy": "^2.0.2", 20 | "gatsby-source-filesystem": "^3.0.0", 21 | "react": "^17.0.1", 22 | "react-dom": "^17.0.1" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /test_sites/gatsby_v3/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dylanon/gatsby-source-etsy/5a1c4775aff58f92b0a303be677e12dda2b4bb46/test_sites/gatsby_v3/src/images/icon.png -------------------------------------------------------------------------------- /test_sites/gatsby_v3/src/pages/404.js: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import { Link } from "gatsby" 3 | 4 | // styles 5 | const pageStyles = { 6 | color: "#232129", 7 | padding: "96px", 8 | fontFamily: "-apple-system, Roboto, sans-serif, serif", 9 | } 10 | const headingStyles = { 11 | marginTop: 0, 12 | marginBottom: 64, 13 | maxWidth: 320, 14 | } 15 | 16 | const paragraphStyles = { 17 | marginBottom: 48, 18 | } 19 | const codeStyles = { 20 | color: "#8A6534", 21 | padding: 4, 22 | backgroundColor: "#FFF4DB", 23 | fontSize: "1.25rem", 24 | borderRadius: 4, 25 | } 26 | 27 | // markup 28 | const NotFoundPage = () => { 29 | return ( 30 |
31 | Not found 32 |

Page not found

33 |

34 | Sorry{" "} 35 | 36 | 😔 37 | {" "} 38 | we couldn’t find what you were looking for. 39 |
40 | {process.env.NODE_ENV === "development" ? ( 41 | <> 42 |
43 | Try creating a page in src/pages/. 44 |
45 | 46 | ) : null} 47 |
48 | Go home. 49 |

50 |
51 | ) 52 | } 53 | 54 | export default NotFoundPage 55 | -------------------------------------------------------------------------------- /test_sites/gatsby_v3/src/pages/index.js: -------------------------------------------------------------------------------- 1 | import { graphql } from 'gatsby' 2 | import * as React from 'react' 3 | 4 | // styles 5 | const pageStyles = { 6 | color: '#232129', 7 | padding: 96, 8 | fontFamily: '-apple-system, Roboto, sans-serif, serif', 9 | } 10 | const headingStyles = { 11 | marginTop: 0, 12 | marginBottom: 64, 13 | maxWidth: 320, 14 | } 15 | const headingAccentStyles = { 16 | color: '#663399', 17 | } 18 | const paragraphStyles = { 19 | marginBottom: 48, 20 | } 21 | const codeStyles = { 22 | color: '#8A6534', 23 | padding: 4, 24 | backgroundColor: '#FFF4DB', 25 | fontSize: '1.25rem', 26 | borderRadius: 4, 27 | } 28 | const listStyles = { 29 | marginBottom: 96, 30 | paddingLeft: 0, 31 | } 32 | const listItemStyles = { 33 | fontWeight: 300, 34 | fontSize: 24, 35 | maxWidth: 560, 36 | marginBottom: 30, 37 | } 38 | 39 | const linkStyle = { 40 | color: '#8954A8', 41 | fontWeight: 'bold', 42 | fontSize: 16, 43 | verticalAlign: '5%', 44 | } 45 | 46 | const docLinkStyle = { 47 | ...linkStyle, 48 | listStyleType: 'none', 49 | marginBottom: 24, 50 | } 51 | 52 | const descriptionStyle = { 53 | color: '#232129', 54 | fontSize: 14, 55 | marginTop: 10, 56 | marginBottom: 0, 57 | lineHeight: 1.25, 58 | } 59 | 60 | const docLink = { 61 | text: 'Documentation', 62 | url: 'https://www.gatsbyjs.com/docs/', 63 | color: '#8954A8', 64 | } 65 | 66 | const badgeStyle = { 67 | color: '#fff', 68 | backgroundColor: '#088413', 69 | border: '1px solid #088413', 70 | fontSize: 11, 71 | fontWeight: 'bold', 72 | letterSpacing: 1, 73 | borderRadius: 4, 74 | padding: '4px 6px', 75 | display: 'inline-block', 76 | position: 'relative', 77 | top: -2, 78 | marginLeft: 10, 79 | lineHeight: 1, 80 | } 81 | 82 | // data 83 | const links = [ 84 | { 85 | text: 'Tutorial', 86 | url: 'https://www.gatsbyjs.com/docs/tutorial/', 87 | description: 88 | "A great place to get started if you're new to web development. Designed to guide you through setting up your first Gatsby site.", 89 | color: '#E95800', 90 | }, 91 | { 92 | text: 'How to Guides', 93 | url: 'https://www.gatsbyjs.com/docs/how-to/', 94 | description: 95 | "Practical step-by-step guides to help you achieve a specific goal. Most useful when you're trying to get something done.", 96 | color: '#1099A8', 97 | }, 98 | { 99 | text: 'Reference Guides', 100 | url: 'https://www.gatsbyjs.com/docs/reference/', 101 | description: 102 | "Nitty-gritty technical descriptions of how Gatsby works. Most useful when you need detailed information about Gatsby's APIs.", 103 | color: '#BC027F', 104 | }, 105 | { 106 | text: 'Conceptual Guides', 107 | url: 'https://www.gatsbyjs.com/docs/conceptual/', 108 | description: 109 | 'Big-picture explanations of higher-level Gatsby concepts. Most useful for building understanding of a particular topic.', 110 | color: '#0D96F2', 111 | }, 112 | { 113 | text: 'Plugin Library', 114 | url: 'https://www.gatsbyjs.com/plugins', 115 | description: 116 | 'Add functionality and customize your Gatsby site or app with thousands of plugins built by our amazing developer community.', 117 | color: '#8EB814', 118 | }, 119 | { 120 | text: 'Build and Host', 121 | url: 'https://www.gatsbyjs.com/cloud', 122 | badge: true, 123 | description: 124 | 'Now you’re ready to show the world! Give your Gatsby site superpowers: Build and host on Gatsby Cloud. Get started for free!', 125 | color: '#663399', 126 | }, 127 | ] 128 | 129 | // markup 130 | const IndexPage = ({ data }) => { 131 | return ( 132 |
133 | Home Page 134 |

135 | Congratulations 136 |
137 | — you just made a Gatsby site! 138 | 139 | 🎉🎉🎉 140 | 141 |

142 |

143 | Edit src/pages/index.js to see this page 144 | update in real-time.{' '} 145 | 146 | 😎 147 | 148 |

149 | 177 | Gatsby G Logo 181 |

Etsy Listings

182 | 197 |
198 | ) 199 | } 200 | 201 | export default IndexPage 202 | 203 | export const query = graphql` 204 | query ListingsQuery { 205 | allEtsyListing(sort: { fields: featured_rank, order: ASC }, limit: 4) { 206 | nodes { 207 | currency_code 208 | title 209 | listing_id 210 | price 211 | url 212 | } 213 | } 214 | } 215 | ` 216 | -------------------------------------------------------------------------------- /test_sites/gatsby_v4/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .cache/ 3 | .env 4 | .env.development 5 | public 6 | -------------------------------------------------------------------------------- /test_sites/gatsby_v4/README.md: -------------------------------------------------------------------------------- 1 | # Gatsby v4 Test Site 2 | 3 | ## Developing 4 | 5 | In this repo's root directory: 6 | 7 | - Run `yalc publish` 8 | 9 | In this directory: 10 | 11 | - Create `.env.development` and populate with your secrets: 12 | 13 | ```bash 14 | GATSBY_ETSY_API_KEY=your-api-key 15 | GATSBY_ETSY_STORE_ID=your-store-id 16 | ``` 17 | 18 | - `yalc link gatsby-source-etsy` 19 | - `yarn develop` 20 | -------------------------------------------------------------------------------- /test_sites/gatsby_v4/gatsby-config.js: -------------------------------------------------------------------------------- 1 | require('dotenv').config({ 2 | path: `.env.${process.env.NODE_ENV}`, 3 | }) 4 | 5 | module.exports = { 6 | siteMetadata: { 7 | siteUrl: 'https://www.yourdomain.tld', 8 | title: 'Gatsby v4 Test Site', 9 | }, 10 | plugins: [ 11 | { 12 | resolve: `gatsby-source-etsy`, 13 | options: { 14 | api_key: process.env.GATSBY_ETSY_API_KEY, 15 | shop_id: process.env.GATSBY_ETSY_STORE_ID, 16 | }, 17 | }, 18 | ], 19 | } 20 | -------------------------------------------------------------------------------- /test_sites/gatsby_v4/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-v4-test-site", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "Gatsby v4 Test Site", 6 | "author": "Dylan On", 7 | "keywords": [ 8 | "gatsby" 9 | ], 10 | "scripts": { 11 | "develop": "gatsby develop", 12 | "start": "gatsby develop", 13 | "build": "gatsby build", 14 | "serve": "gatsby serve", 15 | "clean": "gatsby clean" 16 | }, 17 | "dependencies": { 18 | "gatsby": "^4.4.0", 19 | "gatsby-source-etsy": "^2.0.2", 20 | "gatsby-source-filesystem": "^4.0.0", 21 | "react": "^17.0.1", 22 | "react-dom": "^17.0.1" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /test_sites/gatsby_v4/src/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dylanon/gatsby-source-etsy/5a1c4775aff58f92b0a303be677e12dda2b4bb46/test_sites/gatsby_v4/src/images/icon.png -------------------------------------------------------------------------------- /test_sites/gatsby_v4/src/pages/404.js: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import { Link } from "gatsby" 3 | 4 | // styles 5 | const pageStyles = { 6 | color: "#232129", 7 | padding: "96px", 8 | fontFamily: "-apple-system, Roboto, sans-serif, serif", 9 | } 10 | const headingStyles = { 11 | marginTop: 0, 12 | marginBottom: 64, 13 | maxWidth: 320, 14 | } 15 | 16 | const paragraphStyles = { 17 | marginBottom: 48, 18 | } 19 | const codeStyles = { 20 | color: "#8A6534", 21 | padding: 4, 22 | backgroundColor: "#FFF4DB", 23 | fontSize: "1.25rem", 24 | borderRadius: 4, 25 | } 26 | 27 | // markup 28 | const NotFoundPage = () => { 29 | return ( 30 |
31 | Not found 32 |

Page not found

33 |

34 | Sorry{" "} 35 | 36 | 😔 37 | {" "} 38 | we couldn’t find what you were looking for. 39 |
40 | {process.env.NODE_ENV === "development" ? ( 41 | <> 42 |
43 | Try creating a page in src/pages/. 44 |
45 | 46 | ) : null} 47 |
48 | Go home. 49 |

50 |
51 | ) 52 | } 53 | 54 | export default NotFoundPage 55 | -------------------------------------------------------------------------------- /test_sites/gatsby_v4/src/pages/index.js: -------------------------------------------------------------------------------- 1 | import { graphql } from 'gatsby' 2 | import * as React from 'react' 3 | 4 | // styles 5 | const pageStyles = { 6 | color: '#232129', 7 | padding: 96, 8 | fontFamily: '-apple-system, Roboto, sans-serif, serif', 9 | } 10 | const headingStyles = { 11 | marginTop: 0, 12 | marginBottom: 64, 13 | maxWidth: 320, 14 | } 15 | const headingAccentStyles = { 16 | color: '#663399', 17 | } 18 | const paragraphStyles = { 19 | marginBottom: 48, 20 | } 21 | const codeStyles = { 22 | color: '#8A6534', 23 | padding: 4, 24 | backgroundColor: '#FFF4DB', 25 | fontSize: '1.25rem', 26 | borderRadius: 4, 27 | } 28 | const listStyles = { 29 | marginBottom: 96, 30 | paddingLeft: 0, 31 | } 32 | const listItemStyles = { 33 | fontWeight: 300, 34 | fontSize: 24, 35 | maxWidth: 560, 36 | marginBottom: 30, 37 | } 38 | 39 | const linkStyle = { 40 | color: '#8954A8', 41 | fontWeight: 'bold', 42 | fontSize: 16, 43 | verticalAlign: '5%', 44 | } 45 | 46 | const docLinkStyle = { 47 | ...linkStyle, 48 | listStyleType: 'none', 49 | marginBottom: 24, 50 | } 51 | 52 | const descriptionStyle = { 53 | color: '#232129', 54 | fontSize: 14, 55 | marginTop: 10, 56 | marginBottom: 0, 57 | lineHeight: 1.25, 58 | } 59 | 60 | const docLink = { 61 | text: 'Documentation', 62 | url: 'https://www.gatsbyjs.com/docs/', 63 | color: '#8954A8', 64 | } 65 | 66 | const badgeStyle = { 67 | color: '#fff', 68 | backgroundColor: '#088413', 69 | border: '1px solid #088413', 70 | fontSize: 11, 71 | fontWeight: 'bold', 72 | letterSpacing: 1, 73 | borderRadius: 4, 74 | padding: '4px 6px', 75 | display: 'inline-block', 76 | position: 'relative', 77 | top: -2, 78 | marginLeft: 10, 79 | lineHeight: 1, 80 | } 81 | 82 | // data 83 | const links = [ 84 | { 85 | text: 'Tutorial', 86 | url: 'https://www.gatsbyjs.com/docs/tutorial/', 87 | description: 88 | "A great place to get started if you're new to web development. Designed to guide you through setting up your first Gatsby site.", 89 | color: '#E95800', 90 | }, 91 | { 92 | text: 'How to Guides', 93 | url: 'https://www.gatsbyjs.com/docs/how-to/', 94 | description: 95 | "Practical step-by-step guides to help you achieve a specific goal. Most useful when you're trying to get something done.", 96 | color: '#1099A8', 97 | }, 98 | { 99 | text: 'Reference Guides', 100 | url: 'https://www.gatsbyjs.com/docs/reference/', 101 | description: 102 | "Nitty-gritty technical descriptions of how Gatsby works. Most useful when you need detailed information about Gatsby's APIs.", 103 | color: '#BC027F', 104 | }, 105 | { 106 | text: 'Conceptual Guides', 107 | url: 'https://www.gatsbyjs.com/docs/conceptual/', 108 | description: 109 | 'Big-picture explanations of higher-level Gatsby concepts. Most useful for building understanding of a particular topic.', 110 | color: '#0D96F2', 111 | }, 112 | { 113 | text: 'Plugin Library', 114 | url: 'https://www.gatsbyjs.com/plugins', 115 | description: 116 | 'Add functionality and customize your Gatsby site or app with thousands of plugins built by our amazing developer community.', 117 | color: '#8EB814', 118 | }, 119 | { 120 | text: 'Build and Host', 121 | url: 'https://www.gatsbyjs.com/cloud', 122 | badge: true, 123 | description: 124 | 'Now you’re ready to show the world! Give your Gatsby site superpowers: Build and host on Gatsby Cloud. Get started for free!', 125 | color: '#663399', 126 | }, 127 | ] 128 | 129 | // markup 130 | const IndexPage = ({ data }) => { 131 | return ( 132 |
133 | Home Page 134 |

135 | Congratulations 136 |
137 | — you just made a Gatsby site! 138 | 139 | 🎉🎉🎉 140 | 141 |

142 |

143 | Edit src/pages/index.js to see this page 144 | update in real-time.{' '} 145 | 146 | 😎 147 | 148 |

149 | 177 | Gatsby G Logo 181 |

Etsy Listings

182 | 197 |
198 | ) 199 | } 200 | 201 | export default IndexPage 202 | 203 | export const query = graphql` 204 | query ListingsQuery { 205 | allEtsyListing(sort: { fields: featured_rank, order: ASC }, limit: 4) { 206 | nodes { 207 | currency_code 208 | title 209 | listing_id 210 | price 211 | url 212 | } 213 | } 214 | } 215 | ` 216 | -------------------------------------------------------------------------------- /testing/jest.setup.js: -------------------------------------------------------------------------------- 1 | const nock = require('nock') 2 | 3 | jest.setTimeout(20000) 4 | 5 | afterAll(() => { 6 | // Prevent [nock + Jest memory leaks](https://github.com/nock/nock#memory-issues-with-jest) 7 | nock.restore() 8 | }) 9 | -------------------------------------------------------------------------------- /utils.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch') 2 | const Bottleneck = require('bottleneck') 3 | const querystring = require('querystring') 4 | const { ETSY_BASE_URL, ETSY_PAGE_LIMIT } = require('./constants') 5 | 6 | async function asyncForEach(sourceArray, callback) { 7 | for (let i = 0; i < sourceArray.length; i = i + 1) { 8 | await callback(sourceArray[i], i, sourceArray) 9 | } 10 | } 11 | 12 | function createThrottledFetch(limiterOptions = {}) { 13 | const defaultLimiterOptions = { 14 | minTime: 100, 15 | } 16 | const limiter = new Bottleneck({ 17 | ...defaultLimiterOptions, 18 | ...limiterOptions, 19 | }) 20 | function throttledFetch(...args) { 21 | return limiter.schedule(() => fetch(...args)) 22 | } 23 | return throttledFetch 24 | } 25 | 26 | async function getListingsRecursively( 27 | shop_id, 28 | api_key, 29 | etsyFetch, 30 | queryParams = {}, 31 | offset = 0 32 | ) { 33 | const { 34 | shop_id: _shop_id, 35 | page: _page, 36 | ...allowableQueryParams 37 | } = queryParams 38 | const definedQueryParams = {} 39 | Object.entries(allowableQueryParams).forEach(([key, value]) => { 40 | if (value !== undefined) { 41 | definedQueryParams[key] = value 42 | } 43 | }) 44 | const queryObject = { 45 | ...definedQueryParams, 46 | api_key: api_key, 47 | limit: ETSY_PAGE_LIMIT, 48 | offset, 49 | } 50 | const query = querystring.stringify(queryObject) 51 | const { results } = await etsyFetch( 52 | `${ETSY_BASE_URL}/shops/${shop_id}/listings/active?${query}` 53 | ).then(res => res.json()) 54 | 55 | let nextResults = [] 56 | 57 | if (results.length) { 58 | nextResults = await getListingsRecursively( 59 | shop_id, 60 | api_key, 61 | etsyFetch, 62 | queryParams, 63 | offset + ETSY_PAGE_LIMIT 64 | ) 65 | } 66 | 67 | return [...results, ...nextResults] 68 | } 69 | 70 | module.exports = { 71 | asyncForEach, 72 | createThrottledFetch, 73 | getListingsRecursively, 74 | } 75 | --------------------------------------------------------------------------------