├── lib ├── classes │ ├── ListingUser.js │ ├── Base.js │ ├── ListingSale.js │ ├── Listing.js │ ├── Histogram.js │ └── ListingItem.js └── index.js ├── .eslintrc.js ├── resources ├── currency.js ├── regex.js ├── endpoint.js └── dom.js ├── README.md ├── .gitignore ├── package.json ├── LICENSE └── examples └── all.js /lib/classes/ListingUser.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Base = require('./Base'); 4 | 5 | /** 6 | * This class is for the user market listings like /recent and /recentcompleted 7 | */ 8 | class ListingUser { 9 | constructor(listing, assets) { 10 | 11 | Object.assign(this, listing) 12 | 13 | this.assetinfo = assets[this.asset.appid][this.asset.contextid][this.asset.id] 14 | 15 | } 16 | } 17 | 18 | // Export Class 19 | module.exports = ListingUser; 20 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "extends": "standard", 3 | "installedESLint": true, 4 | "plugins": [ 5 | "standard", 6 | "promise" 7 | ], 8 | "rules": { 9 | "semi": 0, 10 | "camelcase": 0, 11 | "space-in-parens": 0, 12 | "space-unary-ops": 0, 13 | "keyword-spacing": 0, 14 | "indent": ["error", 4], 15 | "space-before-function-paren": ["error", "never"], 16 | "no-useless-escape": 0, 17 | "new-parens": 0 18 | } 19 | }; 20 | -------------------------------------------------------------------------------- /resources/currency.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 'USD': 1, 3 | 'GBP': 2, 4 | 'EUR': 3, 5 | 'CHF': 4, 6 | 'RUB': 5, 7 | 'BRL': 7, 8 | 'JPY': 8, 9 | 'SEK': 9, 10 | 'IDR': 10, 11 | 'MYR': 11, 12 | 'PHP': 12, 13 | 'SGD': 13, 14 | 'THB': 14, 15 | 'KRW': 16, 16 | 'TRY': 17, 17 | 'MXN': 19, 18 | 'CAD': 20, 19 | 'NZD': 22, 20 | 'CNY': 23, 21 | 'INR': 24, 22 | 'CLP': 25, 23 | 'PEN': 26, 24 | 'COP': 27, 25 | 'ZAR': 28, 26 | 'HKD': 29, 27 | 'TWD': 30, 28 | 'SRD': 31, 29 | 'AED': 32, 30 | 31 | getCurrencyID: function(currency) { 32 | return 2000 + currency; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Steam Market Crawler for Node.js 2 | [![npm](https://img.shields.io/npm/v/node-steam-market-crawler.svg?style=flat-square)](https://www.npmjs.com/package/node-steam-market-crawler) 3 | [![npm](https://img.shields.io/npm/dm/node-steam-market-crawler.svg?style=flat-square)](https://www.npmjs.com/package/node-steam-market-crawler) 4 | [![npm](https://img.shields.io/npm/l/express.svg?style=flat-square)](https://github.com/pepzwee/node-steam-market-crawlerblob/master/LICENSE) 5 | [![steam](https://img.shields.io/badge/steam-donate-green.svg?style=flat-square)](https://steamcommunity.com/tradeoffer/new/?partner=78261062&token=2_WUiltH) 6 | 7 | This module is designed to make Steam market scraping easier. It's usable but in development. 8 | 9 | Wiki 10 | ---- 11 | TODO 12 | 13 | License 14 | ---- 15 | MIT 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | *.pid.lock 11 | 12 | # Directory for instrumented libs generated by jscoverage/JSCover 13 | lib-cov 14 | 15 | # Coverage directory used by tools like istanbul 16 | coverage 17 | 18 | # nyc test coverage 19 | .nyc_output 20 | 21 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 22 | .grunt 23 | 24 | # node-waf configuration 25 | .lock-wscript 26 | 27 | # Compiled binary addons (http://nodejs.org/api/addons.html) 28 | build/Release 29 | 30 | # Dependency directories 31 | node_modules 32 | jspm_packages 33 | 34 | # Optional npm cache directory 35 | .npm 36 | 37 | # Optional eslint cache 38 | .eslintcache 39 | 40 | # Optional REPL history 41 | .node_repl_history 42 | 43 | # Output of 'npm pack' 44 | *.tgz 45 | 46 | # Yarn Integrity file 47 | .yarn-integrity 48 | test.js 49 | old 50 | -------------------------------------------------------------------------------- /lib/classes/Base.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | class Base { 4 | 5 | constructor($) { 6 | const dom = require('../../resources/dom')($); 7 | 8 | // URL of the item 9 | this.url = dom.url(); 10 | // Game ID 11 | this.appID = parseInt( 12 | this.splitURL(5), 10 13 | ); 14 | // Which game has this item 15 | this.game_name = $( dom.selectors.game_name ).first().text() || null; 16 | // Item names 17 | this.market_name = dom.marketName() || this.market_hash_name; 18 | this.market_hash_name = this.splitURL(6); 19 | // Item image 20 | this.image = dom.image(); 21 | } 22 | 23 | splitURL(index) { 24 | try { 25 | return decodeURIComponent(this.url.split('/')[index]).toString('utf8'); 26 | } catch(e) { 27 | throw new Error(['Failed to split the URL.']); 28 | } 29 | } 30 | } 31 | 32 | // Export Class 33 | module.exports = Base; 34 | -------------------------------------------------------------------------------- /resources/regex.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | get: function(regex, string, index, json) { 5 | let m; 6 | if((m = regex.exec(string)) !== null) { 7 | if(m.index === regex.lastIndex) { 8 | regex.lastIndex++; 9 | } 10 | let value = null; 11 | if(typeof m[index] !== 'undefined') value = m[index]; 12 | if(json && value) { 13 | try { return JSON.parse(value); } catch(e) { 14 | // Sometimes my Regex is shit and it is missing an "}" in the end, try to check again 15 | value += '}'; 16 | try { return JSON.parse(value); } catch(e) { return null; } 17 | } 18 | } 19 | return value; 20 | } 21 | }, 22 | listings: { 23 | 'appContextData': /(g_rgAppContextData) = ([\"\\']?)(.*?)\};/, 24 | 'assets': /(g_rgAssets) = ([\"\\']?)(.*?)\};/, 25 | 'medianSalePrices': /(line1)\=([^)]+)\;/, 26 | 'nameID': /(Market_LoadOrderSpread)\( ([^)]+) \)/ 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-steam-market-crawler", 3 | "version": "1.4.1", 4 | "description": "Crawl and scrape Steam market listings for data using Node.", 5 | "main": "./lib/index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "cheerio": "^0.22.0", 11 | "request": "^2.83.0", 12 | "request-promise": "^4.2.2" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "git+https://github.com/pepzwee/node-steam-market-crawler.git" 17 | }, 18 | "keywords": [ 19 | "steam", 20 | "market", 21 | "crawler", 22 | "scrape", 23 | "gaben", 24 | "steamapis" 25 | ], 26 | "author": "Kristjan Kirpu", 27 | "license": "MIT", 28 | "bugs": { 29 | "url": "https://github.com/pepzwee/node-steam-market-crawler/issues" 30 | }, 31 | "homepage": "https://github.com/pepzwee/node-steam-market-crawler#readme", 32 | "devDependencies": { 33 | "eslint": "^3.19.0", 34 | "eslint-config-standard": "^6.2.1", 35 | "eslint-plugin-promise": "^3.8.0", 36 | "eslint-plugin-standard": "^2.3.1" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /lib/classes/ListingSale.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * This class is for the user sales @ market listing page 5 | */ 6 | class ListingSale { 7 | constructor($, listingInfo) { 8 | const dom = require('../../resources/dom')($); 9 | // Sale listing id 10 | this.id = this.listing_id = $('.market_listing_row').eq(0).attr('id').replace('listing_', '') || null; 11 | // Item image 12 | this.image = dom.image(); 13 | // Item names 14 | this.market_name = dom.marketName(); 15 | // Which game has this item 16 | this.game_name = $( dom.selectors.game_name ).first().text() || null; 17 | // Seller avatar 18 | this.sellerAvatar = dom.sellerAvatar(); 19 | // Item border color 20 | this.border_color = dom.borderColor(); 21 | // Sale prices 22 | this.normal_price = dom.normalPrice(); 23 | this.sale_price = dom.salePrice(); 24 | // listinginfo if possible 25 | if(this.id) { 26 | this.listinginfo = listingInfo[this.id]; 27 | } 28 | } 29 | } 30 | 31 | // Export Class 32 | module.exports = ListingSale; 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Kristjan Kirpu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/classes/Listing.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Base = require('./Base'); 4 | 5 | /** 6 | * This class is for the standard market listings like /search and /popular 7 | */ 8 | class Listing extends Base { 9 | constructor($, parameters, popularity) { 10 | super($); 11 | 12 | const dom = require('../../resources/dom')($); 13 | 14 | // Item border color 15 | this.border_color = dom.borderColor(); 16 | // Amount being sold on the market currently 17 | this.quantity = parseInt( 18 | $( dom.selectors.quantity ).text(), 10 19 | ) || 0; 20 | // Prices 21 | this.normal_price = dom.normalPrice(); 22 | this.sale_price = dom.salePrice(); 23 | 24 | // Add popularity index 25 | if(popularity.use && parameters) { 26 | // Add it only when sort_column was set to `popular` and sort direction is descending 27 | if(parameters.sort_column === 'popular' && parameters.sort_dir === 'desc') { 28 | this.popularityIndex = parameters.start / popularity.divider; 29 | } 30 | } 31 | } 32 | } 33 | 34 | // Export Class 35 | module.exports = Listing; 36 | -------------------------------------------------------------------------------- /examples/all.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const SteamMarketCrawler = require('../lib/index'); // require('node-steam-market-crawler'); 4 | const MarketCrawler = new SteamMarketCrawler(); 5 | 6 | // getSearch 7 | MarketCrawler.getSearch({ 8 | appid: 730, 9 | q: 'PP-Bizon | Harvester' 10 | }).then((listings) => { 11 | console.log(listings); 12 | }).catch((reason) => { 13 | console.log(reason); 14 | }); 15 | 16 | // getSearchRender 17 | MarketCrawler.getSearchRender({ 18 | start: 0, 19 | count: 1 20 | }).then((listings) => { 21 | console.log(listings); 22 | }).catch((reason) => { 23 | console.log(reason); 24 | }); 25 | 26 | // getListings without Histogram 27 | MarketCrawler.getListings(730, 'AK-47 | Redline (Field-Tested)').then((listing) => { 28 | console.log(listing); 29 | }).catch((reason) => { 30 | console.log(reason); 31 | }); 32 | // getListings with Histogram 33 | MarketCrawler.getListings(730, 'AK-47 | Redline (Field-Tested)', true).then((listing) => { 34 | console.log(listing); 35 | }).catch((reason) => { 36 | console.log(reason); 37 | }); 38 | // getListings and sales 39 | MarketCrawler.getListings(730, 'AK-47 | Redline (Field-Tested)').then((listing) => { 40 | MarketCrawler.getListingSales(listing).then((sales) => { 41 | console.log(sales); 42 | }).catch((reason) => { 43 | console.log('salesErr', reason); 44 | }); 45 | }).catch((reason) => { 46 | console.log(reason); 47 | }); 48 | 49 | // getPopular 50 | MarketCrawler.getPopular(0, 10).then((listings) => { 51 | console.log(listings); 52 | }).catch((reason) => { 53 | console.log(reason); 54 | }); 55 | 56 | // getRecent 57 | MarketCrawler.getRecent().then((listings) => { 58 | console.log(listings); 59 | }).catch((reason) => { 60 | console.log(reason); 61 | }); 62 | 63 | // getRecentCompleted 64 | MarketCrawler.getRecentCompleted().then((listings) => { 65 | console.log(listings); 66 | }).catch((reason) => { 67 | console.log(reason); 68 | }); 69 | -------------------------------------------------------------------------------- /lib/classes/Histogram.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const cheerio = require('cheerio'); 4 | 5 | /** 6 | * This class is to format the histogam data. We don't want any HTML 7 | */ 8 | class Histogram { 9 | constructor(data) { 10 | this.sell_order_array = this.histogramTableToArray(data.sell_order_table); 11 | this.sell_order_summary = this.histogramSummary(data.sell_order_summary); 12 | this.buy_order_array = this.histogramTableToArray(data.buy_order_table); 13 | this.buy_order_summary = this.histogramSummary(data.buy_order_summary); 14 | this.highest_buy_order = parseInt(data.highest_buy_order, 10) || 0; 15 | this.lowest_sell_order = parseInt(data.lowest_sell_order, 10) || 0; 16 | 17 | this.buy_order_graph = data.buy_order_graph; 18 | this.sell_order_graph = data.sell_order_graph; 19 | 20 | this.graph_max_y = data.graph_max_y; 21 | this.graph_min_x = data.graph_min_x; 22 | this.graph_max_x = data.graph_max_x; 23 | this.price_prefix = data.price_prefix; 24 | this.price_suffix = data.price_suffix; 25 | } 26 | 27 | histogramTableToArray(html) { 28 | if( ! html || html === '') { 29 | return []; 30 | } 31 | let $ = cheerio.load(html); 32 | let values = []; 33 | $('tr').each((index, element) => { 34 | if($(element).find('td').length) { 35 | values.push({ 36 | price: parseFloat($(element).find('td').eq(0).text().replace('$', '').replace(' or less', '').replace(' or more', '')), 37 | quantity: parseInt($(element).find('td').eq(1).text(), 10) 38 | }); 39 | } 40 | }); 41 | return values; 42 | } 43 | 44 | histogramSummary(html) { 45 | if( ! html || html === '' || html.indexOf('no active') !== -1) { 46 | return { 47 | price: 0, 48 | quantity: 0 49 | } 50 | } 51 | let $ = cheerio.load(html); 52 | return { 53 | price: parseFloat($('span').eq(1).text().replace('$', '')), 54 | quantity: parseInt($('span').eq(0).text()) 55 | } 56 | }; 57 | } 58 | 59 | // Export Class 60 | module.exports = Histogram; 61 | -------------------------------------------------------------------------------- /lib/classes/ListingItem.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Base = require('./Base'); 4 | const regex = require('../../resources/regex'); 5 | 6 | /** 7 | * This class is for the single market item listing pages like /listings/730/AK-47%20%7C%20Redline%20%28Field-Tested%29 8 | */ 9 | class ListingItem extends Base { 10 | constructor($) { 11 | super($); 12 | 13 | // Empty script string 14 | let script = ''; 15 | $('body script').each(function(index, element) { 16 | let insides = $(this).text().toString('utf8'); 17 | 18 | if(insides.indexOf('line1') !== -1 || insides.indexOf('Market_LoadOrderSpread') !== -1) { 19 | // Populate script variable 20 | script = insides; 21 | // Break each loop 22 | return false; 23 | } 24 | }); 25 | 26 | // Item unique nameID 27 | this.nameID = regex.get(regex.listings.nameID, script, 2) || null; 28 | 29 | this.median_sale_prices = this.getMedianSalePrices(script); 30 | this.assets = this.getAssets(script); 31 | this.app_context_data = this.getAppContextData(script); 32 | 33 | // Additional data that needs to be loaded in using functions 34 | this.histogram = []; 35 | this.recent_activity = []; 36 | } 37 | 38 | getAppContextData(script) { 39 | const app_context_data = regex.get( regex.listings.appContextData, script, 3, true); 40 | if( ! app_context_data) return {}; 41 | return app_context_data; 42 | } 43 | 44 | getAssets(script) { 45 | const assets = regex.get( regex.listings.assets, script, 3, true); 46 | if( ! assets) return {}; 47 | return assets; 48 | } 49 | 50 | getMedianSalePrices(script) { 51 | if( ! this.nameID) return []; 52 | const median_sale_prices = regex.get( regex.listings.medianSalePrices, script, 2, true ); 53 | if( ! median_sale_prices) return []; 54 | // Fix quantity to integer 55 | return median_sale_prices.map((array) => { 56 | return [ 57 | array[0], 58 | +array[1], 59 | +array[2] 60 | ]; 61 | }); 62 | } 63 | } 64 | 65 | // Export Class 66 | module.exports = ListingItem; 67 | -------------------------------------------------------------------------------- /resources/endpoint.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // SteamCommunity URL's 4 | const SC = 'http://steamcommunity.com/'; 5 | const SCM = SC + 'market/'; 6 | /** 7 | * This is solely here because of SteamApis.com, 8 | * it helps our ProxyAPI identify if the loaded page was indeed valid. 9 | * Feel free to ignore this. 10 | */ 11 | const Contains = { 12 | web: 'g_steamID', 13 | json: '"success"' 14 | }; 15 | 16 | // Available endpoints 17 | module.exports = { 18 | /** 19 | * Search page 20 | */ 21 | 'search': function(object) { 22 | let parameters = ''; 23 | 24 | if(typeof object === 'object') { 25 | parameters = Object.keys(object).map((key) => { 26 | return `${key}=${object[key]}`; 27 | }).join('&'); 28 | } 29 | 30 | return new function() { 31 | this.contains = Contains.web; 32 | this.url = `${SCM}search?${parameters}&l=english`; 33 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 34 | }; 35 | }, 36 | /** 37 | * Search render API 38 | */ 39 | 'searchRender': function(object) { 40 | let parameters = ''; 41 | 42 | // Default search params if object was not set. 43 | if( ! object) { 44 | object = { 45 | query: '', 46 | start: 0, 47 | count: 10, 48 | search_descriptions: 0, 49 | sort_column: 'popular', 50 | sort_dir: 'desc' 51 | } 52 | } 53 | 54 | if(typeof object === 'object') { 55 | parameters = Object.keys(object).map((key) => { 56 | return `${key}=${object[key]}`; 57 | }).join('&'); 58 | } 59 | 60 | return new function() { 61 | this.contains = Contains.json; 62 | this.url = `${SCM}search/render?${parameters}`; 63 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 64 | }; 65 | }, 66 | /** 67 | * Gets the popular listings 68 | */ 69 | 'popular': function(start, count) { 70 | return new function() { 71 | this.contains = Contains.json; 72 | this.url = `${SCM}popular?language=english¤cy=1&start=${start}&count=${count}`; 73 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 74 | }; 75 | }, 76 | /** 77 | * Gets the recently created listings 78 | */ 79 | 'recent': function() { 80 | return new function() { 81 | this.contains = Contains.json; 82 | this.url = `${SCM}recent?country=US&language=english¤cy=1`; 83 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 84 | }; 85 | }, 86 | /** 87 | * Gets the recently completed/sold listings 88 | */ 89 | 'recentcompleted': function() { 90 | return new function() { 91 | this.contains = Contains.json; 92 | this.url = `${SCM}recentcompleted`; 93 | }; 94 | }, 95 | /** 96 | * Gets the listings page using appID and marketHashName 97 | */ 98 | 'listings': function(appID, marketHashName) { 99 | return new function() { 100 | this.contains = Contains.web; 101 | this.url = `${SCM}listings/${appID}/${encodeURIComponent(marketHashName)}?l=english`; 102 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 103 | }; 104 | }, 105 | /** 106 | * Gets the listings page sales using appID and marketHashName 107 | */ 108 | 'listingSales': function(appID, marketHashName, parameters) { 109 | if( ! parameters) { 110 | parameters = {}; 111 | } 112 | if( ! parameters.currency) { 113 | parameters.currency = 1; 114 | } 115 | if( ! parameters.count) { 116 | parameters.count = 100; 117 | } 118 | const paramString = Object.keys(parameters).map(k => `${encodeURIComponent(k)}=${encodeURIComponent(parameters[k])}`).join('&'); 119 | return new function() { 120 | this.contains = Contains.json; 121 | this.url = `${SCM}listings/${appID}/${encodeURIComponent(marketHashName)}/render/?${paramString}`; 122 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 123 | }; 124 | }, 125 | /** 126 | * Gets the item activity on the listings page 127 | */ 128 | 'itemordersactivity': function(nameID) { 129 | return new function() { 130 | this.contains = Contains.json; 131 | this.url = `${SCM}itemordersactivity?language=english¤cy=1&item_nameid=${nameID}`; 132 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 133 | }; 134 | }, 135 | /** 136 | * Gets the item sale history on the listings page 137 | */ 138 | 'itemordershistogram': function(nameID) { 139 | return new function() { 140 | this.contains = Contains.json; 141 | this.url = `${SCM}itemordershistogram?language=english¤cy=1&item_nameid=${nameID}`; 142 | this.base64 = encodeURIComponent(new Buffer(this.url).toString('base64')); 143 | }; 144 | } 145 | }; 146 | -------------------------------------------------------------------------------- /resources/dom.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | module.exports = function($) { 6 | let dom = {}; 7 | 8 | dom.selectors = { 9 | row: '.market_listing_row_link', 10 | market_name: [ 11 | '.market_listing_item_name:first', 12 | '.market_listing_item_name_link:first', 13 | 'h1#largeiteminfo_item_name:first', 14 | '.market_listing_nav a:last' 15 | ], 16 | market_link: [ 17 | '.market_listing_item_name_link', 18 | '.market_listing_nav a' 19 | ], 20 | game_name: '.market_listing_game_name', 21 | image: [ 22 | 'img.market_listing_item_img', 23 | '.market_listing_largeimage img' 24 | ], 25 | quantity: '.market_listing_num_listings_qty', 26 | normal_price: [ 27 | '.normal_price', 28 | '.market_listing_price_with_fee' 29 | ], 30 | sale_price: [ 31 | '.sale_price', 32 | '.market_listing_price_with_publisher_fee_only' 33 | ], 34 | sellerAvatar: '.playerAvatar img' 35 | }; 36 | 37 | dom.url = function() { 38 | // All possible matches 39 | const elements = [ 40 | $( this ).attr('href'), 41 | $( dom.selectors.row ).attr('href'), 42 | $( dom.selectors.market_link[0] ).attr('href'), 43 | $( dom.selectors.market_link[1] ).last().attr('href') 44 | ]; 45 | 46 | // Return a match that is not false nor undefined 47 | let url = _.find(elements, (element) => { 48 | // Check if element has any value 49 | return element !== false && typeof element !== 'undefined' && element.length >= 1; 50 | }); 51 | 52 | // Replace HTTPS with HTTP if possible and return value 53 | return url.replace('https://', 'http://') || null; 54 | }; 55 | 56 | dom.marketName = function() { 57 | // Find a possible match from array 58 | for(let i in dom.selectors.market_name) { 59 | // current selector from the array 60 | let element = dom.selectors.market_name[i]; 61 | // Cheerio does not support ":last" and ":first" at the moment so we have to use .first() and .last() 62 | if(element.indexOf(':') !== -1) { 63 | element = element.split(':'); 64 | // element[0] is the selector 65 | // element[1] is either "first" or "last" 66 | let selector = $(element[0])[element[1]](); 67 | // Check if it exists and has any content 68 | if(selector.length && selector.text().trim().length >= 1) { 69 | return selector.text(); 70 | } 71 | } else { 72 | // Check if exists and has any content 73 | if($(element).length && $(element).text().trim().length >= 1) { 74 | return $(element).text(); 75 | } 76 | } 77 | } 78 | // We did not find a match for market_name 79 | return false; 80 | }; 81 | 82 | dom.borderColor = function() { 83 | const element = $( dom.selectors.image[0] ); 84 | if(element.attr('style')) { 85 | return element.css('border-color'); 86 | } else { 87 | return null; 88 | } 89 | }; 90 | 91 | dom.image = function() { 92 | // All possible matches 93 | const elements = [ 94 | $( dom.selectors.image[0] ).attr('src'), 95 | $( dom.selectors.image[1] ).first().attr('src') 96 | ]; 97 | 98 | // Return a match that is not false nor undefined 99 | let image = _.find(elements, (element) => { 100 | // Check if element has any value 101 | return element !== false && typeof element !== 'undefined' && element.length >= 1; 102 | }); 103 | 104 | // If we didn't get an image (some Steam items trully have no image) return null 105 | if( ! image) { 106 | return null; 107 | } 108 | 109 | // Remove icon size 110 | return image.replace('/62fx62f', '').replace('/360fx360f', ''); 111 | }; 112 | 113 | dom.isSold = function() { 114 | // All possible matches 115 | const elements = [ 116 | $( dom.selectors.normal_price[0] ).eq(1).text().trim(), 117 | $( dom.selectors.normal_price[1] ).text().trim() 118 | ]; 119 | 120 | let isSold = _.find(elements, (element) => { 121 | // Check if element has any value 122 | return element !== false && typeof element !== 'undefined' && element.length >= 1; 123 | }); 124 | 125 | // Item has been sold already 126 | if(isSold.indexOf('Sold!') !== -1) { 127 | return true; 128 | } 129 | // Not sold yet 130 | return false; 131 | }; 132 | 133 | dom.sellerAvatar = function() { 134 | return $( dom.selectors.sellerAvatar ).attr('src') || null; 135 | }; 136 | 137 | dom.normalPrice = function() { 138 | // All possible matches 139 | const elements = [ 140 | $( dom.selectors.normal_price[0] ).eq(1).text().trim(), 141 | $( dom.selectors.normal_price[1] ).text().trim() 142 | ]; 143 | 144 | let normalPrice = _.find(elements, (element) => { 145 | // Check if element has any value 146 | return element !== false && typeof element !== 'undefined' && element.length >= 1; 147 | }); 148 | 149 | // Return value as float, remove currency symbol 150 | return parseFloat(normalPrice.replace('$', '')) || null; 151 | }; 152 | 153 | dom.salePrice = function() { 154 | // All possible matches 155 | const elements = [ 156 | $( dom.selectors.sale_price[0] ).text().trim(), 157 | $( dom.selectors.sale_price[1] ).text().trim() 158 | ]; 159 | 160 | let salePrice = _.find(elements, (element) => { 161 | // Check if element has any value 162 | return element !== false && typeof element !== 'undefined' && element.length >= 1; 163 | }); 164 | 165 | // Return value as float, remove currency symbol 166 | return parseFloat(salePrice.replace('$', '')) || null; 167 | }; 168 | 169 | return dom; 170 | } 171 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Regex = require('../resources/regex'); 4 | const Currency = require('../resources/currency'); 5 | const Endpoint = require('../resources/endpoint'); 6 | 7 | const Listing = require('./classes/Listing'); 8 | const ListingUser = require('./classes/ListingUser'); 9 | const ListingItem = require('./classes/ListingItem'); 10 | const ListingSale = require('./classes/ListingSale'); 11 | const Histogram = require('./classes/Histogram'); 12 | 13 | const requestp = require('request-promise'); 14 | const cheerio = require('cheerio'); 15 | 16 | class SteamCrawler { 17 | constructor(options) { 18 | options = options || {}; 19 | 20 | this._cheerio = cheerio; 21 | 22 | // Default currrency 23 | this.currency = options.currency || Currency.USD; 24 | // Add popularity index to class when searching for items? 25 | this.popularity = { 26 | use: options.usePopularityIndex || false, 27 | divider: options.popularityDivider || 10000 28 | }; 29 | // Use HTTP/HTTPS proxy to mask requests 30 | this.proxy = options.proxy || null; 31 | // Set timeout for requests 32 | this.timeout = options.timeout || 5000; 33 | // Set MAX_RETRIES for requests 34 | this.maxRetries = options.maxRetries || 3; 35 | // Regex's 36 | this.Regex = Regex; 37 | // Currencies 38 | this.Currency = Currency; 39 | // Endpoints 40 | this.Endpoint = Endpoint; 41 | 42 | /** 43 | * Mostly options that are necessary for SteamApis, 44 | * you can safely ignore these. 45 | */ 46 | // Use WebProxy to proxy the requests 47 | this.webProxy = options.webProxy || false; 48 | // Use Base64 encoded strings as URL's where possible, 49 | // good when using webProxy - other than that pointless. 50 | this.base64 = options.base64 || false; 51 | // Base64 prefix 52 | this.base64Prefix = options.base64Prefix || 'base64'; 53 | 54 | /** 55 | * Request and its defaults 56 | */ 57 | this.requestDefaults = options.requestDefaults || {} 58 | this._request = requestp.defaults(Object.assign({ 59 | proxy: this.proxy, 60 | headers: { 61 | 'accept-charset': 'utf-8' 62 | }, 63 | timeout: this.timeout, 64 | maxRedirects: 5 65 | }, this.requestDefaults)); 66 | } 67 | 68 | _requestFn (url, json, retries) { 69 | const params = {}; 70 | if (json) { 71 | params.transform = body => { 72 | return JSON.parse(body); 73 | }; 74 | params.transform2xxOnly = true; 75 | } 76 | return this._request({ url, ...params }) 77 | .then(body => { 78 | if (!json) return body; 79 | // We received a bad response, lets try to get a good one 80 | if(typeof body === 'undefined' || !body || body.success === false) { 81 | // Check if we can retry 82 | const retryable = this.retryable(retries); 83 | if(retryable.shouldRetry) { 84 | return this.request(url, json, retryable.retries); 85 | } else { 86 | // The request succeeded but we received a bad/malformed response 87 | return Promise.reject('Received bad response from Steam.'); 88 | } 89 | } else { 90 | return body; 91 | } 92 | }) 93 | .catch(error => { 94 | // Check if we can retry 95 | const retryable = this.retryable(retries); 96 | if(retryable.shouldRetry) { 97 | return this.request(url, json, retryable.retries); 98 | } else { 99 | return Promise.reject(error); 100 | } 101 | }); 102 | } 103 | 104 | request (url, json) { 105 | return new Promise((resolve, reject) => { 106 | const req = this._requestFn(url, json, this.maxRetries); 107 | // Proxies can make request's timeout not function properly 108 | // have to do this hackish method to trigger our own 109 | const timer = setTimeout(() => { 110 | req.cancel(); 111 | return reject('Timed out!'); 112 | }, this.timeout + 3000); 113 | 114 | req.then(resolve).catch(reject); 115 | }) 116 | } 117 | 118 | retryable(retries) { 119 | // Check if retries is set, otherwise set it at 0 120 | if(typeof retries === 'undefined') { 121 | retries = 0; 122 | } else { 123 | retries++; 124 | } 125 | // Not hit the max retries yet 126 | if(retries <= this.maxRetries) { 127 | return { retries: retries, shouldRetry: true }; 128 | } else { 129 | // We hit the max retry amount 130 | return { retries: retries, shouldRetry: false }; 131 | } 132 | } 133 | 134 | getSearch(parameters) { 135 | const url = this.buildUrl(Endpoint.search(parameters)); 136 | 137 | return this.request(url, false).then((body) => { 138 | let results = this.seperateListingsFromHTML(body); 139 | let listings = []; 140 | 141 | for(let i in results) { 142 | listings.push(new Listing(results[i], parameters, this.popularity)); 143 | } 144 | 145 | // return the Listings array 146 | return listings; 147 | }).catch((error) => { 148 | // return error 149 | return Promise.reject(error); 150 | }); 151 | } 152 | 153 | getSearchRender(parameters) { 154 | let returnResponseOnly = false; 155 | if(parameters.responseOnly) { 156 | returnResponseOnly = true; 157 | delete parameters.responseOnly; 158 | } 159 | 160 | const url = this.buildUrl(Endpoint.searchRender(parameters)); 161 | 162 | return this.request(url, true).then((body) => { 163 | if( ! returnResponseOnly) { 164 | let results = this.seperateListingsFromHTML(body.results_html); 165 | let listings = []; 166 | 167 | for(let i in results) { 168 | listings.push(new Listing(results[i], parameters, this.popularity)); 169 | } 170 | 171 | // return the Listings array 172 | body = listings; 173 | } 174 | 175 | // returns either listings arr or body 176 | return body; 177 | }).catch((error) => { 178 | return Promise.reject(error); 179 | }); 180 | } 181 | 182 | getListings(appID, market_hash_name, loadHistogram) { 183 | const url = this.buildUrl(Endpoint.listings(appID, market_hash_name)); 184 | 185 | return this.request(url, false).then((body) => { 186 | body = this._cheerio.load(body); 187 | let Listing = new ListingItem(body); 188 | 189 | if(loadHistogram) { 190 | return this.getHistogram(Listing) 191 | .then(data => { 192 | return [body, data] 193 | }) 194 | .catch(err => { 195 | return [body, Listing] 196 | }) 197 | } else { 198 | return [body, Listing]; 199 | } 200 | }).then((results) => { 201 | // return the Listing item 202 | return results[1]; 203 | }).catch((error) => { 204 | // return error 205 | return Promise.reject(error); 206 | }); 207 | } 208 | 209 | getListingSales(ListingItem, parameters) { 210 | if( ! ListingItem) { 211 | return Promise.reject('You have to provide the `nameID` value of the ListingItem'); 212 | } 213 | 214 | const url = this.buildUrl(Endpoint.listingSales(ListingItem.appID, ListingItem.market_hash_name, parameters)); 215 | 216 | return this.request(url, true).then((data) => { 217 | let results = this.seperateListingsFromHTML(data.results_html); 218 | let sales = []; 219 | 220 | for(let i in results) { 221 | sales.push(new ListingSale(results[i], data.listinginfo)); 222 | } 223 | 224 | return sales; 225 | }).catch((error) => { 226 | return Promise.reject(error); 227 | }); 228 | } 229 | 230 | getHistogram(ListingItem) { 231 | if( ! ListingItem) { 232 | return Promise.reject('You have to provide the `nameID` value of the ListingItem'); 233 | } 234 | let nameID; 235 | if(typeof ListingItem === 'object') { 236 | nameID = ListingItem.nameID; 237 | } else { 238 | nameID = ListingItem; 239 | } 240 | if(typeof nameID === 'undefined' || ! nameID) { 241 | return Promise.reject('Invalid `nameID` value.'); 242 | } 243 | 244 | const url = this.buildUrl(Endpoint.itemordershistogram(nameID)); 245 | 246 | return this.request(url, true).then((data) => { 247 | // If we got an object, return data with the object 248 | if(typeof ListingItem === 'object') { 249 | ListingItem.histogram = new Histogram(data); 250 | return ListingItem; 251 | } 252 | return new Histogram(data); 253 | }).catch((error) => { 254 | return Promise.reject(error); 255 | }); 256 | } 257 | 258 | getRecentActivity(ListingItem) { 259 | if( ! ListingItem) { 260 | return Promise.reject('You have to provide the `nameID` value of the ListingItem'); 261 | } 262 | let nameID; 263 | if(typeof ListingItem === 'object') { 264 | nameID = ListingItem.nameID; 265 | } else { 266 | nameID = ListingItem; 267 | } 268 | if(typeof nameID === 'undefined' || ! nameID) { 269 | return Promise.reject('Invalid `nameID` value.'); 270 | } 271 | 272 | const url = this.buildUrl(Endpoint.itemordersactivity(nameID)); 273 | 274 | return this.request(url, true).then((data) => { 275 | // If we got an object, return data with the object 276 | if(typeof ListingItem === 'object') { 277 | ListingItem.recent_activity = data; 278 | return ListingItem; 279 | } 280 | return data; 281 | }).catch((error) => { 282 | return Promise.reject(error); 283 | }); 284 | } 285 | 286 | getPopular(start, count) { 287 | const url = this.buildUrl(Endpoint.popular(start, count)); 288 | 289 | return this.request(url, true).then((data) => { 290 | let listings = []; 291 | 292 | for(let i in data.results_html) { 293 | let $ = this._cheerio.load(data.results_html[i].toString('utf8')); 294 | listings.push(new Listing($, {}, this.popularity)); 295 | } 296 | 297 | // return the Listings array 298 | return listings; 299 | }).catch((error) => { 300 | // return error 301 | return Promise.reject(error); 302 | }); 303 | } 304 | 305 | getRecent() { 306 | const url = this.buildUrl(Endpoint.recent()); 307 | 308 | return this.request(url, true).then((data) => { 309 | let json = { 310 | listings: [], 311 | last_time: data.last_time, 312 | last_listing: data.last_listing 313 | }; 314 | 315 | for(let i in data.listinginfo) { 316 | json.listings.push(new ListingUser(data.listinginfo[i], data.assets)); 317 | } 318 | 319 | // return the listings object 320 | return json; 321 | }).catch((error) => { 322 | // return error 323 | return Promise.reject(error); 324 | }); 325 | } 326 | 327 | getRecentCompleted() { 328 | const url = this.buildUrl(Endpoint.recentcompleted()); 329 | 330 | return this.request(url, true).then((data) => { 331 | let json = { 332 | listings: [], 333 | last_time: data.last_time, 334 | last_listing: data.last_listing 335 | }; 336 | 337 | for(let i in data.purchaseinfo) { 338 | json.listings.push(new ListingUser(data.purchaseinfo[i], data.assets)); 339 | } 340 | 341 | // return the listings object 342 | return json; 343 | }).catch((error) => { 344 | // return error 345 | return Promise.reject(error); 346 | }); 347 | } 348 | 349 | setProxy(proxy) { 350 | this.proxy = proxy; 351 | this._request = requestp.defaults(Object.assign({ 352 | proxy: this.proxy, 353 | headers: { 354 | 'accept-charset': 'utf-8' 355 | }, 356 | timeout: this.timeout, 357 | maxRedirects: 5 358 | }, this.requestDefaults)); 359 | } 360 | 361 | setDefaults(options) { 362 | this._request = requestp.defaults(Object.assign({ 363 | headers: { 364 | 'accept-charset': 'utf-8' 365 | }, 366 | timeout: this.timeout, 367 | maxRedirects: 5 368 | }, options)) 369 | } 370 | 371 | buildUrl(endpoint) { 372 | // Check if we need to use base64 or default url 373 | const URL = (this.base64 && endpoint.base64) ? this.base64Prefix + endpoint.base64 : endpoint.url; 374 | // Check if webProxy was specified 375 | if(this.webProxy) { 376 | return `${this.webProxy}&contains=${endpoint.contains}&url=${URL}`; 377 | } 378 | // No webProxy - return default url 379 | return endpoint.url; 380 | } 381 | 382 | seperateListingsFromHTML(html) { 383 | const $ = cheerio.load(html.toString('utf8')); 384 | let results = []; 385 | 386 | let selector = '.market_listing_row'; 387 | if($('.market_listing_row_link').length) { 388 | selector = '.market_listing_row_link'; 389 | } 390 | 391 | $(selector).each((index, element) => { 392 | results.push(cheerio.load(element)); 393 | }); 394 | 395 | return results; 396 | } 397 | } 398 | 399 | module.exports = SteamCrawler; 400 | --------------------------------------------------------------------------------