├── img ├── 20200403174042.png ├── 202004031740421.png ├── 202004031740422.png └── 202004031740423.png ├── package.json ├── README.md ├── index.js ├── lib ├── request.js ├── utils.js └── logger.js └── src ├── renarts.js ├── bdgastore.js ├── juiceStore.js ├── kith.js ├── footlocker.js ├── yeezysupply.js ├── snkrsJp.js ├── snkrsCn.js ├── snkrsUk.js └── snkrsUs.js /img/20200403174042.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TomLeel/snkrsSpider/HEAD/img/20200403174042.png -------------------------------------------------------------------------------- /img/202004031740421.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TomLeel/snkrsSpider/HEAD/img/202004031740421.png -------------------------------------------------------------------------------- /img/202004031740422.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TomLeel/snkrsSpider/HEAD/img/202004031740422.png -------------------------------------------------------------------------------- /img/202004031740423.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TomLeel/snkrsSpider/HEAD/img/202004031740423.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "snkrs", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node index.js" 9 | }, 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "bearyincoming": "0.0.2", 14 | "cheerio": "^1.0.0-rc.3", 15 | "child_process": "^1.0.2", 16 | "log4js": "^3.0.5", 17 | "moment": "^2.24.0", 18 | "mongodb": "^3.3.2", 19 | "superagent": "^5.1.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | npm i 2 | 3 | node index 4 | 5 | 写出来使用了两个月，包含了上新和补货，速度比付费软件快的多，今天决定开源出来 6 | 7 | 由于工作原因，服务停掉后很久没有维护，NIKE yeezy可能已经变了路径，还请自行替换 8 | 9 | 包含了线程挂掉自起，记录日志，比较稳定，使用期间未曾断过 10 | 11 | 使用前请先安装mongoDB，并替换数据库路径 12 | 13 | 借用的倍恰推送通知，使用前也请替换推送WEBHOOK_URL 14 | 15 | :blush:对你有帮助，点个星星吧 ~ :star: 16 | 17 | 效果图： 18 | 19 | ![效果图1](https://github.com/TomLeel/snkrsSpider/blob/master/img/202004031740423.png) 20 | 21 | ![效果图2](https://github.com/TomLeel/snkrsSpider/blob/master/img/202004031740422.png) 22 | 23 | ![效果图3](https://github.com/TomLeel/snkrsSpider/blob/master/img/202004031740421.png) 24 | 25 | ![效果图4](https://github.com/TomLeel/snkrsSpider/blob/master/img/20200403174042.png) 26 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const childProcess = require('child_process') 2 | const Log = require('./lib/logger')('index') 3 | 4 | let workers = [] 5 | let file = { 6 | 0: './src/snkrsCn.js', 7 | 1: './src/snkrsJp.js', 8 | 2: './src/snkrsUk.js', 9 | 3: './src/snkrsUs.js', 10 | 4: './src/juiceStore.js', 11 | 5: './src/kith.js', 12 | 6: './src/bdgastore.js', 13 | // 7: './src/yeezysupply.js' 14 | } 15 | 16 | Object.keys(file).forEach(key => { 17 | workers.push(childProcess.fork(file[key])) 18 | }) 19 | 20 | for (let i = 0; i < workers.length; ++i) { 21 | // 工作进程退出后重启 22 | workers[i].on( 23 | 'exit', 24 | (i => { 25 | return () => { 26 | Log.info('Worker-' + workers[i] + ' exited') 27 | workers[i] = childProcess.fork(file[i]) 28 | Log.info('Create worker-' + workers[i].pid) 29 | } 30 | })(i) 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /lib/request.js: -------------------------------------------------------------------------------- 1 | const superagent = require('superagent') 2 | 3 | class Request { 4 | get(url, query = {}, headers = {}, type) { 5 | return new Promise((r, j) => { 6 | superagent 7 | .get(url) 8 | .query(query) 9 | .set(headers) 10 | .end((err, res) => { 11 | if (err) { 12 | j(err) 13 | } else { 14 | r((!!type ? res[type] : res.body) || null) 15 | } 16 | }) 17 | }) 18 | } 19 | 20 | post(url, query = {}, headers = {}) { 21 | return new Promise((r, j) => { 22 | superagent 23 | .post(url) 24 | .send(query) 25 | .set(headers) 26 | .end((err, res) => { 27 | if (err) { 28 | j(err) 29 | } else { 30 | r(res.body || null) 31 | } 32 | }) 33 | }) 34 | } 35 | } 36 | 37 | module.exports = new Request() 38 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | // 数组差集 2 | function subSetWithStock(arr1, arr2) { 3 | let newArr = [] 4 | for (let i = 0; i < arr1.length; i++) { 5 | let num = 0 6 | let item = {} 7 | for (let j = 0; j < arr2.length; j++) { 8 | if (arr1[i].id === arr2[j].id && arr1[i].stock !== arr2[j].stock) { 9 | num++ 10 | item = arr1[i] 11 | item['stock2'] = arr2[j].stock 12 | item['stockUpdate'] = true 13 | break 14 | } 15 | } 16 | if (num !== 0) { 17 | newArr.push(item) 18 | } 19 | } 20 | 21 | return newArr 22 | } 23 | 24 | function subSetByArray(arr1, arr2) { 25 | let newArr = [] 26 | if (arr1.length) { 27 | for (let i = 0; i < arr1.length; i++) { 28 | var num = 0 29 | if (arr2.length) { 30 | for (let j = 0; j < arr2.length; j++) { 31 | if (arr1[i] === arr2[j]) { 32 | num++ 33 | break 34 | } 35 | } 36 | } 37 | 38 | if (num == 0) { 39 | newArr.push(arr1[i]) 40 | } 41 | } 42 | } 43 | 44 | return newArr 45 | } 46 | 47 | function subSet(arr1, arr2) { 48 | let newArr = [] 49 | for (let i = 0; i < arr1.length; i++) { 50 | var num = 0 51 | for (let j = 0; j < arr2.length; j++) { 52 | if (arr1[i].id === arr2[j].id) { 53 | num++ 54 | break 55 | } 56 | } 57 | if (num == 0) { 58 | newArr.push(arr1[i]) 59 | } 60 | } 61 | 62 | return newArr 63 | } 64 | 65 | // 去重 66 | function distinct(arr) { 67 | let result = [] 68 | let obj = {} 69 | 70 | for (let i of arr) { 71 | if (!obj[i.id]) { 72 | result.push(i) 73 | obj[i.id] = 1 74 | } 75 | } 76 | 77 | return result 78 | } 79 | 80 | module.exports = { 81 | subSet, 82 | subSetWithStock, 83 | subSetByArray, 84 | distinct 85 | } 86 | -------------------------------------------------------------------------------- /lib/logger.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 日志 3 | */ 4 | 5 | const log4js = require('log4js') 6 | 7 | class Logger { 8 | constructor(filename = '') { 9 | this.filename = filename 10 | } 11 | 12 | _init() { 13 | let logFileName = 14 | this.filename || this.constructor.name.replace('Logger', '') 15 | if (!logFileName) logFileName = 'default' 16 | 17 | log4js.configure({ 18 | appenders: { 19 | dateFile: { 20 | type: 'dateFile', 21 | filename: 'logs/', 22 | pattern: 'yyyy-MM/dd/' + logFileName + '.log', 23 | maxLogSize: 10 * 1024 * 1024, // = 10Mb 24 | numBackups: 5, // keep five backup files 25 | compress: false, // compress the backups 26 | encoding: 'utf-8', 27 | alwaysIncludePattern: true 28 | }, 29 | out: { 30 | type: 'stdout' 31 | } 32 | }, 33 | categories: { 34 | default: { 35 | appenders: ['dateFile', 'out'], 36 | level: 'trace' 37 | } 38 | } 39 | }) 40 | 41 | this.logger = log4js.getLogger() 42 | } 43 | 44 | _log(level = 'info', ...args) { 45 | this._init() 46 | 47 | let infos = [] 48 | for (var key in args) { 49 | if (args.hasOwnProperty(key)) { 50 | if (typeof args[key] == 'object') { 51 | infos.push(JSON.stringify(args[key])) 52 | } else if (typeof args[key] == 'undefined') { 53 | infos.push('undefined') 54 | } else if (typeof args[key] == 'function') { 55 | infos.push('function') 56 | } else { 57 | infos.push(args[key]) 58 | } 59 | } 60 | } 61 | 62 | let logStr = infos.join(' | ') 63 | this.logger[level](logStr) 64 | } 65 | 66 | info(...args) { 67 | // let args = arguments || [] 68 | this._log('info', ...args) 69 | } 70 | 71 | trace(...args) { 72 | this._log('trace', ...args) 73 | } 74 | 75 | debug(...args) { 76 | this._log('debug', ...args) 77 | } 78 | 79 | warn(...args) { 80 | this._log('warn', ...args) 81 | } 82 | 83 | error(...args) { 84 | this._log('error', ...args) 85 | } 86 | 87 | fatal(...args) { 88 | this._log('fatal', ...args) 89 | } 90 | } 91 | 92 | module.exports = prx => { 93 | return new Logger(prx) 94 | } 95 | -------------------------------------------------------------------------------- /src/renarts.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const MongoClient = require('mongodb').MongoClient 3 | const bearychat = require('bearyincoming') 4 | const cheerio = require('cheerio') 5 | const { 6 | subSetWithStock, 7 | subSetByArray, 8 | subSet, 9 | distinct 10 | } = require('../lib/utils') 11 | const Log = require('../lib/logger')('renarts') 12 | 13 | const dbUrl = 'mongodb://localhost:27017/runoob' 14 | const _URL = 'https://renarts.com/collections/new-arrivals-footwear' 15 | const WEBHOOK_URL = 16 | '' 17 | const query = {} 18 | const headers = { 19 | 'User-Agent': 20 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 21 | } 22 | const type = 'text' 23 | 24 | class renarts { 25 | async index() { 26 | let requestData = await this.fetchData() 27 | Log.info(requestData) 28 | // this.saveData(requestData, this) 29 | } 30 | 31 | async fetchData() { 32 | let data = {} 33 | try { 34 | data = await Request.get(_URL, query, headers, type) 35 | } catch (err) { 36 | Log.info('爬取数据错误') 37 | return null 38 | } 39 | 40 | return data 41 | } 42 | 43 | saveData(data, that) { 44 | if (!data) { 45 | return 46 | } 47 | 48 | let shoesData = [] 49 | let $ = cheerio.load(data) 50 | $('.product-grid-item').each((idx, ele) => { 51 | let $1 = cheerio.load(ele) 52 | shoesData.push({ 53 | id: $(ele).attr('data-product-id'), 54 | title: $1('h3').text(), 55 | subtitle: $1('h4').text(), 56 | image: $1('.featured-img').attr('data-src'), 57 | price: $1('.prod-price') 58 | .text() 59 | .replace(/[ \n]/g, '') 60 | // stock: , 61 | }) 62 | }) 63 | 64 | let whereId = shoesData.map(item => item.id) 65 | let insertData = [] 66 | let updateData = [] 67 | MongoClient.connect( 68 | dbUrl, 69 | { useNewUrlParser: true, useUnifiedTopology: true }, 70 | function(err, db) { 71 | if (err) { 72 | Log.error('打开数据库错误', err) 73 | } 74 | var dbo = db.db('runoob') 75 | dbo 76 | .collection('renarts') 77 | .find({ id: { $in: whereId } }) 78 | .toArray(function(err, result) { 79 | if (err) { 80 | Log.error('查询数据库错误', err) 81 | } 82 | if (!result.length) { 83 | insertData = shoesData 84 | } else if ( 85 | result.length && 86 | whereId.length && 87 | result.length !== whereId.length 88 | ) { 89 | // 取差集 90 | insertData = subSet(shoesData, result) 91 | } 92 | 93 | if (insertData.length) { 94 | // 去重 95 | insertData = distinct(insertData) 96 | 97 | insertData.length && that.sendMessage(insertData) 98 | 99 | insertData.length && 100 | dbo 101 | .collection('renarts') 102 | .insertMany(insertData, function(err, res) { 103 | if (err) { 104 | Log.error('插入数据库错误', err) 105 | } 106 | Log.info(`上新 | ${res.insertedCount}`, insertData) 107 | db.close() 108 | }) 109 | } 110 | 111 | db.close() 112 | }) 113 | } 114 | ) 115 | } 116 | 117 | async sendMessage(array) { 118 | for await (let item of array) { 119 | const content = `\n价格：$${item.price}` 120 | bearychat 121 | .withText(`${item.title}`) 122 | .withAttachment({ 123 | title: item.subtitle, 124 | text: content, 125 | color: '#ffa500', 126 | images: [{ url: `https:${item.image}` }] 127 | }) 128 | .pushTo(WEBHOOK_URL) 129 | } 130 | } 131 | } 132 | 133 | module.exports = new renarts() 134 | 135 | setInterval(() => { 136 | new renarts().index() 137 | }, 4000) 138 | -------------------------------------------------------------------------------- /src/bdgastore.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const MongoClient = require('mongodb').MongoClient 3 | const bearychat = require('bearyincoming') 4 | const cheerio = require('cheerio') 5 | const { 6 | subSetWithStock, 7 | subSetByArray, 8 | subSet, 9 | distinct 10 | } = require('../lib/utils') 11 | const Log = require('../lib/logger')('bdgastore') 12 | 13 | const dbUrl = 'mongodb://localhost:27017/runoob' 14 | const _URL = 'https://bdgastore.com/collections/footwear?page=1' 15 | const WEBHOOK_URL = 16 | 'https://hook.bearychat.com/=bwHsk/incoming/770ec4b55d7016eb0976ca05e3645e52' 17 | const query = {} 18 | const headers = { 19 | 'User-Agent': 20 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 21 | } 22 | const type = 'text' 23 | 24 | class bdgastore { 25 | async index() { 26 | let requestData = await this.fetchData() 27 | 28 | this.saveData(requestData, this) 29 | } 30 | 31 | async fetchData() { 32 | let data = {} 33 | try { 34 | data = await Request.get(_URL, query, headers, type) 35 | } catch (err) { 36 | Log.info('爬取数据错误') 37 | return null 38 | } 39 | 40 | return data 41 | } 42 | 43 | saveData(data, that) { 44 | if (!data) { 45 | return 46 | } 47 | 48 | let shoesData = [] 49 | let $ = cheerio.load(data) 50 | $('.product-grid-item').each((idx, ele) => { 51 | let $1 = cheerio.load(ele) 52 | shoesData.push({ 53 | id: $(ele).attr('data-product-id'), 54 | title: $1('h3').text(), 55 | subtitle: $1('h4').text(), 56 | image: $1('.featured-img').attr('data-src'), 57 | price: $1('.prod-price') 58 | .text() 59 | .replace(/[ \n]/g, '') 60 | // stock: , 61 | }) 62 | }) 63 | 64 | let whereId = shoesData.map(item => item.id) 65 | let insertData = [] 66 | let updateData = [] 67 | MongoClient.connect( 68 | dbUrl, 69 | { useNewUrlParser: true, useUnifiedTopology: true }, 70 | function(err, db) { 71 | if (err) { 72 | Log.error('打开数据库错误', err) 73 | } 74 | var dbo = db.db('runoob') 75 | dbo 76 | .collection('bdgastore') 77 | .find({ id: { $in: whereId } }) 78 | .toArray(function(err, result) { 79 | if (err) { 80 | Log.error('查询数据库错误', err) 81 | } 82 | if (!result.length) { 83 | insertData = shoesData 84 | } else if ( 85 | result.length && 86 | whereId.length && 87 | result.length !== whereId.length 88 | ) { 89 | // 取差集 90 | insertData = subSet(shoesData, result) 91 | } 92 | 93 | if (insertData.length) { 94 | // 去重 95 | insertData = distinct(insertData) 96 | 97 | insertData.length && that.sendMessage(insertData) 98 | 99 | insertData.length && 100 | dbo 101 | .collection('bdgastore') 102 | .insertMany(insertData, function(err, res) { 103 | if (err) { 104 | Log.error('插入数据库错误', err) 105 | } 106 | Log.info(`上新 | ${res.insertedCount}`, insertData) 107 | db.close() 108 | }) 109 | } 110 | 111 | db.close() 112 | }) 113 | } 114 | ) 115 | } 116 | 117 | async sendMessage(array) { 118 | for await (let item of array) { 119 | const content = `\n价格：$${item.price}` 120 | bearychat 121 | .withText(`${item.title}`) 122 | .withAttachment({ 123 | title: item.subtitle, 124 | text: content, 125 | color: '#ffa500', 126 | images: [{ url: `https:${item.image}` }] 127 | }) 128 | .pushTo(WEBHOOK_URL) 129 | } 130 | } 131 | } 132 | 133 | module.exports = new bdgastore() 134 | 135 | setInterval(() => { 136 | new bdgastore().index() 137 | }, 4000) 138 | -------------------------------------------------------------------------------- /src/juiceStore.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const MongoClient = require('mongodb').MongoClient 3 | const bearychat = require('bearyincoming') 4 | const cheerio = require('cheerio') 5 | const { 6 | subSet, 7 | subSetByArray, 8 | subSetWithStock, 9 | distinct 10 | } = require('../lib/utils') 11 | const Log = require('../lib/logger')('juiceStore') 12 | 13 | const dbUrl = 'mongodb://localhost:27017/runoob' 14 | const JUICE_URL = 'https://juicestore.com/collections/new-arrivals' 15 | const WEBHOOK_URL = 16 | 'https://hook.bearychat.com/=bwHsk/incoming/4126e386dd246432519899729e76cc36' 17 | const query = {} 18 | const headers = { 19 | 'User-Agent': 20 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 21 | } 22 | const type = 'text' 23 | 24 | class juiceStore { 25 | async index() { 26 | let requestData = await this.fetchData() 27 | 28 | this.saveData(requestData, this) 29 | } 30 | 31 | async fetchData() { 32 | let data = {} 33 | try { 34 | data = await Request.get(JUICE_URL, query, headers, type) 35 | } catch (err) { 36 | Log.info('爬取数据错误') 37 | return null 38 | } 39 | 40 | return data 41 | } 42 | 43 | saveData(data, that) { 44 | if (!data) { 45 | return 46 | } 47 | 48 | let shoesData = [] 49 | let $ = cheerio.load(data) 50 | $('div .grid__item--collection-template').each((idx, ele) => { 51 | let $1 = cheerio.load(ele) 52 | 53 | shoesData.push({ 54 | id: $1('div .grid-view-item__title').text(), 55 | title: $1('div .grid-view-item__title').text(), 56 | image: $1('.grid-view-item__image-wrapper .grid-view-item__image') 57 | .first() 58 | .attr('src'), 59 | price: $1('.price-item--sale span').text(), 60 | subtitle: $1('div .price__vendor').text(), 61 | stock: $1('.hover') 62 | .contents() 63 | .filter(function() { 64 | return this.nodeType == 3 65 | }) 66 | .text() 67 | .replace(/[ \t\n]/g, '') 68 | .replace(/[/]/g, ', ') 69 | }) 70 | }) 71 | 72 | let whereId = shoesData.map(item => item.id) 73 | let insertData = [] 74 | let updateData = [] 75 | MongoClient.connect( 76 | dbUrl, 77 | { useNewUrlParser: true, useUnifiedTopology: true }, 78 | function(err, db) { 79 | if (err) { 80 | Log.error('打开数据库错误', err) 81 | } 82 | var dbo = db.db('runoob') 83 | dbo 84 | .collection('juiceStore') 85 | .find({ id: { $in: whereId } }) 86 | .toArray(function(err, result) { 87 | if (err) { 88 | Log.error('查询数据库错误', err) 89 | } 90 | if (!result.length) { 91 | insertData = shoesData 92 | } else if ( 93 | result.length && 94 | whereId.length && 95 | result.length !== whereId.length 96 | ) { 97 | // 取差集 98 | insertData = subSet(shoesData, result) 99 | } 100 | 101 | updateData = subSetWithStock(shoesData, result) 102 | 103 | if (insertData.length) { 104 | // 去重 105 | insertData = distinct(insertData) 106 | 107 | insertData.length && that.sendMessage(insertData) 108 | 109 | insertData.length && 110 | dbo 111 | .collection('juiceStore') 112 | .insertMany(insertData, function(err, res) { 113 | if (err) { 114 | Log.error('插入数据库错误', err) 115 | } 116 | Log.info(`上新 | ${res.insertedCount}`, insertData) 117 | }) 118 | } 119 | 120 | if (updateData.length) { 121 | updateData = distinct(updateData) 122 | 123 | updateData.length && 124 | updateData.forEach(item => { 125 | let whereStr = { id: item.id } // 查询条件 126 | let updateStr = { $set: { stock: item.stock } } 127 | dbo 128 | .collection('juiceStore') 129 | .updateOne(whereStr, updateStr, function(err, res) { 130 | if (err) { 131 | Log.error('更新数据库错误', err) 132 | } 133 | Log.info('库存更新', item) 134 | }) 135 | }) 136 | 137 | updateData.forEach(item => { 138 | let stock = `${item['stock']}` 139 | let stock2 = `${item['stock2']}` 140 | let reStock = subSetByArray( 141 | stock.split(', '), 142 | stock2.split(', ') 143 | ) 144 | if (reStock.length) { 145 | item.stock = reStock.join(', ') 146 | that.sendMessage([item]) 147 | } 148 | }) 149 | } 150 | 151 | db.close() 152 | }) 153 | } 154 | ) 155 | } 156 | 157 | async sendMessage(array) { 158 | for await (let item of array) { 159 | const content = `\n价格：${item.price}\n${ 160 | item.stockUpdate ? '补货：' : '在售尺码：' 161 | }${item.stock}` 162 | bearychat 163 | .withText(`${item.title}`) 164 | .withAttachment({ 165 | title: item.subtitle, 166 | text: content, 167 | color: '#ffa500', 168 | images: [{ url: `https:${item.image}` }] 169 | }) 170 | .pushTo(WEBHOOK_URL) 171 | } 172 | } 173 | } 174 | 175 | module.exports = new juiceStore() 176 | 177 | setInterval(() => { 178 | new juiceStore().index() 179 | }, 4000) 180 | -------------------------------------------------------------------------------- /src/kith.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const MongoClient = require('mongodb').MongoClient 3 | const bearychat = require('bearyincoming') 4 | const cheerio = require('cheerio') 5 | const { 6 | subSet, 7 | subSetByArray, 8 | subSetWithStock, 9 | distinct 10 | } = require('../lib/utils') 11 | const Log = require('../lib/logger')('kith') 12 | 13 | const dbUrl = 'mongodb://localhost:27017/runoob' 14 | const KITH_MEN_URL = 'https://kith.com/collections/mens-footwear' 15 | const KITH_WOMEN_URL = 'https://kith.com/collections/womens-footwear' 16 | const WEBHOOK_URL = 17 | 'https://hook.bearychat.com/=bwHsk/incoming/0b3971fbc85ebed610d565afba2421b6' 18 | const query = {} 19 | const headers = { 20 | 'User-Agent': 21 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 22 | } 23 | const type = 'text' 24 | 25 | class kith { 26 | async index() { 27 | let menData = await this.fetchData(KITH_MEN_URL) 28 | this.filterData(menData) 29 | 30 | let womenData = await this.fetchData(KITH_WOMEN_URL) 31 | this.filterData(womenData) 32 | } 33 | 34 | async fetchData(url) { 35 | let data = {} 36 | try { 37 | data = await Request.get(url, query, headers, type) 38 | } catch (err) { 39 | Log.info('爬取数据错误') 40 | return null 41 | } 42 | 43 | return data 44 | } 45 | 46 | filterData(data) { 47 | if (!data) { 48 | return 49 | } 50 | 51 | let shoesData = [] 52 | let $ = cheerio.load(data) 53 | $('.collection-product').each((idx, ele) => { 54 | let $1 = cheerio.load(ele) 55 | const stock = [] 56 | const image = $1('.product-card__image-slide') 57 | .attr('style') 58 | .match(/\(([^)]*)\)/) 59 | $1('.product-card__variants li') 60 | .children() 61 | .each((idx, item) => stock.push($1(item).text())) 62 | shoesData.push({ 63 | stock: stock.join(', '), 64 | id: `https:${image[1]}`, 65 | title: $1('.product-card__title').text(), 66 | subtitle: $1('.product-card__color').text(), 67 | image: `https:${image[1]}`, 68 | price: $1('.product-card__price') 69 | .text() 70 | .replace(/[ \t\n]/g, '') 71 | }) 72 | }) 73 | 74 | this.saveData(shoesData, this) 75 | } 76 | 77 | saveData(shoesData, that) { 78 | let whereId = shoesData.map(item => item.id) 79 | let insertData = [] 80 | let updateData = [] 81 | MongoClient.connect( 82 | dbUrl, 83 | { useNewUrlParser: true, useUnifiedTopology: true }, 84 | function(err, db) { 85 | if (err) { 86 | Log.error('打开数据库错误', err) 87 | } 88 | var dbo = db.db('runoob') 89 | dbo 90 | .collection('kith') 91 | .find({ id: { $in: whereId } }) 92 | .toArray(function(err, result) { 93 | if (err) { 94 | Log.error('查询数据库错误', err) 95 | } 96 | if (!result.length) { 97 | insertData = shoesData 98 | } else if ( 99 | result.length && 100 | whereId.length && 101 | result.length !== whereId.length 102 | ) { 103 | // 取差集 104 | insertData = subSet(shoesData, result) 105 | } 106 | 107 | updateData = subSetWithStock(shoesData, result) 108 | 109 | if (insertData.length) { 110 | // 去重 111 | insertData = distinct(insertData) 112 | 113 | insertData.length && that.sendMessage(insertData) 114 | 115 | insertData.length && 116 | dbo 117 | .collection('kith') 118 | .insertMany(insertData, function(err, res) { 119 | if (err) { 120 | Log.error('插入数据库错误', err) 121 | } 122 | Log.info(`上新 | ${res.insertedCount}`, insertData) 123 | }) 124 | } 125 | 126 | if (updateData.length) { 127 | updateData = distinct(updateData) 128 | 129 | updateData.length && 130 | updateData.forEach(item => { 131 | let whereStr = { id: item.id } // 查询条件 132 | let updateStr = { $set: { stock: item.stock } } 133 | dbo 134 | .collection('kith') 135 | .updateOne(whereStr, updateStr, function(err, res) { 136 | if (err) { 137 | Log.error('更新数据库错误', err) 138 | } 139 | Log.info('库存更新', item) 140 | }) 141 | }) 142 | 143 | updateData.forEach(item => { 144 | let stock = `${item['stock']}` 145 | let stock2 = `${item['stock2']}` 146 | let reStock = subSetByArray( 147 | stock.split(', '), 148 | stock2.split(', ') 149 | ) 150 | if (reStock.length) { 151 | item.stock = reStock.join(', ') 152 | that.sendMessage([item]) 153 | } 154 | }) 155 | } 156 | 157 | db.close() 158 | }) 159 | } 160 | ) 161 | } 162 | 163 | async sendMessage(array) { 164 | for await (let item of array) { 165 | const content = `\n价格：${item.price}\n${ 166 | item.stockUpdate ? '补货：' : '在售尺码：' 167 | }${item.stock}` 168 | bearychat 169 | .withText(`${item.title}`) 170 | .withAttachment({ 171 | title: item.subtitle, 172 | text: content, 173 | color: '#ffa500', 174 | images: [{ url: item.image }] 175 | }) 176 | .pushTo(WEBHOOK_URL) 177 | } 178 | } 179 | } 180 | 181 | module.exports = new kith() 182 | 183 | setInterval(() => { 184 | new kith().index() 185 | }, 4000) 186 | -------------------------------------------------------------------------------- /src/footlocker.js: -------------------------------------------------------------------------------- 1 | const superagent = require('superagent') 2 | const Request = require('../lib/request') 3 | const MongoClient = require('mongodb').MongoClient 4 | const bearychat = require('bearyincoming') 5 | const { subSetWithStock, distinct } = require('../lib/utils') 6 | const Log = require('../lib/logger')('footlocker') 7 | 8 | const dbUrl = 'mongodb://localhost:27017/runoob' 9 | const MEN_URL = 10 | 'https://www.footlocker.com/api/products/search?query=%3Arelevance%3Abrand%3AJordan%3Agender%3AMen%27s%3Aproducttype%3AShoes¤tPage=&pageSize=48×tamp=4' 11 | const WOMEN_URL = 12 | 'https://www.footlocker.com/api/products/search?query=%3Arelevance%3Agender%3AWomen%27s%3Abrand%3AJordan%3Aproducttype%3AShoes¤tPage=&pageSize=48×tamp=4' 13 | const WEBHOOK_URL = 14 | 'https://hook.bearychat.com/=bwHsk/incoming/0b3971fbc85ebed610d565afba2421b6' 15 | const query = {} 16 | const headers = { 17 | 'User-Agent': 'PostmanRuntime/7.18.0' 18 | } 19 | 20 | class footlocker { 21 | async index() { 22 | const menData = await this.fetchData(MEN_URL) 23 | let men_products = this.filterData(menData) 24 | men_products = this.getStock(men_products) 25 | 26 | // const womenData = await this.fetchData(WOMEN_URL) 27 | // let women_products = this.filterData(womenData) 28 | // women_products.length && 29 | // women_products.forEach(item => { 30 | // this.fetchStockData(item).then(res => (item['stock'] = res)) 31 | // }) 32 | Log.info(men_products) 33 | } 34 | 35 | async fetchData(url) { 36 | let data = {} 37 | try { 38 | data = await Request.get(url, query, headers) 39 | } catch (err) { 40 | Log.info('爬取数据错误') 41 | return null 42 | } 43 | 44 | return data 45 | } 46 | 47 | getStock(dataSource) { 48 | if (!dataSource.length) { 49 | return dataSource 50 | } 51 | let currentIndex = 0 52 | let result = [] 53 | 54 | function fetchStockData() { 55 | if (currentIndex >= dataSource.length) { 56 | return 57 | } 58 | let item = dataSource[currentIndex] 59 | 60 | superagent 61 | .get(`https://www.footlocker.com/api/products/pdp/${item.id}`) 62 | .query({ 63 | timestamp: new Date().getTime() 64 | }) 65 | .set(headers) 66 | .end((err, res) => { 67 | if (err) { 68 | Log.info(`爬取${item.subtitle}鞋码数据错误`) 69 | currentIndex++ 70 | fetchStockData() 71 | } else { 72 | const { sellableUnits = [] } = res 73 | let sizeData = [] 74 | sellableUnits.length && 75 | sellableUnits.forEach(ele => { 76 | ele.stockLevelStatus === 'inStock' && 77 | ele.attributes[1].value === item.subtitle && 78 | sizeData.push(ele.attributes[0].value) 79 | }) 80 | item['stock'] = sizeData.join(', ') 81 | result.push(item) 82 | currentIndex++ 83 | fetchStockData() 84 | } 85 | }) 86 | } 87 | fetchStockData() 88 | return result 89 | } 90 | 91 | filterData(data) { 92 | if (!data) { 93 | return [] 94 | } 95 | 96 | let shoesData = [] 97 | data.products.length && 98 | data.products.forEach(item => { 99 | shoesData.push({ 100 | id: item.url, 101 | title: item.name, 102 | subtitle: item.baseOptions[0].selected.style, 103 | image: item.images[0].url, 104 | price: item.price.formattedValue 105 | }) 106 | }) 107 | 108 | return shoesData 109 | } 110 | 111 | saveData(shoesData, that) { 112 | let whereId = shoesData.map(item => item.id) 113 | let insertData = [] 114 | MongoClient.connect( 115 | dbUrl, 116 | { useNewUrlParser: true, useUnifiedTopology: true }, 117 | function(err, db) { 118 | if (err) { 119 | Log.error('打开数据库错误', err) 120 | } 121 | var dbo = db.db('runoob') 122 | dbo 123 | .collection('footlocker') 124 | .find({ id: { $in: whereId } }) 125 | .toArray(function(err, result) { 126 | if (err) { 127 | Log.error('查询数据库错误', err) 128 | } 129 | if (!result.length) { 130 | insertData = shoesData 131 | } else if ( 132 | result.length && 133 | whereId.length && 134 | result.length !== whereId.length 135 | ) { 136 | // 取差集 137 | insertData = subSet(shoesData, result) 138 | } 139 | 140 | if (insertData.length) { 141 | // 去重 142 | insertData = distinct(insertData) 143 | 144 | // insertData.length && that.sendMessage(insertData) 145 | 146 | insertData.length && 147 | dbo 148 | .collection('footlocker') 149 | .insertMany(insertData, function(err, res) { 150 | if (err) { 151 | Log.error('插入数据库错误', err) 152 | } 153 | Log.info(`上新 | ${res.insertedCount}`, insertData) 154 | db.close() 155 | }) 156 | } 157 | 158 | db.close() 159 | }) 160 | } 161 | ) 162 | } 163 | 164 | async sendMessage(array) { 165 | for await (let item of array) { 166 | const content = `\n价格：${item.price}\n在售尺码：${item.stock}` 167 | bearychat 168 | .withText(`${item.title}`) 169 | .withAttachment({ 170 | title: item.subtitle, 171 | text: content, 172 | color: '#ffa500', 173 | images: [{ url: item.image }] 174 | }) 175 | .pushTo(WEBHOOK_URL) 176 | } 177 | } 178 | } 179 | 180 | module.exports = new footlocker() 181 | new footlocker().index() 182 | // setInterval(() => { 183 | // new footlocker().index() 184 | // }, 5000) 185 | -------------------------------------------------------------------------------- /src/yeezysupply.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const MongoClient = require('mongodb').MongoClient 3 | const bearychat = require('bearyincoming') 4 | const cheerio = require('cheerio') 5 | const { 6 | subSetWithStock, 7 | subSetByArray, 8 | subSet, 9 | distinct 10 | } = require('../lib/utils') 11 | const Log = require('../lib/logger')('YEEZY') 12 | 13 | const dbUrl = 'mongodb://localhost:27017/runoob' 14 | const YEEZY_URL = 'https://yeezysupply.com' 15 | const WEBHOOK_URL = 16 | 'https://hook.bearychat.com/=bwHsk/incoming/981df263b8b35cc90929c1d27c4a9744' 17 | const query = {} 18 | const headers = { 19 | 'User-Agent': 20 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 21 | } 22 | const type = 'text' 23 | 24 | class yeezysupply { 25 | async index() { 26 | let requestData = await this.fetchData() 27 | 28 | this.saveData(requestData, this) 29 | } 30 | 31 | async fetchData() { 32 | let data = {} 33 | try { 34 | data = await Request.get(YEEZY_URL, query, headers, type) 35 | } catch (err) { 36 | Log.info('爬取数据错误') 37 | return null 38 | } 39 | 40 | return data 41 | } 42 | 43 | saveData(data, that) { 44 | if (!data) { 45 | return 46 | } 47 | 48 | let shoesData = [] 49 | let $ = cheerio.load(data) 50 | $('.MC__inner_for_side_by_side .js-product-json').each((idx, ele) => { 51 | const dataSource = JSON.parse($(ele).html()) 52 | const price = dataSource.price.toString() 53 | const stock = [] 54 | let saleDate = '' 55 | dataSource.variants.forEach(item => { 56 | if (item.available) stock.push(item.option1) 57 | }) 58 | if (!stock.length && dataSource.type === 'PLACEHOLDER') { 59 | $('.MC__inner_for_side_by_side .PI__desc').each((index, ele) => { 60 | if (index === idx) { 61 | const description = $(ele).html() 62 | const i = description.lastIndexOf('>') 63 | saleDate = description.substring(i + 1, description.length) 64 | } 65 | }) 66 | } 67 | shoesData.push({ 68 | saleDate, 69 | id: dataSource.id, 70 | title: dataSource.title, 71 | subtitle: dataSource.handle, 72 | image: dataSource.featured_image, 73 | price: price.substring(0, price.length - 2), 74 | stock: stock.join(', ') 75 | }) 76 | }) 77 | 78 | let whereId = shoesData.map(item => item.id) 79 | let insertData = [] 80 | let updateData = [] 81 | MongoClient.connect( 82 | dbUrl, 83 | { useNewUrlParser: true, useUnifiedTopology: true }, 84 | function(err, db) { 85 | if (err) { 86 | Log.error('打开数据库错误', err) 87 | } 88 | var dbo = db.db('runoob') 89 | dbo 90 | .collection('yeezysupply') 91 | .find({ id: { $in: whereId } }) 92 | .toArray(function(err, result) { 93 | if (err) { 94 | Log.error('查询数据库错误', err) 95 | } 96 | if (!result.length) { 97 | insertData = shoesData 98 | } else if ( 99 | result.length && 100 | whereId.length && 101 | result.length !== whereId.length 102 | ) { 103 | // 取差集 104 | insertData = subSet(shoesData, result) 105 | } 106 | 107 | updateData = subSetWithStock(shoesData, result) 108 | 109 | if (insertData.length) { 110 | // 去重 111 | insertData = distinct(insertData) 112 | 113 | insertData.length && that.sendMessage(insertData) 114 | 115 | insertData.length && 116 | dbo 117 | .collection('yeezysupply') 118 | .insertMany(insertData, function(err, res) { 119 | if (err) { 120 | Log.error('插入数据库错误', err) 121 | } 122 | Log.info(`上新 | ${res.insertedCount}`, insertData) 123 | }) 124 | } 125 | 126 | if (updateData.length) { 127 | updateData = distinct(updateData) 128 | 129 | updateData.length && 130 | updateData.forEach(item => { 131 | let whereStr = { id: item.id } // 查询条件 132 | let updateStr = { $set: { stock: item.stock } } 133 | dbo 134 | .collection('yeezysupply') 135 | .updateOne(whereStr, updateStr, function(err, res) { 136 | if (err) { 137 | Log.error('更新数据库错误', err) 138 | } 139 | Log.info('库存更新', item) 140 | }) 141 | }) 142 | 143 | updateData.forEach(item => { 144 | let stock = `${item['stock']}` 145 | let stock2 = `${item['stock2']}` 146 | let reStock = subSetByArray( 147 | stock.split(', '), 148 | stock2.split(', ') 149 | ) 150 | if (reStock.length) { 151 | item.stock = reStock.join(', ') 152 | that.sendMessage([item]) 153 | } 154 | }) 155 | } 156 | 157 | db.close() 158 | }) 159 | } 160 | ) 161 | } 162 | 163 | async sendMessage(array) { 164 | for await (let item of array) { 165 | const content = item.stock 166 | ? `\n价格：$${item.price}\n${ 167 | item.stockUpdate ? '补货：' : '在售尺码：' 168 | }${item.stock}` 169 | : `\n价格：$${item.price}\n发售日期：${item.saleDate}` 170 | bearychat 171 | .withText(`${item.title}`) 172 | .withAttachment({ 173 | title: item.subtitle, 174 | text: content, 175 | color: '#ffa500', 176 | images: [{ url: `https:${item.image}` }] 177 | }) 178 | .pushTo(WEBHOOK_URL) 179 | } 180 | } 181 | } 182 | 183 | module.exports = new yeezysupply() 184 | 185 | setInterval(() => { 186 | new yeezysupply().index() 187 | }, 5000) 188 | -------------------------------------------------------------------------------- /src/snkrsJp.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const bearychat = require('bearyincoming') 3 | const moment = require('moment') 4 | const MongoClient = require('mongodb').MongoClient 5 | const { subSet, distinct } = require('../lib/utils') 6 | const Log = require('../lib/logger')('snkrsJp') 7 | 8 | const dbUrl = 'mongodb://localhost:27017/runoob' 9 | const WEBHOOK_URL = 10 | 'https://hook.bearychat.com/=bwHsk/incoming/7d14237c44091fa1ba6fb0cc65272a47' 11 | const SNKRS_URL = 12 | 'https://api.nike.com/product_feed/threads/v2/?anchor=0&count=10&filter=marketplace%28JP%29&filter=language%28ja%29&filter=channelId%28010794e5-35fe-4e32-aaff-cd2c74f89d61%29&filter=exclusiveAccess%28true%2Cfalse%29&fields=active&fields=id&fields=lastFetchTime&fields=productInfo&fields=publishedContent.nodes&fields=publishedContent.properties.coverCard&fields=publishedContent.properties.productCard&fields=publishedContent.properties.products&fields=publishedContent.properties.publish.collections&fields=publishedContent.properties.relatedThreads&fields=publishedContent.properties.seo&fields=publishedContent.properties.threadType&fields=publishedContent.properties.custom&fields=publishedContent.properties.title' 13 | const headers = { 14 | 'User-Agent': 15 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 16 | } 17 | 18 | class snkrsJp { 19 | async index() { 20 | let requestData = await this.fetchData() 21 | 22 | this.saveData(requestData, this) 23 | } 24 | 25 | async fetchData() { 26 | let data = {} 27 | try { 28 | data = await Request.get(SNKRS_URL, {}, headers) 29 | } catch (err) { 30 | Log.info('爬取数据错误') 31 | return {} 32 | } 33 | 34 | return data 35 | } 36 | 37 | saveData(data, that) { 38 | if (!data.objects) { 39 | return 40 | } 41 | let shoesData = [] 42 | data.objects.length && 43 | data.objects.forEach(item => { 44 | let title = item.publishedContent.properties.seo.title 45 | title.search('【NIKE公式】') !== -1 ? (title = title.substr(8)) : null 46 | item.publishedContent.properties.custom.restricted 47 | ? (title = `【专属】 ${title}`) 48 | : null 49 | 50 | if (item.productInfo && item.productInfo.length) { 51 | item.productInfo.forEach(ele => { 52 | let stock = '' 53 | ele.skus.length && 54 | ele.skus.forEach(cont => { 55 | stock = 56 | stock === '' ? cont.nikeSize : `${stock}, ${cont.nikeSize}` 57 | }) 58 | 59 | const method = ele.launchView 60 | ? ele.launchView.method 61 | : ele.merchProduct.publishType 62 | 63 | const time = ele.launchView 64 | ? moment(ele.launchView.startEntryDate).format( 65 | 'YYYY-MM-DD HH:mm:ss' 66 | ) 67 | : moment(ele.merchProduct.commerceStartDate).format( 68 | 'YYYY-MM-DD HH:mm:ss' 69 | ) 70 | 71 | shoesData.push({ 72 | title: 73 | ele.merchProduct.productType === 'APPAREL' 74 | ? `【服装】 ${title}` 75 | : title, 76 | method, 77 | time, 78 | stock, 79 | activity: false, 80 | id: ele.availability.productId, 81 | subtitle: ele.productContent.subtitle, 82 | price: ele.merchPrice.currentPrice, 83 | styleColor: ele.merchProduct.styleColor, 84 | image: ele.imageUrls.productImageUrl 85 | }) 86 | }) 87 | } else { 88 | shoesData.push({ 89 | activity: true, 90 | id: item.publishedContent.properties.coverCard.id, 91 | title: `【新活动】 ${item.publishedContent.properties.coverCard 92 | .properties.title || 93 | title || 94 | '-'}`, 95 | subtitle: 96 | item.publishedContent.properties.coverCard.properties.subtitle || 97 | item.publishedContent.properties.seo.description || 98 | '-', 99 | image: 100 | item.publishedContent.properties.coverCard.properties.portraitURL 101 | }) 102 | } 103 | }) 104 | 105 | // 查询是否有该鞋款 106 | let whereId = shoesData.map(item => item.id) 107 | let insertData = [] 108 | MongoClient.connect( 109 | dbUrl, 110 | { useNewUrlParser: true, useUnifiedTopology: true }, 111 | function(err, db) { 112 | if (err) { 113 | Log.error('打开数据库错误', err) 114 | } 115 | var dbo = db.db('runoob') 116 | dbo 117 | .collection('snkrsJp') 118 | .find({ id: { $in: whereId } }) 119 | .toArray(function(err, result) { 120 | if (err) { 121 | Log.error('查询数据库错误', err) 122 | } 123 | if (!result.length) { 124 | insertData = shoesData 125 | } else if ( 126 | result.length && 127 | whereId.length && 128 | result.length !== whereId.length 129 | ) { 130 | // 取差集 131 | insertData = subSet(shoesData, result) 132 | } 133 | 134 | if (insertData.length) { 135 | // 去重 136 | insertData = distinct(insertData) 137 | insertData.length && that.sendMessage(insertData) 138 | 139 | insertData.length && 140 | dbo 141 | .collection('snkrsJp') 142 | .insertMany(insertData, function(err, res) { 143 | if (err) { 144 | Log.error('插入数据库错误', err) 145 | } 146 | Log.info(`上新 | ${res.insertedCount}`, insertData) 147 | db.close() 148 | }) 149 | } 150 | 151 | db.close() 152 | }) 153 | } 154 | ) 155 | } 156 | 157 | async sendMessage(array) { 158 | for await (let item of array) { 159 | if (item.activity) { 160 | bearychat 161 | .withText(`${item.title}`) 162 | .withAttachment({ 163 | title: item.subtitle, 164 | text: '-', 165 | color: '#ffa500', 166 | images: [{ url: item.image }] 167 | }) 168 | .pushTo(WEBHOOK_URL) 169 | } else { 170 | const content = `\n发售时间：${item.time}\n发售方式：${item.method === 'FLOW' ? 'FLOW 先到先得' : item.method === 'LEO' ? 'LEO 2分钟抽70%' : 'DAN 15分钟随机抽取'}\n价格：${item.price}\n货号：${item.styleColor}\nsize：${item.stock}` 171 | bearychat 172 | .withText(`${item.title}`) 173 | .withAttachment({ 174 | title: item.subtitle, 175 | text: content, 176 | color: '#ffa500', 177 | images: [{ url: item.image }] 178 | }) 179 | .pushTo(WEBHOOK_URL) 180 | } 181 | } 182 | } 183 | } 184 | 185 | module.exports = new snkrsJp() 186 | 187 | setInterval(() => { 188 | new snkrsJp().index() 189 | }, 4000) 190 | -------------------------------------------------------------------------------- /src/snkrsCn.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const bearychat = require('bearyincoming') 3 | const moment = require('moment') 4 | const MongoClient = require('mongodb').MongoClient 5 | const { subSet, distinct } = require('../lib/utils') 6 | const Log = require('../lib/logger')('snkrsCn') 7 | 8 | const dbUrl = 'mongodb://localhost:27017/runoob' 9 | const WEBHOOK_URL = 10 | 'https://hook.bearychat.com/=bwHsk/incoming/8f3470e3a9b43e807289ccdba8f4566d' 11 | const SNKRS_URL = 12 | 'https://api.nike.com/product_feed/threads/v2/?anchor=0&count=10&filter=marketplace%28CN%29&filter=language%28zh-Hans%29&filter=channelId%28010794e5-35fe-4e32-aaff-cd2c74f89d61%29&filter=exclusiveAccess%28true%2Cfalse%29&fields=active&fields=id&fields=lastFetchTime&fields=productInfo&fields=publishedContent.nodes&fields=publishedContent.properties.coverCard&fields=publishedContent.properties.productCard&fields=publishedContent.properties.products&fields=publishedContent.properties.publish.collections&fields=publishedContent.properties.relatedThreads&fields=publishedContent.properties.seo&fields=publishedContent.properties.threadType&fields=publishedContent.properties.custom&fields=publishedContent.properties.title' 13 | const headers = { 14 | 'User-Agent': 15 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 16 | } 17 | 18 | class snkrsCn { 19 | async index() { 20 | let requestData = await this.fetchData() 21 | 22 | this.saveData(requestData, this) 23 | } 24 | 25 | async fetchData() { 26 | let data = {} 27 | try { 28 | data = await Request.get(SNKRS_URL, {}, headers) 29 | } catch (err) { 30 | Log.info('爬取数据错误') 31 | return {} 32 | } 33 | 34 | return data 35 | } 36 | 37 | saveData(data, that) { 38 | if (!data.objects) { 39 | return 40 | } 41 | let shoesData = [] 42 | data.objects.length && 43 | data.objects.forEach(item => { 44 | let title = item.publishedContent.properties.seo.title 45 | title.search('发布日期') !== -1 46 | ? (title = title.substr(0, title.length - 4)) 47 | : null 48 | item.publishedContent.properties.custom.restricted 49 | ? (title = `【专属】 ${title}`) 50 | : null 51 | 52 | if (item.productInfo && item.productInfo.length) { 53 | item.productInfo.forEach(ele => { 54 | let stock = '' 55 | ele.skus.length && 56 | ele.skus.forEach(cont => { 57 | stock = 58 | stock === '' ? cont.nikeSize : `${stock}, ${cont.nikeSize}` 59 | }) 60 | 61 | const method = ele.launchView 62 | ? ele.launchView.method 63 | : ele.merchProduct.publishType 64 | 65 | const time = ele.launchView 66 | ? moment(ele.launchView.startEntryDate).format( 67 | 'YYYY-MM-DD HH:mm:ss' 68 | ) 69 | : moment(ele.merchProduct.commerceStartDate).format( 70 | 'YYYY-MM-DD HH:mm:ss' 71 | ) 72 | 73 | shoesData.push({ 74 | title: 75 | ele.merchProduct.productType === 'APPAREL' 76 | ? `【服装系列】 ${title}` 77 | : title, 78 | method, 79 | time, 80 | stock, 81 | activity: false, 82 | id: ele.availability.productId, 83 | subtitle: ele.productContent.subtitle, 84 | price: ele.merchPrice.currentPrice, 85 | styleColor: ele.merchProduct.styleColor, 86 | image: ele.imageUrls.productImageUrl 87 | }) 88 | }) 89 | } else { 90 | shoesData.push({ 91 | activity: true, 92 | id: item.publishedContent.properties.coverCard.id, 93 | title: `【新活动】 ${item.publishedContent.properties.coverCard 94 | .properties.title || 95 | title || 96 | '-'}`, 97 | subtitle: 98 | item.publishedContent.properties.coverCard.properties.subtitle || 99 | item.publishedContent.properties.seo.description || 100 | '-', 101 | image: 102 | item.publishedContent.properties.coverCard.properties.portraitURL 103 | }) 104 | } 105 | }) 106 | 107 | // 查询是否有该鞋款 108 | let whereId = shoesData.map(item => item.id) 109 | let insertData = [] 110 | MongoClient.connect( 111 | dbUrl, 112 | { useNewUrlParser: true, useUnifiedTopology: true }, 113 | function(err, db) { 114 | if (err) { 115 | Log.error('打开数据库错误', err) 116 | } 117 | var dbo = db.db('runoob') 118 | dbo 119 | .collection('snkrsCn') 120 | .find({ id: { $in: whereId } }) 121 | .toArray(function(err, result) { 122 | if (err) { 123 | Log.error('查询数据库错误', err) 124 | } 125 | if (!result.length) { 126 | insertData = shoesData 127 | } else if ( 128 | result.length && 129 | whereId.length && 130 | result.length !== whereId.length 131 | ) { 132 | // 取差集 133 | insertData = subSet(shoesData, result) 134 | } 135 | 136 | if (insertData.length) { 137 | // 去重 138 | insertData = distinct(insertData) 139 | 140 | insertData.length && that.sendMessage(insertData) 141 | 142 | insertData.length && 143 | dbo 144 | .collection('snkrsCn') 145 | .insertMany(insertData, function(err, res) { 146 | if (err) { 147 | Log.error('插入数据库错误', err) 148 | } 149 | Log.info(`上新 | ${res.insertedCount}`, insertData) 150 | db.close() 151 | }) 152 | } 153 | 154 | db.close() 155 | }) 156 | } 157 | ) 158 | } 159 | 160 | async sendMessage(array) { 161 | for await (let item of array) { 162 | if (item.activity) { 163 | bearychat 164 | .withText(`${item.title}`) 165 | .withAttachment({ 166 | title: item.subtitle, 167 | text: '-', 168 | color: '#ffa500', 169 | images: [{ url: item.image }] 170 | }) 171 | .pushTo(WEBHOOK_URL) 172 | } else { 173 | const content = `\n发售时间：${item.time}\n发售方式：${item.method === 'FLOW' ? 'FLOW 先到先得' : item.method === 'LEO' ? 'LEO 2分钟抽70%' : 'DAN 15分钟随机抽取'}\n价格：${item.price}\n货号：${item.styleColor}\nsize：${item.stock}` 174 | bearychat 175 | .withText(`${item.title}`) 176 | .withAttachment({ 177 | title: item.subtitle, 178 | text: content, 179 | color: '#ffa500', 180 | images: [{ url: item.image }] 181 | }) 182 | .pushTo(WEBHOOK_URL) 183 | } 184 | } 185 | } 186 | } 187 | 188 | module.exports = new snkrsCn() 189 | 190 | setInterval(() => { 191 | new snkrsCn().index() 192 | }, 4000) 193 | -------------------------------------------------------------------------------- /src/snkrsUk.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const bearychat = require('bearyincoming') 3 | const moment = require('moment') 4 | const MongoClient = require('mongodb').MongoClient 5 | const { subSet, distinct } = require('../lib/utils') 6 | const Log = require('../lib/logger')('snkrsUk') 7 | 8 | const dbUrl = 'mongodb://localhost:27017/runoob' 9 | const WEBHOOK_URL = 10 | 'https://hook.bearychat.com/=bwHsk/incoming/0e7e7238e5563c1290605910985e7c96' 11 | const SNKRS_URL = 12 | 'https://api.nike.com/product_feed/threads/v2/?anchor=0&count=10&filter=marketplace%28GB%29&filter=language%28en-GB%29&filter=channelId%28010794e5-35fe-4e32-aaff-cd2c74f89d61%29&filter=exclusiveAccess%28true%2Cfalse%29&fields=active&fields=id&fields=lastFetchTime&fields=productInfo&fields=publishedContent.nodes&fields=publishedContent.properties.coverCard&fields=publishedContent.properties.productCard&fields=publishedContent.properties.products&fields=publishedContent.properties.publish.collections&fields=publishedContent.properties.relatedThreads&fields=publishedContent.properties.seo&fields=publishedContent.properties.threadType&fields=publishedContent.properties.custom&fields=publishedContent.properties.title' 13 | const headers = { 14 | 'User-Agent': 15 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 16 | } 17 | 18 | class snkrsUk { 19 | async index() { 20 | let requestData = await this.fetchData() 21 | 22 | this.saveData(requestData, this) 23 | } 24 | 25 | async fetchData() { 26 | let data = {} 27 | try { 28 | data = await Request.get(SNKRS_URL, {}, headers) 29 | } catch (err) { 30 | Log.info('爬取数据错误') 31 | return {} 32 | } 33 | 34 | return data 35 | } 36 | 37 | saveData(data, that) { 38 | if (!data.objects) { 39 | return 40 | } 41 | let shoesData = [] 42 | data.objects.length && 43 | data.objects.forEach(item => { 44 | let title = item.publishedContent.properties.seo.title 45 | title.search('Release Date') !== -1 46 | ? (title = title.substr(0, title.length - 12)) 47 | : null 48 | item.publishedContent.properties.custom.restricted 49 | ? (title = `【专属】 ${title}`) 50 | : null 51 | 52 | if (item.productInfo && item.productInfo.length) { 53 | item.productInfo.forEach(ele => { 54 | let stock = '' 55 | ele.skus.length && 56 | ele.skus.forEach(cont => { 57 | stock = 58 | stock === '' ? cont.nikeSize : `${stock}, ${cont.nikeSize}` 59 | }) 60 | 61 | const method = ele.launchView 62 | ? ele.launchView.method 63 | : ele.merchProduct.publishType 64 | 65 | const time = ele.launchView 66 | ? moment(ele.launchView.startEntryDate).format( 67 | 'YYYY-MM-DD HH:mm:ss' 68 | ) 69 | : moment(ele.merchProduct.commerceStartDate).format( 70 | 'YYYY-MM-DD HH:mm:ss' 71 | ) 72 | 73 | shoesData.push({ 74 | title: 75 | ele.merchProduct.productType === 'APPAREL' 76 | ? `【服装】 ${title}` 77 | : title, 78 | method, 79 | time, 80 | stock, 81 | activity: false, 82 | id: ele.availability.productId, 83 | subtitle: ele.productContent.subtitle, 84 | price: ele.merchPrice.currentPrice, 85 | styleColor: ele.merchProduct.styleColor, 86 | image: ele.imageUrls.productImageUrl 87 | }) 88 | }) 89 | } else { 90 | shoesData.push({ 91 | activity: true, 92 | id: item.publishedContent.properties.coverCard.id, 93 | title: `【新活动】 ${item.publishedContent.properties.coverCard 94 | .properties.title || 95 | title || 96 | '-'}`, 97 | subtitle: 98 | item.publishedContent.properties.coverCard.properties.subtitle || 99 | item.publishedContent.properties.seo.description || 100 | '-', 101 | image: 102 | item.publishedContent.properties.coverCard.properties.portraitURL 103 | }) 104 | } 105 | }) 106 | 107 | // 查询是否有该鞋款 108 | let whereId = shoesData.map(item => item.id) 109 | let insertData = [] 110 | MongoClient.connect( 111 | dbUrl, 112 | { useNewUrlParser: true, useUnifiedTopology: true }, 113 | function(err, db) { 114 | if (err) { 115 | Log.error('打开数据库错误', err) 116 | } 117 | var dbo = db.db('runoob') 118 | dbo 119 | .collection('snkrsUk') 120 | .find({ id: { $in: whereId } }) 121 | .toArray(function(err, result) { 122 | if (err) { 123 | Log.error('查询数据库错误', err) 124 | } 125 | if (!result.length) { 126 | insertData = shoesData 127 | } else if ( 128 | result.length && 129 | whereId.length && 130 | result.length !== whereId.length 131 | ) { 132 | // 取差集 133 | insertData = subSet(shoesData, result) 134 | } 135 | 136 | if (insertData.length) { 137 | // 去重 138 | insertData = distinct(insertData) 139 | 140 | insertData.length && that.sendMessage(insertData) 141 | 142 | insertData.length && 143 | dbo 144 | .collection('snkrsUk') 145 | .insertMany(insertData, function(err, res) { 146 | if (err) { 147 | Log.error('插入数据库错误', err) 148 | } 149 | Log.info(`上新 | ${res.insertedCount}`, insertData) 150 | db.close() 151 | }) 152 | } 153 | 154 | db.close() 155 | }) 156 | } 157 | ) 158 | } 159 | 160 | async sendMessage(array) { 161 | for await (let item of array) { 162 | if (item.activity) { 163 | bearychat 164 | .withText(`${item.title}`) 165 | .withAttachment({ 166 | title: item.subtitle, 167 | text: '-', 168 | color: '#ffa500', 169 | images: [{ url: item.image }] 170 | }) 171 | .pushTo(WEBHOOK_URL) 172 | } else { 173 | const content = `\n发售时间：${item.time}\n发售方式：${item.method === 'FLOW' ? 'FLOW 先到先得' : item.method === 'LEO' ? 'LEO 2分钟抽70%' : 'DAN 15分钟随机抽取'}\n价格：${item.price}\n货号：${item.styleColor}\nsize：${item.stock}` 174 | bearychat 175 | .withText(`${item.title}`) 176 | .withAttachment({ 177 | title: item.subtitle, 178 | text: content, 179 | color: '#ffa500', 180 | images: [{ url: item.image }] 181 | }) 182 | .pushTo(WEBHOOK_URL) 183 | } 184 | } 185 | } 186 | } 187 | 188 | module.exports = new snkrsUk() 189 | 190 | setInterval(() => { 191 | new snkrsUk().index() 192 | }, 4000) 193 | -------------------------------------------------------------------------------- /src/snkrsUs.js: -------------------------------------------------------------------------------- 1 | const Request = require('../lib/request') 2 | const bearychat = require('bearyincoming') 3 | const moment = require('moment') 4 | const MongoClient = require('mongodb').MongoClient 5 | const { subSet, distinct } = require('../lib/utils') 6 | const Log = require('../lib/logger')('snkrsUs') 7 | 8 | const dbUrl = 'mongodb://localhost:27017/runoob' 9 | const WEBHOOK_URL = 10 | 'https://hook.bearychat.com/=bwHsk/incoming/eb137f9c3e3b7e5250c5933634718f43' 11 | const SNKRS_URL = 12 | 'https://api.nike.com/product_feed/threads/v2/?anchor=0&count=10&filter=marketplace%28US%29&filter=language%28en%29&filter=channelId%28010794e5-35fe-4e32-aaff-cd2c74f89d61%29&filter=exclusiveAccess%28true%2Cfalse%29&fields=active&fields=id&fields=lastFetchTime&fields=productInfo&fields=publishedContent.nodes&fields=publishedContent.properties.coverCard&fields=publishedContent.properties.productCard&fields=publishedContent.properties.products&fields=publishedContent.properties.publish.collections&fields=publishedContent.properties.relatedThreads&fields=publishedContent.properties.seo&fields=publishedContent.properties.threadType&fields=publishedContent.properties.custom&fields=publishedContent.properties.title' 13 | const headers = { 14 | 'User-Agent': 15 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' 16 | } 17 | 18 | class snkrsUs { 19 | async index() { 20 | let requestData = await this.fetchData() 21 | 22 | this.saveData(requestData, this) 23 | } 24 | 25 | async fetchData() { 26 | let data = {} 27 | try { 28 | data = await Request.get(SNKRS_URL, {}, headers) 29 | } catch (err) { 30 | Log.info('爬取数据错误') 31 | return {} 32 | } 33 | 34 | return data 35 | } 36 | 37 | saveData(data, that) { 38 | if (!data.objects) { 39 | return 40 | } 41 | let shoesData = [] 42 | data.objects.length && 43 | data.objects.forEach(item => { 44 | let title = item.publishedContent.properties.seo.title 45 | title.search('Release Date') !== -1 46 | ? (title = title.substr(0, title.length - 12)) 47 | : null 48 | item.publishedContent.properties.custom.restricted 49 | ? (title = `【专属】 ${title}`) 50 | : null 51 | 52 | if (item.productInfo && item.productInfo.length) { 53 | item.productInfo.forEach(ele => { 54 | let stock = '' 55 | ele.skus.length && 56 | ele.skus.forEach(cont => { 57 | stock = 58 | stock === '' ? cont.nikeSize : `${stock}, ${cont.nikeSize}` 59 | }) 60 | 61 | const method = ele.launchView 62 | ? ele.launchView.method 63 | : ele.merchProduct.publishType 64 | 65 | const time = ele.launchView 66 | ? moment(ele.launchView.startEntryDate).format( 67 | 'YYYY-MM-DD HH:mm:ss' 68 | ) 69 | : moment(ele.merchProduct.commerceStartDate).format( 70 | 'YYYY-MM-DD HH:mm:ss' 71 | ) 72 | 73 | shoesData.push({ 74 | title: 75 | ele.merchProduct.productType === 'APPAREL' 76 | ? `【服装】 ${title}` 77 | : title, 78 | method, 79 | time, 80 | stock, 81 | activity: false, 82 | id: ele.availability.productId, 83 | subtitle: ele.productContent.subtitle, 84 | price: ele.merchPrice.currentPrice, 85 | styleColor: ele.merchProduct.styleColor, 86 | image: ele.imageUrls.productImageUrl 87 | }) 88 | }) 89 | } else { 90 | shoesData.push({ 91 | activity: true, 92 | id: item.publishedContent.properties.coverCard.id, 93 | title: `【新活动】 ${item.publishedContent.properties.coverCard 94 | .properties.title || 95 | title || 96 | '-'}`, 97 | subtitle: 98 | item.publishedContent.properties.coverCard.properties.subtitle || 99 | item.publishedContent.properties.seo.description || 100 | '-', 101 | image: 102 | item.publishedContent.properties.coverCard.properties.portraitURL 103 | }) 104 | } 105 | }) 106 | 107 | // 查询是否有该鞋款 108 | let whereId = shoesData.map(item => item.id) 109 | let insertData = [] 110 | MongoClient.connect( 111 | dbUrl, 112 | { useNewUrlParser: true, useUnifiedTopology: true }, 113 | function(err, db) { 114 | if (err) { 115 | Log.error('打开数据库错误', err) 116 | db.close() 117 | } 118 | var dbo = db.db('runoob') 119 | dbo 120 | .collection('snkrsUs') 121 | .find({ id: { $in: whereId } }) 122 | .toArray(function(err, result) { 123 | if (err) { 124 | Log.error('查询数据库错误', err) 125 | db.close() 126 | } 127 | if (!result.length) { 128 | insertData = shoesData 129 | } else if ( 130 | result.length && 131 | whereId.length && 132 | result.length !== whereId.length 133 | ) { 134 | // 取差集 135 | insertData = subSet(shoesData, result) 136 | } 137 | 138 | if (insertData.length) { 139 | // 去重 140 | insertData = distinct(insertData) 141 | 142 | insertData.length && that.sendMessage(insertData) 143 | 144 | insertData.length && 145 | dbo 146 | .collection('snkrsUs') 147 | .insertMany(insertData, function(err, res) { 148 | if (err) { 149 | Log.error('插入数据库错误', err) 150 | db.close() 151 | } 152 | Log.info(`上新 | ${res.insertedCount}`, insertData) 153 | db.close() 154 | }) 155 | } 156 | 157 | db.close() 158 | }) 159 | } 160 | ) 161 | } 162 | 163 | async sendMessage(array) { 164 | for await (let item of array) { 165 | if (item.activity) { 166 | bearychat 167 | .withText(`${item.title}`) 168 | .withAttachment({ 169 | title: item.subtitle, 170 | text: '-', 171 | color: '#ffa500', 172 | images: [{ url: item.image }] 173 | }) 174 | .pushTo(WEBHOOK_URL) 175 | } else { 176 | const content = `\n发售时间：${item.time}\n发售方式：${item.method === 'FLOW' ? 'FLOW 先到先得' : item.method === 'LEO' ? 'LEO 2分钟抽70%' : 'DAN 15分钟随机抽取'}\n价格：${item.price}\n货号：${item.styleColor}\nsize：${item.stock}` 177 | bearychat 178 | .withText(`${item.title}`) 179 | .withAttachment({ 180 | title: item.subtitle, 181 | text: content, 182 | color: '#ffa500', 183 | images: [{ url: item.image }] 184 | }) 185 | .pushTo(WEBHOOK_URL) 186 | } 187 | } 188 | } 189 | } 190 | 191 | module.exports = new snkrsUs() 192 | 193 | setInterval(() => { 194 | new snkrsUs().index() 195 | }, 4000) 196 | --------------------------------------------------------------------------------