├── .gitignore ├── .images └── dashboard.png ├── DuTracker ├── __init__.py ├── db.py ├── items.py ├── middlewares.py ├── pipelines.py ├── settings.py ├── sign │ ├── __init__.py │ ├── sign.js │ └── sign.py ├── spiders │ ├── __init__.py │ ├── brand.py │ ├── product.py │ ├── serie.py │ ├── sign.js │ └── tracker.py ├── tsdb.py └── utils │ ├── __init__.py │ ├── log.py │ └── urls.py ├── Pipfile ├── Pipfile.lock ├── README.md ├── docker-compose.yml ├── dt.py ├── scrapy.cfg └── template.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | 118 | # Pycharm 119 | .idea/ 120 | 121 | # Database 122 | *.sqlite 123 | 124 | # docker volumes 125 | volumes/ 126 | -------------------------------------------------------------------------------- /.images/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuCcc/DuTracker/1dca5ef647651f46b0e918bcdddd0268265f3286/.images/dashboard.png -------------------------------------------------------------------------------- /DuTracker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuCcc/DuTracker/1dca5ef647651f46b0e918bcdddd0268265f3286/DuTracker/__init__.py -------------------------------------------------------------------------------- /DuTracker/db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/10 13:43 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | 7 | from pony.orm import * 8 | import arrow 9 | 10 | db = Database() 11 | 12 | 13 | class Product(db.Entity): 14 | id = PrimaryKey(int, auto=True) 15 | url = Optional(str) 16 | title = Optional(str) 17 | soldNum = Optional(int) 18 | logo = Optional(str) 19 | categoryId = Optional(int) 20 | images = Optional(StrArray) 21 | sellDate = Optional(str) 22 | articleNumber = Optional(str) 23 | authPrice = Optional(int) 24 | goodsId = Optional(int) 25 | sizeList = Optional(StrArray) 26 | json = Optional(Json) 27 | datetime = Optional(str, default=arrow.now().format('YYYY-MM-DD HH:mm:ss')) 28 | brand = Optional(str) 29 | serie = Optional(str) 30 | imageAndText = Optional(str) 31 | 32 | 33 | db.bind(provider='sqlite', filename='../product.sqlite', create_db=True) 34 | db.generate_mapping(create_tables=True) 35 | -------------------------------------------------------------------------------- /DuTracker/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | 10 | 11 | class ProductInfo(scrapy.Item): 12 | id = scrapy.Field() 13 | title = scrapy.Field() 14 | name = scrapy.Field() 15 | 16 | 17 | class ProductItem(scrapy.Item): 18 | id = scrapy.Field() 19 | url = scrapy.Field() 20 | title = scrapy.Field() 21 | soldNum = scrapy.Field() 22 | logo = scrapy.Field() 23 | brandId = scrapy.Field() 24 | categoryId = scrapy.Field() 25 | images = scrapy.Field() 26 | sellDate = scrapy.Field() 27 | articleNumber = scrapy.Field() 28 | authPrice = scrapy.Field() 29 | goodsId = scrapy.Field() 30 | sizeList = scrapy.Field() 31 | imageAndText = scrapy.Field() 32 | detailJson = scrapy.Field() 33 | 34 | 35 | class PriceItem(scrapy.Item): 36 | id = scrapy.Field() 37 | brand = scrapy.Field() 38 | serie = scrapy.Field() 39 | title = scrapy.Field() 40 | size = scrapy.Field() 41 | formatSize = scrapy.Field() 42 | price = scrapy.Field() 43 | soldNum = scrapy.Field() 44 | -------------------------------------------------------------------------------- /DuTracker/middlewares.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | import random 9 | import json 10 | from scrapy.exceptions import IgnoreRequest 11 | from scrapy.downloadermiddlewares.retry import RetryMiddleware 12 | 13 | from DuTracker.utils.log import log 14 | from DuTracker.db import * 15 | 16 | 17 | def checkJson(string): 18 | try: 19 | json.loads(string) 20 | except Exception as e: 21 | return False 22 | else: 23 | return True 24 | 25 | 26 | class RandomUserAgent(object): 27 | def __init__(self, user_agent): 28 | super(RandomUserAgent, self).__init__() 29 | self.user_agent = user_agent 30 | 31 | @classmethod 32 | def from_crawler(cls, crawler): 33 | return cls(crawler.settings.get('RANDOM_USER_AGENT')) 34 | 35 | def process_request(self, request, spider): 36 | request.headers['User-Agent'] = random.choice(self.user_agent) 37 | 38 | 39 | class RandomProxy(object): 40 | @classmethod 41 | def from_crawler(cls, crawler): 42 | settings = crawler.settings 43 | return cls(settings.get('PROXY_URL')) 44 | 45 | def __init__(self, proxy): 46 | self.proxy = proxy 47 | 48 | def process_request(self, request, spider): 49 | request.meta['proxy'] = self.proxy 50 | 51 | 52 | class RetryException(RetryMiddleware): 53 | def process_response(self, request, response, spider): 54 | if request.meta.get('dont_retry', False): 55 | return response 56 | data = response.body_as_unicode() 57 | if not checkJson(data): 58 | log.warn(f'返回Json格式错误无法解析 目标页面 {request.url} 返回数据 {response.body_as_unicode()} 开始重试') 59 | return self._retry(request, 'Json Format Error', spider) or response 60 | json_data = json.loads(data) 61 | msg = json_data['msg'] 62 | status = json_data['status'] 63 | if status == 403: 64 | log.warn(f'目标页面{request.url} 拒绝访问 提示信息 忽略请求') 65 | raise IgnoreRequest() 66 | if status != 200: 67 | log.warn(f'返回Json状态错误 目标页面 {request.url} 提示信息 {msg} 开始重试') 68 | return self._retry(request, 'Json Status Error', spider) or response 69 | return response 70 | 71 | def process_exception(self, request, exception, spider): 72 | if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \ 73 | and not request.meta.get('dont_retry', False): 74 | return self._retry(request, exception, spider) 75 | 76 | def _retry(self, request, reason, spider): 77 | retries = request.meta.get('retry_times', 0) + 1 78 | 79 | retry_times = self.max_retry_times 80 | 81 | if 'max_retry_times' in request.meta: 82 | retry_times = request.meta['max_retry_times'] 83 | 84 | stats = spider.crawler.stats 85 | 86 | if retries <= retry_times: 87 | retryreq = request.copy() 88 | retryreq.meta['retry_times'] = retries 89 | retryreq.dont_filter = True 90 | retryreq.priority = request.priority + self.priority_adjust 91 | 92 | stats.inc_value('retry/count') 93 | stats.inc_value('retry/reason_count/%s' % reason) 94 | log.debug(f'第 [{retries}] 次重试请求 目标页面 {request.url} ') 95 | return retryreq 96 | else: 97 | stats.inc_value('retry/max_reached') 98 | log.error(f'重试请求达最大次数 目标页面 {request.url} 已放弃') 99 | raise IgnoreRequest() 100 | -------------------------------------------------------------------------------- /DuTracker/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | from DuTracker.db import * 10 | from DuTracker.utils.log import log 11 | from DuTracker.tsdb import influxdb, gen_points 12 | 13 | import traceback 14 | 15 | 16 | class SaveProductId(): 17 | @db_session 18 | def process_item(self, item, spider): 19 | pid = item.get('id') 20 | title = item.get('title') 21 | name = item.get('name') 22 | if Product.exists(id=pid): 23 | p = Product[pid] 24 | else: 25 | p = Product(id=pid) 26 | p.title = title 27 | if spider.name == 'brand': 28 | p.brand = name 29 | elif spider.name == 'serie': 30 | p.serie = name 31 | log.success(f'商品：{title} 编号：{pid}') 32 | return item 33 | 34 | 35 | class SaveProductItem(): 36 | @db_session 37 | def process_item(self, item, spider): 38 | pid = item.get('id') 39 | title = item.get('title') 40 | articleNumber = item.get('articleNumber') 41 | url = item.get('url') 42 | soldNum = item.get('soldNum') 43 | logo = item.get('logo') 44 | categoryId = item.get('categoryId') 45 | images = item.get('images') 46 | sellDate = item.get('sellDate') 47 | authPrice = item.get('authPrice') 48 | goodsId = item.get('goodsId') 49 | sizeList = item.get('sizeList') 50 | imageAndText = item.get('imageAndText') 51 | detailJson = item.get('detailJson') 52 | if not Product.exists(id=pid): 53 | p = Product(id=pid) 54 | else: 55 | p = Product[pid] 56 | p.url = url 57 | p.title = title 58 | p.soldNum = soldNum 59 | p.logo = logo 60 | p.categoryId = categoryId 61 | p.images = images 62 | p.sellDate = sellDate 63 | p.articleNumber = articleNumber 64 | p.authPrice = authPrice 65 | p.goodsId = goodsId 66 | p.sizeList = sizeList 67 | p.imageAndText = imageAndText 68 | p.json = detailJson 69 | log.success(f'商品:{title} 编号：{pid} 发售日期：{sellDate} 售出量: {soldNum} ') 70 | return item 71 | 72 | 73 | class SavePriceItem(object): 74 | def process_item(self, item, spider): 75 | pid = item.get('id') 76 | brand = item.get('brand') 77 | serie = item.get('serie') 78 | title = item.get('title') 79 | size = item.get('size') 80 | formatSize = item.get('formatSize') 81 | price = item.get('price') 82 | soldNum = item.get('soldNum') 83 | points = gen_points(brand, serie, pid, title, size, formatSize, price, soldNum) 84 | try: 85 | result = influxdb.write_points(points) 86 | except Exception as e: 87 | log.fail(f'写入数据库失败({e.__class__.__name__}) 商品:{title} 编号：{pid} 尺码: {size} 价格: {price}') 88 | log.debug(traceback.format_exc()) 89 | else: 90 | if not result: 91 | log.fail(f'写入数据库失败 商品:{title} 编号：{pid} 尺码: {size} 价格: {price}') 92 | return item 93 | -------------------------------------------------------------------------------- /DuTracker/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for DuTracker project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # https://doc.scrapy.org/en/latest/topics/settings.html 9 | # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html 10 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'DuTracker' 13 | 14 | SPIDER_MODULES = ['DuTracker.spiders'] 15 | NEWSPIDER_MODULE = 'DuTracker.spiders' 16 | 17 | ROBOTSTXT_OBEY = False 18 | 19 | LOG_ENABLED = False 20 | 21 | DOWNLOADER_MIDDLEWARES = { 22 | 'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None, 23 | 'DuTracker.middlewares.RandomUserAgent': 500, 24 | 'DuTracker.middlewares.RetryException': 551, 25 | # 'DuTracker.middlewares.RandomProxy': 760 26 | } 27 | 28 | # PROXY_URL = 'http://127.0.0.1:8081' 29 | 30 | RANDOM_USER_AGENT = [ 31 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", 32 | "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25", 33 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", 34 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10", 35 | "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3", 36 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", 37 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", 38 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 39 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 40 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 41 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 42 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 43 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 44 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 45 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 46 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 47 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 48 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 49 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 50 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 51 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 52 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 53 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 54 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 55 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 56 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 57 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", 58 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 59 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 60 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 61 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 62 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 63 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 64 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 65 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 66 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 67 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 68 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 69 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 70 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 71 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 72 | "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", 73 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 74 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 75 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 76 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 77 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 78 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 79 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 80 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", 81 | "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", 82 | "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8G4 Safari/6533.18.5", 83 | "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_2_1 like Mac OS X; he-il) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5", 84 | "Mozilla/5.0 (iPhone; U; ru; CPU iPhone OS 4_2_1 like Mac OS X; ru) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5", 85 | "Mozilla/5.0 (iPhone; U; ru; CPU iPhone OS 4_2_1 like Mac OS X; fr) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5", 86 | "Mozilla/5.0 (iPhone; U; fr; CPU iPhone OS 4_2_1 like Mac OS X; fr) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5", 87 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_1 like Mac OS X; zh-tw) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8G4 Safari/6533.18.5", 88 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3 like Mac OS X; pl-pl) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5", 89 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3 like Mac OS X; fr-fr) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5", 90 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3 like Mac OS X; en-gb) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5", 91 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_2_1 like Mac OS X; ru-ru) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5", 92 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_2_1 like Mac OS X; nb-no) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5", 93 | "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8", 94 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; th-th) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8", 95 | "Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/531.2+", 96 | "Mozilla/5.0 (X11; U; Linux x86_64; en-ca) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/531.2+", 97 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 98 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; es-ES) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0 Safari/533.16", 99 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0 Safari/533.16", 100 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 101 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 102 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; fr) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 103 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; zh-cn) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 104 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; ru-ru) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 105 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; ko-kr) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 106 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; it-it) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 107 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; HTC-P715a; en-ca) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 108 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us) AppleWebKit/534.1+ (KHTML, like Gecko) Version/5.0 Safari/533.16", 109 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-au) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 110 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; el-gr) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 111 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; ca-es) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 112 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-tw) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 113 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; ja-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 114 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; it-it) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 115 | "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-en) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16", 116 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; nl-nl) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16", 117 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; ja-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16", 118 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; de-de) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16", 119 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7; en-us) AppleWebKit/533.4 (KHTML, like Gecko) Version/4.1 Safari/533.4", 120 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; nb-no) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16", 121 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/526.9 (KHTML, like Gecko) Version/4.0dp1 Safari/526.8", 122 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; tr) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2", 123 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2", 124 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; de) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2", 125 | "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.1b3pre) Gecko/20081212 Mozilla/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/526.9 (KHTML, like Gecko) Version/4.0dp1 Safari/526.8", 126 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-gb) AppleWebKit/528.10+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2", 127 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; en-us) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2", 128 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; en-gb) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2", 129 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; es-ES) AppleWebKit/531.22.7 (KHTML, like Gecko) Version/4.0.5 Safari/531.22.7", 130 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/4.0.5 Safari/531.22.7" 131 | ] 132 | 133 | 134 | INFLUXDB_HOST = '127.0.0.1' 135 | INFLUXDB_PORT = 8086 136 | INFLUXDB_USER = 'root' 137 | INFLUXDB_PASSWORD = 'root' 138 | FLUXDB_DATABASE = 'duApp' -------------------------------------------------------------------------------- /DuTracker/sign/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/10 15:42 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | -------------------------------------------------------------------------------- /DuTracker/sign/sign.js: -------------------------------------------------------------------------------- 1 | var t62 = { 2 | rotl: function (n, e) { 3 | return n << e | n >>> 32 - e 4 | }, 5 | rotr: function (n, e) { 6 | return n << 32 - e | n >>> e 7 | }, 8 | endian: function (n) { 9 | if (n.constructor == Number) 10 | return 16711935 & t62.rotl(n, 8) | 4278255360 & t62.rotl(n, 24); 11 | for (var e = 0; e < n.length; e++) 12 | n[e] = t62.endian(n[e]); 13 | return n 14 | }, 15 | randomBytes: function (n) { 16 | for (var e = []; n > 0; n--) 17 | e.push(Math.floor(256 * Math.random())); 18 | return e 19 | }, 20 | bytesToWords: function (n) { 21 | for (var e = [], t = 0, i = 0; t < n.length; t++, 22 | i += 8) 23 | e[i >>> 5] |= n[t] << 24 - i % 32; 24 | return e 25 | }, 26 | wordsToBytes: function (n) { 27 | for (var e = [], t = 0; t < 32 * n.length; t += 8) 28 | e.push(n[t >>> 5] >>> 24 - t % 32 & 255); 29 | return e 30 | }, 31 | bytesToHex: function (n) { 32 | for (var e = [], t = 0; t < n.length; t++) 33 | e.push((n[t] >>> 4).toString(16)), 34 | e.push((15 & n[t]).toString(16)); 35 | return e.join("") 36 | }, 37 | hexToBytes: function (n) { 38 | for (var e = [], t = 0; t < n.length; t += 2) 39 | e.push(parseInt(n.substr(t, 2), 16)); 40 | return e 41 | }, 42 | bytesToBase64: function (n) { 43 | for (var t = [], i = 0; i < n.length; i += 3) 44 | for (var o = n[i] << 16 | n[i + 1] << 8 | n[i + 2], r = 0; r < 4; r++) 45 | 8 * i + 6 * r <= 8 * n.length ? t.push(e.charAt(o >>> 6 * (3 - r) & 63)) : t.push("="); 46 | return t.join("") 47 | }, 48 | base64ToBytes: function (n) { 49 | n = n.replace(/[^A-Z0-9+\/]/gi, ""); 50 | for (var t = [], i = 0, o = 0; i < n.length; o = ++i % 4) 51 | 0 != o && t.push((e.indexOf(n.charAt(i - 1)) & Math.pow(2, -2 * o + 8) - 1) << 2 * o | e.indexOf(n.charAt(i)) >>> 6 - 2 * o); 52 | return t 53 | } 54 | }; 55 | var t31 = { 56 | utf8: { 57 | stringToBytes: function (n) { 58 | return t31.bin.stringToBytes(unescape(encodeURIComponent(n))) 59 | }, 60 | bytesToString: function (n) { 61 | return decodeURIComponent(escape(t.bin.bytesToString(n))) 62 | } 63 | }, 64 | bin: { 65 | stringToBytes: function (n) { 66 | for (var e = [], t = 0; t < n.length; t++) 67 | e.push(255 & n.charCodeAt(t)); 68 | return e 69 | }, 70 | bytesToString: function (n) { 71 | for (var e = [], t = 0; t < n.length; t++) 72 | e.push(String.fromCharCode(n[t])); 73 | return e.join("") 74 | } 75 | } 76 | }; 77 | var t57 = { 78 | t: function (n) { 79 | return !!n.constructor && "function" == typeof n.constructor.isBuffer && n.constructor.isBuffer(n) 80 | }, 81 | i: function (n) { 82 | return "function" == typeof n.readFloatLE && "function" == typeof n.slice && t(n.slice(0, 0)) 83 | } 84 | 85 | /*! 86 | * Determine if an object is a Buffer 87 | * 88 | * @author Feross Aboukhadijeh 89 | * @license MIT 90 | */ 91 | }; 92 | 93 | 94 | function getSign(n, e, t) { 95 | var e = t62 96 | , i = t31.utf8 97 | , o = t57 98 | , r = t31.bin 99 | , a = function (n, t) { 100 | n.constructor == String ? n = t && "binary" === t.encoding ? r.stringToBytes(n) : i.stringToBytes(n) : o(n) ? n = Array.prototype.slice.call(n, 0) : Array.isArray(n) || (n = n.toString()); 101 | for (var s = e.bytesToWords(n), l = 8 * n.length, c = 1732584193, d = -271733879, A = -1732584194, p = 271733878, u = 0; u < s.length; u++) 102 | s[u] = 16711935 & (s[u] << 8 | s[u] >>> 24) | 4278255360 & (s[u] << 24 | s[u] >>> 8); 103 | s[l >>> 5] |= 128 << l % 32, 104 | s[14 + (l + 64 >>> 9 << 4)] = l; 105 | for (var h = a._ff, f = a._gg, g = a._hh, m = a._ii, u = 0; u < s.length; u += 16) { 106 | var C = c 107 | , v = d 108 | , b = A 109 | , B = p; 110 | c = h(c, d, A, p, s[u + 0], 7, -680876936), 111 | p = h(p, c, d, A, s[u + 1], 12, -389564586), 112 | A = h(A, p, c, d, s[u + 2], 17, 606105819), 113 | d = h(d, A, p, c, s[u + 3], 22, -1044525330), 114 | c = h(c, d, A, p, s[u + 4], 7, -176418897), 115 | p = h(p, c, d, A, s[u + 5], 12, 1200080426), 116 | A = h(A, p, c, d, s[u + 6], 17, -1473231341), 117 | d = h(d, A, p, c, s[u + 7], 22, -45705983), 118 | c = h(c, d, A, p, s[u + 8], 7, 1770035416), 119 | p = h(p, c, d, A, s[u + 9], 12, -1958414417), 120 | A = h(A, p, c, d, s[u + 10], 17, -42063), 121 | d = h(d, A, p, c, s[u + 11], 22, -1990404162), 122 | c = h(c, d, A, p, s[u + 12], 7, 1804603682), 123 | p = h(p, c, d, A, s[u + 13], 12, -40341101), 124 | A = h(A, p, c, d, s[u + 14], 17, -1502002290), 125 | d = h(d, A, p, c, s[u + 15], 22, 1236535329), 126 | c = f(c, d, A, p, s[u + 1], 5, -165796510), 127 | p = f(p, c, d, A, s[u + 6], 9, -1069501632), 128 | A = f(A, p, c, d, s[u + 11], 14, 643717713), 129 | d = f(d, A, p, c, s[u + 0], 20, -373897302), 130 | c = f(c, d, A, p, s[u + 5], 5, -701558691), 131 | p = f(p, c, d, A, s[u + 10], 9, 38016083), 132 | A = f(A, p, c, d, s[u + 15], 14, -660478335), 133 | d = f(d, A, p, c, s[u + 4], 20, -405537848), 134 | c = f(c, d, A, p, s[u + 9], 5, 568446438), 135 | p = f(p, c, d, A, s[u + 14], 9, -1019803690), 136 | A = f(A, p, c, d, s[u + 3], 14, -187363961), 137 | d = f(d, A, p, c, s[u + 8], 20, 1163531501), 138 | c = f(c, d, A, p, s[u + 13], 5, -1444681467), 139 | p = f(p, c, d, A, s[u + 2], 9, -51403784), 140 | A = f(A, p, c, d, s[u + 7], 14, 1735328473), 141 | d = f(d, A, p, c, s[u + 12], 20, -1926607734), 142 | c = g(c, d, A, p, s[u + 5], 4, -378558), 143 | p = g(p, c, d, A, s[u + 8], 11, -2022574463), 144 | A = g(A, p, c, d, s[u + 11], 16, 1839030562), 145 | d = g(d, A, p, c, s[u + 14], 23, -35309556), 146 | c = g(c, d, A, p, s[u + 1], 4, -1530992060), 147 | p = g(p, c, d, A, s[u + 4], 11, 1272893353), 148 | A = g(A, p, c, d, s[u + 7], 16, -155497632), 149 | d = g(d, A, p, c, s[u + 10], 23, -1094730640), 150 | c = g(c, d, A, p, s[u + 13], 4, 681279174), 151 | p = g(p, c, d, A, s[u + 0], 11, -358537222), 152 | A = g(A, p, c, d, s[u + 3], 16, -722521979), 153 | d = g(d, A, p, c, s[u + 6], 23, 76029189), 154 | c = g(c, d, A, p, s[u + 9], 4, -640364487), 155 | p = g(p, c, d, A, s[u + 12], 11, -421815835), 156 | A = g(A, p, c, d, s[u + 15], 16, 530742520), 157 | d = g(d, A, p, c, s[u + 2], 23, -995338651), 158 | c = m(c, d, A, p, s[u + 0], 6, -198630844), 159 | p = m(p, c, d, A, s[u + 7], 10, 1126891415), 160 | A = m(A, p, c, d, s[u + 14], 15, -1416354905), 161 | d = m(d, A, p, c, s[u + 5], 21, -57434055), 162 | c = m(c, d, A, p, s[u + 12], 6, 1700485571), 163 | p = m(p, c, d, A, s[u + 3], 10, -1894986606), 164 | A = m(A, p, c, d, s[u + 10], 15, -1051523), 165 | d = m(d, A, p, c, s[u + 1], 21, -2054922799), 166 | c = m(c, d, A, p, s[u + 8], 6, 1873313359), 167 | p = m(p, c, d, A, s[u + 15], 10, -30611744), 168 | A = m(A, p, c, d, s[u + 6], 15, -1560198380), 169 | d = m(d, A, p, c, s[u + 13], 21, 1309151649), 170 | c = m(c, d, A, p, s[u + 4], 6, -145523070), 171 | p = m(p, c, d, A, s[u + 11], 10, -1120210379), 172 | A = m(A, p, c, d, s[u + 2], 15, 718787259), 173 | d = m(d, A, p, c, s[u + 9], 21, -343485551), 174 | c = c + C >>> 0, 175 | d = d + v >>> 0, 176 | A = A + b >>> 0, 177 | p = p + B >>> 0 178 | } 179 | return e.endian([c, d, A, p]) 180 | }; 181 | a._ff = function (n, e, t, i, o, r, a) { 182 | var s = n + (e & t | ~e & i) + (o >>> 0) + a; 183 | return (s << r | s >>> 32 - r) + e 184 | } 185 | , 186 | a._gg = function (n, e, t, i, o, r, a) { 187 | var s = n + (e & i | t & ~i) + (o >>> 0) + a; 188 | return (s << r | s >>> 32 - r) + e 189 | } 190 | , 191 | a._hh = function (n, e, t, i, o, r, a) { 192 | var s = n + (e ^ t ^ i) + (o >>> 0) + a; 193 | return (s << r | s >>> 32 - r) + e 194 | } 195 | , 196 | a._ii = function (n, e, t, i, o, r, a) { 197 | var s = n + (t ^ (e | ~i)) + (o >>> 0) + a; 198 | return (s << r | s >>> 32 - r) + e 199 | } 200 | , 201 | a._blocksize = 16, 202 | a._digestsize = 16; 203 | var i = e.wordsToBytes(a(n, t)); 204 | var result = e.bytesToHex(i); 205 | return result 206 | } 207 | 208 | -------------------------------------------------------------------------------- /DuTracker/sign/sign.py: -------------------------------------------------------------------------------- 1 | __all__ = ['sign'] 2 | 3 | # Don't look below, you will not understand this Python code :) I don't. 4 | 5 | from js2py.pyjs import * 6 | # setting scope 7 | var = Scope( JS_BUILTINS ) 8 | set_global_object(var) 9 | 10 | # Code follows: 11 | var.registers(['getSign', 't57', 't31', 't62']) 12 | @Js 13 | def PyJsHoisted_getSign_(n, e, t, this, arguments, var=var): 14 | var = Scope({'n':n, 'e':e, 't':t, 'this':this, 'arguments':arguments}, var) 15 | var.registers(['i', 'result', 'n', 'r', 'a', 't', 'e', 'o']) 16 | var.put('e', var.get('t62')) 17 | var.put('i', var.get('t31').get('utf8')) 18 | var.put('o', var.get('t57')) 19 | var.put('r', var.get('t31').get('bin')) 20 | @Js 21 | def PyJs_anonymous_21_(n, t, this, arguments, var=var): 22 | var = Scope({'n':n, 't':t, 'this':this, 'arguments':arguments}, var) 23 | var.registers(['d', 'C', 'b', 'p', 'm', 'B', 'h', 'g', 's', 'v', 'n', 't', 'l', 'f', 'c', 'A', 'u']) 24 | def PyJs_LONG_22_(var=var): 25 | return (var.put('n', (var.get('r').callprop('stringToBytes', var.get('n')) if (var.get('t') and PyJsStrictEq(Js('binary'),var.get('t').get('encoding'))) else var.get('i').callprop('stringToBytes', var.get('n')))) if (var.get('n').get('constructor')==var.get('String')) else (var.put('n', var.get('Array').get('prototype').get('slice').callprop('call', var.get('n'), Js(0.0))) if var.get('o')(var.get('n')) else (var.get('Array').callprop('isArray', var.get('n')) or var.put('n', var.get('n').callprop('toString'))))) 26 | PyJs_LONG_22_() 27 | #for JS loop 28 | var.put('s', var.get('e').callprop('bytesToWords', var.get('n'))) 29 | var.put('l', (Js(8.0)*var.get('n').get('length'))) 30 | var.put('c', Js(1732584193.0)) 31 | var.put('d', (-Js(271733879.0))) 32 | var.put('A', (-Js(1732584194.0))) 33 | var.put('p', Js(271733878.0)) 34 | var.put('u', Js(0.0)) 35 | while (var.get('u')Js(0.0)): 129 | try: 130 | var.get('e').callprop('push', var.get('Math').callprop('floor', (Js(256.0)*var.get('Math').callprop('random')))) 131 | finally: 132 | (var.put('n',Js(var.get('n').to_number())-Js(1))+Js(1)) 133 | return var.get('e') 134 | PyJs_anonymous_4_._set_name('anonymous') 135 | @Js 136 | def PyJs_anonymous_5_(n, this, arguments, var=var): 137 | var = Scope({'n':n, 'this':this, 'arguments':arguments}, var) 138 | var.registers(['n', 'i', 't', 'e']) 139 | #for JS loop 140 | var.put('e', Js([])) 141 | var.put('t', Js(0.0)) 142 | var.put('i', Js(0.0)) 143 | while (var.get('t') {0}".format(headers())) 64 | yield Request(url, headers=headers()) 65 | 66 | @handle_parse_exception 67 | def parse(self, response): 68 | # log.info('商品详情response') 69 | data = json.loads(response.body_as_unicode())['data'] 70 | imageAndText = data['imageAndText'] 71 | detail = data['detail'] 72 | productId = detail['productId'] 73 | categoryId = detail['categoryId'] 74 | logoUrl = detail['logoUrl'] 75 | images = [image['url'] for image in detail['images']] 76 | title = detail['title'] 77 | soldNum = detail['soldNum'] 78 | sellDate = detail['sellDate'] 79 | articleNumber = detail['articleNumber'] 80 | authPrice = detail['authPrice'] 81 | goodsId = detail['goodsId'] 82 | sizeList = detail['sizeList'] 83 | 84 | # log.info(f'商品详情 response url：{response.url}') 85 | 86 | yield ProductItem( 87 | id=productId, 88 | url=response.url, 89 | title=title, 90 | soldNum=soldNum, 91 | logo=logoUrl, 92 | categoryId=categoryId, 93 | images=images, 94 | sellDate=sellDate, 95 | articleNumber=articleNumber, 96 | authPrice=authPrice, 97 | goodsId=goodsId, 98 | sizeList=sizeList, 99 | imageAndText=imageAndText, 100 | detailJson=detail 101 | ) 102 | -------------------------------------------------------------------------------- /DuTracker/spiders/serie.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import scrapy 3 | from scrapy import Request 4 | from scrapy.exceptions import IgnoreRequest 5 | 6 | import json 7 | from click import prompt 8 | import math 9 | 10 | from DuTracker.utils.log import log, handle_parse_exception 11 | from DuTracker.items import ProductInfo 12 | from DuTracker.utils.urls import get_serie_page_url as page_url 13 | from DuTracker.utils.urls import get_headers as headers 14 | 15 | 16 | class SerieSpider(scrapy.Spider): 17 | name = 'serie' 18 | allowed_domains = ['app.poizon.com'] 19 | start_urls = [ 20 | 'https://app.poizon.com/api/v1/h5/product/fire/search/getCategoryDetail?catId=1&sign=0efcd8daeaac723e588568e45424a7c3' 21 | ] 22 | custom_settings = { 23 | 'ITEM_PIPELINES': { 24 | 'DuTracker.pipelines.SaveProductId': 300, 25 | } 26 | } 27 | 28 | serieIds = {} 29 | Ids = [] 30 | auto = False 31 | 32 | def start_requests(self): 33 | log.info('获取系列列表') 34 | for url in self.start_urls: 35 | yield Request(url, dont_filter=True, callback=self.parse_serieList, headers=headers()) 36 | 37 | @handle_parse_exception 38 | def parse_serieList(self, response): 39 | serieList = json.loads(response.body_as_unicode())['data']['list'] 40 | # log.info(f'{serieList}') 41 | for data in serieList: 42 | for serie in data['seriesList']: 43 | unionId = serie['productSeriesId'] 44 | name = serie['name'] 45 | self.serieIds[unionId] = name 46 | log.success(f'系列：{name} 编号：{unionId}') 47 | if not self.auto: 48 | ids = prompt('输入需要爬取的系列编号', default='').strip().split(' ') 49 | if ids == ['']: return IgnoreRequest() 50 | else: 51 | ids = self.Ids 52 | if not ids: return IgnoreRequest() 53 | 54 | log.info(f'获取 {ids} 系列包含商品') 55 | for unionId in ids: 56 | unionId = int(unionId) 57 | log.info(f'unionId: {unionId}') 58 | yield Request(page_url(unionId), callback=self.parse_serieInfo, meta={ 59 | 'unionId': unionId, 60 | 'name': self.serieIds[unionId] 61 | }, headers=headers()) 62 | 63 | @handle_parse_exception 64 | def parse_serieInfo(self, response): 65 | log.info(f'系列列表响应') 66 | data = json.loads(response.body_as_unicode())['data'] 67 | unionId = response.meta.get('unionId') 68 | name = response.meta.get('name') 69 | 70 | num = data['total'] 71 | page = math.ceil(num / 20) 72 | log.success(f'系列：{name} 编号：{unionId} 商品总数：{num} 页面数：{page}') 73 | 74 | for page in range(1, page + 1): 75 | yield Request(page_url(unionId, page), callback=self.parse_productId, meta={ 76 | 'unionId': unionId, 77 | 'name': self.serieIds[unionId] 78 | }, headers=headers()) 79 | 80 | @handle_parse_exception 81 | def parse_productId(self, response): 82 | productList = json.loads(response.body_as_unicode())['data']['productList'] 83 | for product in productList: 84 | name = response.meta.get('name') 85 | pid = product['productId'] 86 | title = product['title'] 87 | yield ProductInfo( 88 | id=pid, 89 | title=title, 90 | name=name, 91 | ) 92 | -------------------------------------------------------------------------------- /DuTracker/spiders/sign.js: -------------------------------------------------------------------------------- 1 | var t62 = { 2 | rotl: function (n, e) { 3 | return n << e | n >>> 32 - e 4 | }, 5 | rotr: function (n, e) { 6 | return n << 32 - e | n >>> e 7 | }, 8 | endian: function (n) { 9 | if (n.constructor == Number) 10 | return 16711935 & t62.rotl(n, 8) | 4278255360 & t62.rotl(n, 24); 11 | for (var e = 0; e < n.length; e++) 12 | n[e] = t62.endian(n[e]); 13 | return n 14 | }, 15 | randomBytes: function (n) { 16 | for (var e = []; n > 0; n--) 17 | e.push(Math.floor(256 * Math.random())); 18 | return e 19 | }, 20 | bytesToWords: function (n) { 21 | for (var e = [], t = 0, i = 0; t < n.length; t++, 22 | i += 8) 23 | e[i >>> 5] |= n[t] << 24 - i % 32; 24 | return e 25 | }, 26 | wordsToBytes: function (n) { 27 | for (var e = [], t = 0; t < 32 * n.length; t += 8) 28 | e.push(n[t >>> 5] >>> 24 - t % 32 & 255); 29 | return e 30 | }, 31 | bytesToHex: function (n) { 32 | for (var e = [], t = 0; t < n.length; t++) 33 | e.push((n[t] >>> 4).toString(16)), 34 | e.push((15 & n[t]).toString(16)); 35 | return e.join("") 36 | }, 37 | hexToBytes: function (n) { 38 | for (var e = [], t = 0; t < n.length; t += 2) 39 | e.push(parseInt(n.substr(t, 2), 16)); 40 | return e 41 | }, 42 | bytesToBase64: function (n) { 43 | for (var t = [], i = 0; i < n.length; i += 3) 44 | for (var o = n[i] << 16 | n[i + 1] << 8 | n[i + 2], r = 0; r < 4; r++) 45 | 8 * i + 6 * r <= 8 * n.length ? t.push(e.charAt(o >>> 6 * (3 - r) & 63)) : t.push("="); 46 | return t.join("") 47 | }, 48 | base64ToBytes: function (n) { 49 | n = n.replace(/[^A-Z0-9+\/]/gi, ""); 50 | for (var t = [], i = 0, o = 0; i < n.length; o = ++i % 4) 51 | 0 != o && t.push((e.indexOf(n.charAt(i - 1)) & Math.pow(2, -2 * o + 8) - 1) << 2 * o | e.indexOf(n.charAt(i)) >>> 6 - 2 * o); 52 | return t 53 | } 54 | }; 55 | var t31 = { 56 | utf8: { 57 | stringToBytes: function (n) { 58 | return t31.bin.stringToBytes(unescape(encodeURIComponent(n))) 59 | }, 60 | bytesToString: function (n) { 61 | return decodeURIComponent(escape(t.bin.bytesToString(n))) 62 | } 63 | }, 64 | bin: { 65 | stringToBytes: function (n) { 66 | for (var e = [], t = 0; t < n.length; t++) 67 | e.push(255 & n.charCodeAt(t)); 68 | return e 69 | }, 70 | bytesToString: function (n) { 71 | for (var e = [], t = 0; t < n.length; t++) 72 | e.push(String.fromCharCode(n[t])); 73 | return e.join("") 74 | } 75 | } 76 | }; 77 | var t57 = { 78 | t: function (n) { 79 | return !!n.constructor && "function" == typeof n.constructor.isBuffer && n.constructor.isBuffer(n) 80 | }, 81 | i: function (n) { 82 | return "function" == typeof n.readFloatLE && "function" == typeof n.slice && t(n.slice(0, 0)) 83 | } 84 | 85 | /*! 86 | * Determine if an object is a Buffer 87 | * 88 | * @author Feross Aboukhadijeh 89 | * @license MIT 90 | */ 91 | }; 92 | 93 | 94 | function getSign(n, e, t) { 95 | var e = t62 96 | , i = t31.utf8 97 | , o = t57 98 | , r = t31.bin 99 | , a = function (n, t) { 100 | n.constructor == String ? n = t && "binary" === t.encoding ? r.stringToBytes(n) : i.stringToBytes(n) : o(n) ? n = Array.prototype.slice.call(n, 0) : Array.isArray(n) || (n = n.toString()); 101 | for (var s = e.bytesToWords(n), l = 8 * n.length, c = 1732584193, d = -271733879, A = -1732584194, p = 271733878, u = 0; u < s.length; u++) 102 | s[u] = 16711935 & (s[u] << 8 | s[u] >>> 24) | 4278255360 & (s[u] << 24 | s[u] >>> 8); 103 | s[l >>> 5] |= 128 << l % 32, 104 | s[14 + (l + 64 >>> 9 << 4)] = l; 105 | for (var h = a._ff, f = a._gg, g = a._hh, m = a._ii, u = 0; u < s.length; u += 16) { 106 | var C = c 107 | , v = d 108 | , b = A 109 | , B = p; 110 | c = h(c, d, A, p, s[u + 0], 7, -680876936), 111 | p = h(p, c, d, A, s[u + 1], 12, -389564586), 112 | A = h(A, p, c, d, s[u + 2], 17, 606105819), 113 | d = h(d, A, p, c, s[u + 3], 22, -1044525330), 114 | c = h(c, d, A, p, s[u + 4], 7, -176418897), 115 | p = h(p, c, d, A, s[u + 5], 12, 1200080426), 116 | A = h(A, p, c, d, s[u + 6], 17, -1473231341), 117 | d = h(d, A, p, c, s[u + 7], 22, -45705983), 118 | c = h(c, d, A, p, s[u + 8], 7, 1770035416), 119 | p = h(p, c, d, A, s[u + 9], 12, -1958414417), 120 | A = h(A, p, c, d, s[u + 10], 17, -42063), 121 | d = h(d, A, p, c, s[u + 11], 22, -1990404162), 122 | c = h(c, d, A, p, s[u + 12], 7, 1804603682), 123 | p = h(p, c, d, A, s[u + 13], 12, -40341101), 124 | A = h(A, p, c, d, s[u + 14], 17, -1502002290), 125 | d = h(d, A, p, c, s[u + 15], 22, 1236535329), 126 | c = f(c, d, A, p, s[u + 1], 5, -165796510), 127 | p = f(p, c, d, A, s[u + 6], 9, -1069501632), 128 | A = f(A, p, c, d, s[u + 11], 14, 643717713), 129 | d = f(d, A, p, c, s[u + 0], 20, -373897302), 130 | c = f(c, d, A, p, s[u + 5], 5, -701558691), 131 | p = f(p, c, d, A, s[u + 10], 9, 38016083), 132 | A = f(A, p, c, d, s[u + 15], 14, -660478335), 133 | d = f(d, A, p, c, s[u + 4], 20, -405537848), 134 | c = f(c, d, A, p, s[u + 9], 5, 568446438), 135 | p = f(p, c, d, A, s[u + 14], 9, -1019803690), 136 | A = f(A, p, c, d, s[u + 3], 14, -187363961), 137 | d = f(d, A, p, c, s[u + 8], 20, 1163531501), 138 | c = f(c, d, A, p, s[u + 13], 5, -1444681467), 139 | p = f(p, c, d, A, s[u + 2], 9, -51403784), 140 | A = f(A, p, c, d, s[u + 7], 14, 1735328473), 141 | d = f(d, A, p, c, s[u + 12], 20, -1926607734), 142 | c = g(c, d, A, p, s[u + 5], 4, -378558), 143 | p = g(p, c, d, A, s[u + 8], 11, -2022574463), 144 | A = g(A, p, c, d, s[u + 11], 16, 1839030562), 145 | d = g(d, A, p, c, s[u + 14], 23, -35309556), 146 | c = g(c, d, A, p, s[u + 1], 4, -1530992060), 147 | p = g(p, c, d, A, s[u + 4], 11, 1272893353), 148 | A = g(A, p, c, d, s[u + 7], 16, -155497632), 149 | d = g(d, A, p, c, s[u + 10], 23, -1094730640), 150 | c = g(c, d, A, p, s[u + 13], 4, 681279174), 151 | p = g(p, c, d, A, s[u + 0], 11, -358537222), 152 | A = g(A, p, c, d, s[u + 3], 16, -722521979), 153 | d = g(d, A, p, c, s[u + 6], 23, 76029189), 154 | c = g(c, d, A, p, s[u + 9], 4, -640364487), 155 | p = g(p, c, d, A, s[u + 12], 11, -421815835), 156 | A = g(A, p, c, d, s[u + 15], 16, 530742520), 157 | d = g(d, A, p, c, s[u + 2], 23, -995338651), 158 | c = m(c, d, A, p, s[u + 0], 6, -198630844), 159 | p = m(p, c, d, A, s[u + 7], 10, 1126891415), 160 | A = m(A, p, c, d, s[u + 14], 15, -1416354905), 161 | d = m(d, A, p, c, s[u + 5], 21, -57434055), 162 | c = m(c, d, A, p, s[u + 12], 6, 1700485571), 163 | p = m(p, c, d, A, s[u + 3], 10, -1894986606), 164 | A = m(A, p, c, d, s[u + 10], 15, -1051523), 165 | d = m(d, A, p, c, s[u + 1], 21, -2054922799), 166 | c = m(c, d, A, p, s[u + 8], 6, 1873313359), 167 | p = m(p, c, d, A, s[u + 15], 10, -30611744), 168 | A = m(A, p, c, d, s[u + 6], 15, -1560198380), 169 | d = m(d, A, p, c, s[u + 13], 21, 1309151649), 170 | c = m(c, d, A, p, s[u + 4], 6, -145523070), 171 | p = m(p, c, d, A, s[u + 11], 10, -1120210379), 172 | A = m(A, p, c, d, s[u + 2], 15, 718787259), 173 | d = m(d, A, p, c, s[u + 9], 21, -343485551), 174 | c = c + C >>> 0, 175 | d = d + v >>> 0, 176 | A = A + b >>> 0, 177 | p = p + B >>> 0 178 | } 179 | return e.endian([c, d, A, p]) 180 | }; 181 | a._ff = function (n, e, t, i, o, r, a) { 182 | var s = n + (e & t | ~e & i) + (o >>> 0) + a; 183 | return (s << r | s >>> 32 - r) + e 184 | } 185 | , 186 | a._gg = function (n, e, t, i, o, r, a) { 187 | var s = n + (e & i | t & ~i) + (o >>> 0) + a; 188 | return (s << r | s >>> 32 - r) + e 189 | } 190 | , 191 | a._hh = function (n, e, t, i, o, r, a) { 192 | var s = n + (e ^ t ^ i) + (o >>> 0) + a; 193 | return (s << r | s >>> 32 - r) + e 194 | } 195 | , 196 | a._ii = function (n, e, t, i, o, r, a) { 197 | var s = n + (t ^ (e | ~i)) + (o >>> 0) + a; 198 | return (s << r | s >>> 32 - r) + e 199 | } 200 | , 201 | a._blocksize = 16, 202 | a._digestsize = 16; 203 | var i = e.wordsToBytes(a(n, t)); 204 | var result = e.bytesToHex(i); 205 | return result 206 | } 207 | 208 | -------------------------------------------------------------------------------- /DuTracker/spiders/tracker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import scrapy 3 | import json 4 | import arrow 5 | from DuTracker.utils.urls import get_headers as headers 6 | 7 | 8 | from DuTracker.items import PriceItem 9 | from DuTracker.db import * 10 | from DuTracker.utils.log import log, handle_parse_exception 11 | 12 | 13 | class TrackerSpider(scrapy.Spider): 14 | name = 'tracker' 15 | allowed_domains = ['m.poizon.com'] 16 | 17 | custom_settings = { 18 | 'ITEM_PIPELINES': { 19 | 'DuTracker.pipelines.SavePriceItem': 300, 20 | } 21 | } 22 | 23 | soldNum_min = 50 24 | Ids = [] 25 | newItem = False # 适用于商品发售初期 26 | days = 14 27 | 28 | @db_session 29 | def get_items(self): 30 | pools = [] 31 | if not self.newItem: 32 | for p in Product.select(lambda p: p.soldNum > self.soldNum_min).order_by(desc(Product.soldNum)): 33 | pools.append(p) 34 | for pid in self.Ids: 35 | if Product.exists(id=pid): 36 | pools.append(Product[pid]) 37 | else: 38 | log.fail(f'商品编号:{pid} 不存在数据库') 39 | else: 40 | for p in Product.select(): 41 | delay = arrow.now() - arrow.get(p.datetime, 'YYYY-MM-DD HH:mm:ss') 42 | if delay.days < self.days: pools.append(p) 43 | 44 | return pools 45 | 46 | def start_requests(self): 47 | log.info(f'选取商品销量高于 {self.soldNum_min} 开始追踪') 48 | pools = self.get_items() 49 | 50 | for p in pools: 51 | # log.info(f'production url: {p.url}') 52 | yield scrapy.Request(p.url, meta={ 53 | 'productId': p.id, 54 | 'title': p.title, 55 | 'brand': p.brand, 56 | 'serie': p.serie, 57 | 'articleNumber': p.articleNumber 58 | }, headers=headers()) 59 | 60 | @db_session 61 | @handle_parse_exception 62 | def parse(self, response): 63 | title = response.meta.get('title') 64 | pid = response.meta.get('productId') 65 | brand = response.meta.get('brand') 66 | serie = response.meta.get('serie') 67 | 68 | data = json.loads(response.body_as_unicode())['data'] 69 | # log.info(f'data {data}') 70 | soldNum = data['detail']['soldNum'] 71 | Product[pid].soldNum = soldNum 72 | commit() 73 | 74 | sizeList = data['sizeList'] 75 | sizeItem = data['item'] 76 | price = sizeItem['price'] / 100 77 | # formatSize = sizeItem['formatSize'] 78 | # log.success(f'商品:{title} 编号：{pid} 价格: {price}/{formatSize} 交易数量: {soldNum}') 79 | log.success(f'商品:{title} 编号：{pid} 价格: {price} 交易数量: {soldNum}') 80 | 81 | for s in sizeList: 82 | item = s['item'] 83 | if not item: 84 | continue 85 | yield PriceItem( 86 | id=pid, 87 | brand=brand, 88 | serie=serie, 89 | title=title, 90 | size=s['size'], 91 | formatSize=s['formatSize'], 92 | price=item['price'] / 100, 93 | soldNum=soldNum, 94 | ) 95 | 96 | # if __name__ == '__main__': 97 | # with db_session: 98 | # for p in Product.select(): 99 | # delay = arrow.now() - arrow.get(p.datetime, 'YYYY-MM-DD HH:mm:ss') 100 | # if delay.days < 60: 101 | # print(p.title, p.datetime) 102 | -------------------------------------------------------------------------------- /DuTracker/tsdb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/14 00:02 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | 7 | import sys 8 | from scrapy.utils.project import get_project_settings 9 | from influxdb import InfluxDBClient 10 | 11 | from DuTracker.utils.log import log 12 | 13 | _settings = get_project_settings() 14 | 15 | _host = _settings.get('INFLUXDB_HOST', 'localhost') 16 | _port = _settings.get('INFLUXDB_PORT', 8086) 17 | _username = _settings.get('INFLUXDB_USER', 'root') 18 | _password = _settings.get('INFLUXDB_PASSWORD', 'root') 19 | _database = _settings.get('FLUXDB_DATABASE') 20 | 21 | influxdb = InfluxDBClient(_host, _port, _username, _password, _database) 22 | 23 | 24 | 25 | def gen_points(brand, serie, productId, title, size, formatSize, price, soldNum): 26 | return [{ 27 | 'measurement': "pHistory", 28 | 'tags': { 29 | 'productId': productId, 30 | 'title': title, 31 | 'brand': brand, 32 | 'serie': serie, 33 | 'size': size, 34 | 'formatSize': formatSize, 35 | }, 36 | 'fields': { 37 | 'price': price, 38 | 'soldNum': soldNum 39 | } 40 | }] 41 | -------------------------------------------------------------------------------- /DuTracker/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/10 12:04 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | -------------------------------------------------------------------------------- /DuTracker/utils/log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/10 13:33 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | 7 | import sys 8 | from click import style 9 | import logging 10 | from logging.handlers import RotatingFileHandler 11 | import traceback 12 | 13 | class ColorfulText(object): 14 | """Colorful text""" 15 | 16 | @staticmethod 17 | def blue(msg): 18 | return style(msg, fg='blue') 19 | 20 | @staticmethod 21 | def green(msg): 22 | return style(msg, fg='green') 23 | 24 | @staticmethod 25 | def red(msg): 26 | return style(msg, fg='red') 27 | 28 | @staticmethod 29 | def yellow(msg): 30 | return style(msg, fg='yellow') 31 | 32 | @staticmethod 33 | def cyan(msg): 34 | return style(msg, fg='cyan') 35 | 36 | 37 | class Logger(object): 38 | def __init__(self, logger: logging.Logger): 39 | self._logger = logger 40 | 41 | def info(self, msg): 42 | self._logger.info(ColorfulText.blue('[*] ') + msg) 43 | 44 | def success(self, msg): 45 | self._logger.info(ColorfulText.green('[✔] ') + msg) 46 | 47 | def fail(self, msg): 48 | self._logger.info(ColorfulText.red('[✘] ') + msg) 49 | 50 | def warn(self, msg): 51 | self._logger.warning(ColorfulText.yellow('[!] ') + msg) 52 | 53 | def error(self, msg): 54 | self._logger.error(ColorfulText.red('[?] ') + msg) 55 | 56 | def debug(self, msg): 57 | self._logger.debug(ColorfulText.cyan('[#] ') + msg) 58 | 59 | def setLevel(self, level): 60 | self._logger.setLevel(level) 61 | 62 | @property 63 | def level(self): 64 | return self._logger.level 65 | 66 | 67 | formatter = logging.Formatter('%(asctime)s: %(message)s') 68 | file_handler = RotatingFileHandler('spider.log',maxBytes=5*1024*1024,backupCount=3) 69 | file_handler.setFormatter(formatter) 70 | console_handler = logging.StreamHandler(sys.stdout) 71 | console_handler.setFormatter(formatter) 72 | 73 | _logger = logging.getLogger('spider') 74 | _logger.setLevel(logging.INFO) 75 | _logger.addHandler(file_handler) 76 | _logger.addHandler(console_handler) 77 | 78 | log = Logger(_logger) 79 | 80 | 81 | def handle_parse_exception(func): 82 | def wrapper(spider, response): 83 | try: 84 | for result in func(spider, response): 85 | yield result 86 | except Exception as e: 87 | log.fail( 88 | f'[Spider:{spider.name}->Function:{func.__name__}] {response.url} {e.__class__.__name__}:{e}') 89 | log.debug(f'{traceback.format_exc()}') 90 | yield next(func(spider,response)) 91 | 92 | return wrapper 93 | -------------------------------------------------------------------------------- /DuTracker/utils/urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/24 22:02 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | import execjs 7 | import requests 8 | 9 | 10 | def get_headers(): 11 | headers = { 12 | 'Host': "app.poizon.com", 13 | 'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)" 14 | " Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.4.501 NetType/WIFI " 15 | "MiniProgramEnv/Windows WindowsWechat", 16 | 'appid': "wxapp", 17 | 'appversion': "4.4.0", 18 | 'content-type': "application/x-www-form-urlencoded", 19 | 'Accept-Encoding': "gzip, deflate", 20 | 'Accept': "*/*", 21 | } 22 | return headers 23 | 24 | 25 | def get_brand_page_url(unionId, page=0): 26 | sortType = 0 27 | sortMode = 1 28 | with open('DuTracker/sign/sign.js', 'r', encoding='utf-8')as f: 29 | all_ = f.read() 30 | ctx = execjs.compile(all_) 31 | # 53489 32 | sign = ctx.call('getSign', 33 | 'catId{}limit20page{}sortMode{}sortType{}titleunionId{}19bc545a393a25177083d4a748807cc0' 34 | .format(0, page, sortMode, sortType, unionId)) 35 | 36 | return 'https://app.poizon.com/api/v1/h5/search/fire/search/list?title=&page={}&sortType={}&sortMode={}&limit=20&catId=0&unionId={}&sign={}' \ 37 | .format(page, sortType, sortMode, unionId, sign) 38 | 39 | 40 | def get_serie_page_url(unionId, page=0): 41 | sortType = 0 42 | sortMode = 1 43 | with open('DuTracker/sign/sign.js', 'r', encoding='utf-8')as f: 44 | all_ = f.read() 45 | ctx = execjs.compile(all_) 46 | 47 | sign = ctx.call('getSign', 48 | 'catId{}limit20page{}sortMode{}sortType{}titleunionId{}19bc545a393a25177083d4a748807cc0' 49 | .format(1, page, sortMode, sortType, unionId)) 50 | #https://app.poizon.com/api/v1/h5/search/fire/search/list?title=&page=0&sortType=0&sortMode=1&limit=20&catId=1&unionId=1&sign=63531b2336752bc5e22fa71854f4f448 51 | return 'https://app.poizon.com/api/v1/h5/search/fire/search/list?title=&page={}&sortType={}&sortMode={}&limit=20&catId=1&unionId={}&sign={}' \ 52 | .format(page, sortType, sortMode, unionId, sign) 53 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.tuna.tsinghua.edu.cn/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | scrapy = "*" 10 | pony = "*" 11 | arrow = "*" 12 | click = "*" 13 | "js2py" = "*" 14 | apscheduler = "*" 15 | influxdb = "*" 16 | pyexecjs = "*" 17 | 18 | [requires] 19 | python_version = "3.6" 20 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "faf1c14c3bebbda133f9da969bd24144d74a364fa593c482e3e9bf3e304c8bf7" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.tuna.tsinghua.edu.cn/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "apscheduler": { 20 | "hashes": [ 21 | "sha256:3bb5229eed6fbbdafc13ce962712ae66e175aa214c69bed35a06bffcf0c5e244", 22 | "sha256:e8b1ecdb4c7cb2818913f766d5898183c7cb8936680710a4d3a966e02262e526" 23 | ], 24 | "index": "pypi", 25 | "version": "==3.6.3" 26 | }, 27 | "arrow": { 28 | "hashes": [ 29 | "sha256:01a16d8a93eddf86a29237f32ae36b29c27f047e79312eb4df5d55fd5a2b3183", 30 | "sha256:e1a318a4c0b787833ae46302c02488b6eeef413c6a13324b3261ad320f21ec1e" 31 | ], 32 | "index": "pypi", 33 | "version": "==0.15.4" 34 | }, 35 | "attrs": { 36 | "hashes": [ 37 | "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", 38 | "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" 39 | ], 40 | "version": "==19.3.0" 41 | }, 42 | "automat": { 43 | "hashes": [ 44 | "sha256:269a09dfb063a3b078983f4976d83f0a0d3e6e7aaf8e27d8df1095e09dc4a484", 45 | "sha256:81c93c55d2742c55e74e6497a48e048a859fa01d7aa0b91a032be432229837e2" 46 | ], 47 | "version": "==0.8.0" 48 | }, 49 | "certifi": { 50 | "hashes": [ 51 | "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", 52 | "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" 53 | ], 54 | "version": "==2019.11.28" 55 | }, 56 | "cffi": { 57 | "hashes": [ 58 | "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", 59 | "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", 60 | "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", 61 | "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", 62 | "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", 63 | "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", 64 | "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", 65 | "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", 66 | "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", 67 | "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", 68 | "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", 69 | "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", 70 | "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", 71 | "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", 72 | "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", 73 | "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", 74 | "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", 75 | "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", 76 | "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", 77 | "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", 78 | "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", 79 | "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", 80 | "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", 81 | "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", 82 | "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", 83 | "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", 84 | "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", 85 | "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", 86 | "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", 87 | "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", 88 | "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", 89 | "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", 90 | "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" 91 | ], 92 | "version": "==1.13.2" 93 | }, 94 | "chardet": { 95 | "hashes": [ 96 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 97 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 98 | ], 99 | "version": "==3.0.4" 100 | }, 101 | "click": { 102 | "hashes": [ 103 | "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", 104 | "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" 105 | ], 106 | "index": "pypi", 107 | "version": "==7.0" 108 | }, 109 | "constantly": { 110 | "hashes": [ 111 | "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", 112 | "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" 113 | ], 114 | "version": "==15.1.0" 115 | }, 116 | "cryptography": { 117 | "hashes": [ 118 | "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", 119 | "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", 120 | "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", 121 | "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", 122 | "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", 123 | "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", 124 | "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", 125 | "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", 126 | "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", 127 | "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", 128 | "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", 129 | "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", 130 | "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", 131 | "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", 132 | "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", 133 | "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", 134 | "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", 135 | "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", 136 | "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", 137 | "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", 138 | "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" 139 | ], 140 | "version": "==2.8" 141 | }, 142 | "cssselect": { 143 | "hashes": [ 144 | "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf", 145 | "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc" 146 | ], 147 | "version": "==1.1.0" 148 | }, 149 | "hyperlink": { 150 | "hashes": [ 151 | "sha256:4288e34705da077fada1111a24a0aa08bb1e76699c9ce49876af722441845654", 152 | "sha256:ab4a308feb039b04f855a020a6eda3b18ca5a68e6d8f8c899cbe9e653721d04f" 153 | ], 154 | "version": "==19.0.0" 155 | }, 156 | "idna": { 157 | "hashes": [ 158 | "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", 159 | "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" 160 | ], 161 | "version": "==2.8" 162 | }, 163 | "incremental": { 164 | "hashes": [ 165 | "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", 166 | "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" 167 | ], 168 | "version": "==17.5.0" 169 | }, 170 | "influxdb": { 171 | "hashes": [ 172 | "sha256:270ec1ec9cf1927a38cf5ec808e76f364482977577eb8c335f6aed5fcdc4cb25", 173 | "sha256:30276c7e04bf7659424c733b239ba2f0804d7a1f3c59ec5dd3f88c56176c8d36" 174 | ], 175 | "index": "pypi", 176 | "version": "==5.2.3" 177 | }, 178 | "js2py": { 179 | "hashes": [ 180 | "sha256:6e5628abfff2fb4051e8e77a353e44831f474e2ceb865278271897f7f326aeb6", 181 | "sha256:bf87cb4432944470f11fed9c1cb8d0312dd505e7b867362f55102f24379ab94f" 182 | ], 183 | "index": "pypi", 184 | "version": "==0.66" 185 | }, 186 | "lxml": { 187 | "hashes": [ 188 | "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", 189 | "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", 190 | "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", 191 | "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", 192 | "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", 193 | "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", 194 | "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", 195 | "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", 196 | "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", 197 | "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", 198 | "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", 199 | "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", 200 | "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", 201 | "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", 202 | "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", 203 | "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", 204 | "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", 205 | "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", 206 | "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", 207 | "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", 208 | "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", 209 | "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", 210 | "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", 211 | "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", 212 | "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", 213 | "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" 214 | ], 215 | "markers": "python_version != '3.4'", 216 | "version": "==4.4.2" 217 | }, 218 | "parsel": { 219 | "hashes": [ 220 | "sha256:4da4262ba4605573b6b72a5f557616a2fc9dee7a47a1efad562752a28d366723", 221 | "sha256:74f8e9d3b345b14cb1416bd777a03982cde33a74d8b32e0c71e651d07d41d40a" 222 | ], 223 | "version": "==1.5.2" 224 | }, 225 | "pony": { 226 | "hashes": [ 227 | "sha256:d1008d465d3c5db435fe2636f4c9e1a4a04a24f27c5d7cb4943ba5c596de7e17" 228 | ], 229 | "index": "pypi", 230 | "version": "==0.7.11" 231 | }, 232 | "protego": { 233 | "hashes": [ 234 | "sha256:a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4" 235 | ], 236 | "version": "==0.1.16" 237 | }, 238 | "pyasn1": { 239 | "hashes": [ 240 | "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", 241 | "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" 242 | ], 243 | "version": "==0.4.8" 244 | }, 245 | "pyasn1-modules": { 246 | "hashes": [ 247 | "sha256:0c35a52e00b672f832e5846826f1fb7507907f7d52fba6faa9e3c4cbe874fe4b", 248 | "sha256:b6ada4f840fe51abf5a6bd545b45bf537bea62221fa0dde2e8a553ed9f06a4e3" 249 | ], 250 | "version": "==0.2.7" 251 | }, 252 | "pycparser": { 253 | "hashes": [ 254 | "sha256:5291af04c7391ad92ef6b464b4c3f01ccfe20e0fe3da1f1db9d714c252c990eb", 255 | "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" 256 | ], 257 | "version": "==2.19" 258 | }, 259 | "pydispatcher": { 260 | "hashes": [ 261 | "sha256:5570069e1b1769af1fe481de6dd1d3a388492acddd2cdad7a3bde145615d5caf", 262 | "sha256:5be4a8be12805ef7d712dd9a93284fb8bc53f309867e573f653a72e5fd10e433" 263 | ], 264 | "version": "==2.0.5" 265 | }, 266 | "pyexecjs": { 267 | "hashes": [ 268 | "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c" 269 | ], 270 | "index": "pypi", 271 | "version": "==1.5.1" 272 | }, 273 | "pyhamcrest": { 274 | "hashes": [ 275 | "sha256:6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420", 276 | "sha256:8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd" 277 | ], 278 | "version": "==1.9.0" 279 | }, 280 | "pyjsparser": { 281 | "hashes": [ 282 | "sha256:2b12842df98d83f65934e0772fa4a5d8b123b3bc79f1af1789172ac70265dd21", 283 | "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579" 284 | ], 285 | "version": "==2.7.1" 286 | }, 287 | "pyopenssl": { 288 | "hashes": [ 289 | "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504", 290 | "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507" 291 | ], 292 | "version": "==19.1.0" 293 | }, 294 | "python-dateutil": { 295 | "hashes": [ 296 | "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", 297 | "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" 298 | ], 299 | "version": "==2.8.1" 300 | }, 301 | "pytz": { 302 | "hashes": [ 303 | "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", 304 | "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" 305 | ], 306 | "version": "==2019.3" 307 | }, 308 | "queuelib": { 309 | "hashes": [ 310 | "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17", 311 | "sha256:ff43b5b74b9266f8df4232a8f768dc4d67281a271905e2ed4a3689d4d304cd02" 312 | ], 313 | "version": "==1.5.0" 314 | }, 315 | "requests": { 316 | "hashes": [ 317 | "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", 318 | "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" 319 | ], 320 | "version": "==2.22.0" 321 | }, 322 | "scrapy": { 323 | "hashes": [ 324 | "sha256:4352c64c7ffc70148a7988db837bb25bccafb3350ab9c978c1f9a8930521959b", 325 | "sha256:fe06576f9a4971de9dc0175c60fd92561e8275f2bad585c1cb5d65c5181b2db0" 326 | ], 327 | "index": "pypi", 328 | "version": "==1.8.0" 329 | }, 330 | "service-identity": { 331 | "hashes": [ 332 | "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36", 333 | "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d" 334 | ], 335 | "version": "==18.1.0" 336 | }, 337 | "six": { 338 | "hashes": [ 339 | "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", 340 | "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" 341 | ], 342 | "version": "==1.13.0" 343 | }, 344 | "twisted": { 345 | "hashes": [ 346 | "sha256:0f39698c2aac318032ed4fe95e28ee2bd7d72327c2f6927139811ad403770885", 347 | "sha256:1f0919a0363b4fbed5def5315383db36fd581464bca80290764f8c4465e91c04", 348 | "sha256:257dbc78e72bc69c2970035fc74df54b04573d5ddd380251a8a23f74d619db03", 349 | "sha256:3f651c52ad78cc5a643f61e3b786a6b5c9b4ee68eced975c04fdf6b02026f470", 350 | "sha256:58b581ae4eee5a831aac9d03edc331d662fa028f601015bb3df47f8704bfe876", 351 | "sha256:611ef7696d406605962d9a7b040d357f3e91df20cf75c0b06e350947f541538b", 352 | "sha256:6338e5b987e95c94360acb14e78b41097be9b45d44d15a68060db9c3bf89e102", 353 | "sha256:7394ba7f272ae722a74f3d969dcf599bc4ef093bc392038748a490f1724a515d", 354 | "sha256:776c65270b57ac074d5b7a471142f434b0ac5a8b39d9c974769c855c32abfd91", 355 | "sha256:8b2f7f4dded5ad02931bed38042e55329d1e4919b63078f5a29f05502a163f1d", 356 | "sha256:97f8a76632bf051a27179337fe937df315920a08e9bd146126e0126629db3721", 357 | "sha256:a1de7598ce977943b3edbcc0a7d2112f134cc1b98b2fd7e348ee9e0bef092e50", 358 | "sha256:d145c418a46f8a7021030a3246b9e5111f64531278e5252f2073f23c1661c8be", 359 | "sha256:d53e1f883bc429b14fd2999d355352974f9d7b4ae7554154bbfe3f90aecede5f", 360 | "sha256:d9037ff5e07909b1d31f81db71a3bbc8227ba1ed20c87332bdb2eb48e55f326e", 361 | "sha256:ef1396d1680d6a1ae6dff293d778755c8e10d471f286aff678877b2cee235d42", 362 | "sha256:f1fe9139fdcf7721d308e36c51cf975474678a8241a9799af02a7bb1c531b722", 363 | "sha256:f28355e61ce0b5c1ce47784522022322cc5059c8ed80638e0caae8c7301e1705", 364 | "sha256:f7cc56a45c983e4e48601fbeec879b62c740cb848c75164f69a48ac0c6e8a21c" 365 | ], 366 | "markers": "python_version >= '3.5'", 367 | "version": "==19.10.0" 368 | }, 369 | "tzlocal": { 370 | "hashes": [ 371 | "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048", 372 | "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590" 373 | ], 374 | "version": "==2.0.0" 375 | }, 376 | "urllib3": { 377 | "hashes": [ 378 | "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", 379 | "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" 380 | ], 381 | "version": "==1.25.7" 382 | }, 383 | "w3lib": { 384 | "hashes": [ 385 | "sha256:847704b837b2b973cddef6938325d466628e6078266bc2e1f7ac49ba85c34823", 386 | "sha256:8b1854fef570b5a5fc84d960e025debd110485d73fd283580376104762774315" 387 | ], 388 | "version": "==1.21.0" 389 | }, 390 | "zope.interface": { 391 | "hashes": [ 392 | "sha256:048b16ac882a05bc7ef534e8b9f15c9d7a6c190e24e8938a19b7617af4ed854a", 393 | "sha256:05816cf8e7407cf62f2ec95c0a5d69ec4fa5741d9ccd10db9f21691916a9a098", 394 | "sha256:065d6a1ac89d35445168813bed45048ed4e67a4cdfc5a68fdb626a770378869f", 395 | "sha256:14157421f4121a57625002cc4f48ac7521ea238d697c4a4459a884b62132b977", 396 | "sha256:18dc895945694f397a0be86be760ff664b790f95d8e7752d5bab80284ff9105d", 397 | "sha256:1962c9f838bd6ae4075d0014f72697510daefc7e1c7e48b2607df0b6e157989c", 398 | "sha256:1a67408cacd198c7e6274a19920bb4568d56459e659e23c4915528686ac1763a", 399 | "sha256:21bf781076dd616bd07cf0223f79d61ab4f45176076f90bc2890e18c48195da4", 400 | "sha256:21c0a5d98650aebb84efa16ce2c8df1a46bdc4fe8a9e33237d0ca0b23f416ead", 401 | "sha256:23cfeea25d1e42ff3bf4f9a0c31e9d5950aa9e7c4b12f0c4bd086f378f7b7a71", 402 | "sha256:24b6fce1fb71abf9f4093e3259084efcc0ef479f89356757780685bd2b06ef37", 403 | "sha256:24f84ce24eb6b5fcdcb38ad9761524f1ae96f7126abb5e597f8a3973d9921409", 404 | "sha256:25e0ef4a824017809d6d8b0ce4ab3288594ba283e4d4f94d8cfb81d73ed65114", 405 | "sha256:2e8fdd625e9aba31228e7ddbc36bad5c38dc3ee99a86aa420f89a290bd987ce9", 406 | "sha256:2f3bc2f49b67b1bea82b942d25bc958d4f4ea6709b411cb2b6b9718adf7914ce", 407 | "sha256:35d24be9d04d50da3a6f4d61de028c1dd087045385a0ff374d93ef85af61b584", 408 | "sha256:35dbe4e8c73003dff40dfaeb15902910a4360699375e7b47d3c909a83ff27cd0", 409 | "sha256:3dfce831b824ab5cf446ed0c350b793ac6fa5fe33b984305cb4c966a86a8fb79", 410 | "sha256:3f7866365df5a36a7b8de8056cd1c605648f56f9a226d918ed84c85d25e8d55f", 411 | "sha256:455cc8c01de3bac6f9c223967cea41f4449f58b4c2e724ec8177382ddd183ab4", 412 | "sha256:4bb937e998be9d5e345f486693e477ba79e4344674484001a0b646be1d530487", 413 | "sha256:52303a20902ca0888dfb83230ca3ee6fbe63c0ad1dd60aa0bba7958ccff454d8", 414 | "sha256:6e0a897d4e09859cc80c6a16a29697406ead752292ace17f1805126a4f63c838", 415 | "sha256:6e1816e7c10966330d77af45f77501f9a68818c065dec0ad11d22b50a0e212e7", 416 | "sha256:73b5921c5c6ce3358c836461b5470bf675601c96d5e5d8f2a446951470614f67", 417 | "sha256:8093cd45cdb5f6c8591cfd1af03d32b32965b0f79b94684cd0c9afdf841982bb", 418 | "sha256:864b4a94b60db301899cf373579fd9ef92edddbf0fb2cd5ae99f53ef423ccc56", 419 | "sha256:8a27b4d3ea9c6d086ce8e7cdb3e8d319b6752e2a03238a388ccc83ccbe165f50", 420 | "sha256:91b847969d4784abd855165a2d163f72ac1e58e6dce09a5e46c20e58f19cc96d", 421 | "sha256:b47b1028be4758c3167e474884ccc079b94835f058984b15c145966c4df64d27", 422 | "sha256:b68814a322835d8ad671b7acc23a3b2acecba527bb14f4b53fc925f8a27e44d8", 423 | "sha256:bcb50a032c3b6ec7fb281b3a83d2b31ab5246c5b119588725b1350d3a1d9f6a3", 424 | "sha256:c56db7d10b25ce8918b6aec6b08ac401842b47e6c136773bfb3b590753f7fb67", 425 | "sha256:c94b77a13d4f47883e4f97f9fa00f5feadd38af3e6b3c7be45cfdb0a14c7149b", 426 | "sha256:db381f6fdaef483ad435f778086ccc4890120aff8df2ba5cfeeac24d280b3145", 427 | "sha256:e6487d01c8b7ed86af30ea141fcc4f93f8a7dde26f94177c1ad637c353bd5c07", 428 | "sha256:e86923fa728dfba39c5bb6046a450bd4eec8ad949ac404eca728cfce320d1732", 429 | "sha256:f6ca36dc1e9eeb46d779869c60001b3065fb670b5775c51421c099ea2a77c3c9", 430 | "sha256:fb62f2cbe790a50d95593fb40e8cca261c31a2f5637455ea39440d6457c2ba25" 431 | ], 432 | "version": "==4.7.1" 433 | } 434 | }, 435 | "develop": {} 436 | } 437 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

DuTracker

2 | 3 | 4 | :warning: 现在每天要板砖了 此项目不再维护 部分接口已失效 5 | 6 | --- 7 | 8 | 9 | 本项目旨在追踪 [毒-运动x潮流x装备](http://m.poizon.com/website/pc)各个商品价格变化 10 | 11 | ![dashboard](./.images/dashboard.png) 12 | 13 | ## Requires 14 | 15 | - python requirements `pipenv install` 16 | - docker envirements `docker-compose up -d` 17 | - [wildcat/scylla](https://github.com/imWildCat/scylla)(默认未开启) Scylla 是一款高质量的免费代理 IP 池工具 18 | - [docker.io/influxdb:latest](https://docs.docker.com/samples/library/influxdb/) 开源时序数据库 19 | - [grafana/grafana](https://github.com/grafana/grafana) 时序数据展示平台 20 | 21 | ## Getting started 22 | 1. `docker-compose up -d`启动程序 23 | 2. 访问:`http://localhost:3000/`配置influxDB,如图: 24 | ![](http://easy-file.never615.com/upic/068VIJ.png) 25 | 3. 从[template.json](./template.json)导入DashBoard 26 | 4. 利用抓包工具获取 *毒App内:购买-分类-品牌/系列-任意子项(Nike,Converse)* 访问的URL,替换`DuTracker/utils/urls.py`中的信息 27 | 28 | ## Usage 29 | 30 | - 导入需要追踪的商品ID 31 | - 导入指定商品ID `python dt.py addproduct 1 2 3` 32 | - 导入指定品牌、系列所有附属商品 33 | - [更新 `DuTracker/utils/urls.py`信息](#url更新方式) 34 | - `python dt.py crawl` 35 | - 追踪商品价格趋势 36 | - `python dt.py start -v` 37 | - 自动更新指定品牌、系列包含商品 `python dt.py start -v -b 176 -s 53` 38 | - 追踪销量高于 xx 的商品 `python dt.py start --min xx` 39 | - 通过**scylla**代理 `python dt.py start -v --proxy http://127.0.0.1:8081` 40 | - Grafana配置 41 | - 添加InfluxDB 数据源 42 | - 从[template.json](./template.json)导入DashBoard 43 | 44 | ## FAQ 45 | 46 | ### URL更新方式 47 | 48 | 利用**Fildder** 获取 *毒App内:购买-分类-品牌/系列-任意子项(Nike,Converse)* 访问的URL,替换`DuTracker/utils/urls.py`中的信息 49 | 50 | ### grafana docker 启动失败 51 | 52 | 查看容器log得到以下信息 53 | 54 | ```bash 55 | # docker logs dutracker_ui_1 56 | GF_PATHS_DATA='/var/lib/grafana' is not writable. 57 | You may have issues with file permissions, more information here: http://docs.grafana.org/installation/docker/#migration-from-a-previous-version-of-the-docker-container-to-5-1-or-later 58 | mkdir: cannot create directory '/var/lib/grafana/plugins': Permission denied 59 | ``` 60 | 61 | Solution: 62 | 63 | ```bash 64 | chmod -R 777 volumes/grafana 65 | docker start dutracker_ui_1 66 | ``` 67 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | # proxy: 4 | # image: wildcat/scylla 5 | # ports: 6 | # - 8899:8899 7 | # - 8081:8081 8 | # volumes: 9 | # - ./volumes/scylla:/var/www/scylla 10 | db: 11 | image: docker.io/influxdb:latest 12 | ports: 13 | - 8086:8086 14 | volumes: 15 | - ./volumes/influxdb:/var/lib/influxdb 16 | environment: 17 | INFLUXDB_DB: duApp 18 | ui: 19 | image: grafana/grafana 20 | ports: 21 | - 3000:3000 22 | volumes: 23 | - ./volumes/grafana:/var/lib/grafana 24 | 25 | -------------------------------------------------------------------------------- /dt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/03/12 21:10 4 | # @Author : Xu 5 | # @Site : https://xuccc.github.io/ 6 | 7 | import click 8 | import logging 9 | import sys 10 | from scrapy.crawler import CrawlerRunner, CrawlerProcess 11 | from scrapy.utils.project import get_project_settings 12 | from twisted.internet import reactor, defer 13 | from apscheduler.schedulers.twisted import TwistedScheduler 14 | 15 | from DuTracker.spiders.brand import BrandSpider 16 | from DuTracker.spiders.serie import SerieSpider 17 | from DuTracker.spiders.product import ProductSpider 18 | from DuTracker.spiders.tracker import TrackerSpider 19 | from DuTracker.utils.log import log 20 | 21 | 22 | @click.group() 23 | def cli(): 24 | pass 25 | 26 | 27 | @cli.command() 28 | def show(): 29 | settings = get_project_settings() 30 | log.info('显示远程品牌&系列信息') 31 | runner = CrawlerRunner(settings) 32 | 33 | @defer.inlineCallbacks 34 | def crawl(): 35 | yield runner.crawl(BrandSpider, auto=True) 36 | yield runner.crawl(SerieSpider, auto=True) 37 | reactor.stop() 38 | 39 | crawl() 40 | reactor.run() 41 | 42 | 43 | @cli.command(help='Init Database') 44 | @click.option('--verbose', '-v', is_flag=True, default=False, ) 45 | @click.option('--debug', is_flag=True, default=False, help='show scrapy log') 46 | @click.option('--proxy', help='proxy url') 47 | def crawl(verbose, debug, proxy, ): 48 | settings = get_project_settings() 49 | 50 | if verbose: 51 | log.setLevel(logging.DEBUG) 52 | if proxy: 53 | settings['DOWNLOADER_MIDDLEWARES'].update({ 54 | 'DuTracker.middlewares.RandomProxy': 760 55 | }) 56 | settings['PROXY_URL'] = proxy 57 | if debug: 58 | settings['LOG_ENABLED'] = True 59 | 60 | log.info('初始化数据库 product.sqlite') 61 | runner = CrawlerRunner(settings) 62 | 63 | @defer.inlineCallbacks 64 | def crawl(): 65 | yield runner.crawl(BrandSpider) 66 | yield runner.crawl(SerieSpider) 67 | yield runner.crawl(ProductSpider, fromDB=True) 68 | reactor.stop() 69 | 70 | crawl() 71 | reactor.run() 72 | 73 | 74 | @cli.command(help='add product information by productId') 75 | @click.argument('pid', type=int, nargs=-1) 76 | @click.option('--verbose', '-v', is_flag=True, default=False, ) 77 | @click.option('--debug', is_flag=True, default=False, help='show scrapy log') 78 | def addproduct(pid, debug, verbose): 79 | settings = get_project_settings() 80 | 81 | if verbose: log.setLevel(logging.DEBUG) 82 | if debug: settings['LOG_ENABLED'] = True 83 | 84 | process = CrawlerProcess(settings) 85 | process.crawl(ProductSpider, productIds=pid) 86 | process.start() 87 | 88 | 89 | @cli.command(help='Monitor products\' price') 90 | @click.option('--verbose', '-v', is_flag=True, default=False, ) 91 | @click.option('--debug', is_flag=True, default=False, help='show scrapy log') 92 | @click.option('--proxy', help='proxy url') 93 | # @click.option('--day', type=int, default=1) 94 | @click.option('--min', type=int, default=1000) 95 | @click.option('--product', '-p', multiple=True, type=int, help='product ids') 96 | @click.option('--brand', '-b', multiple=True, type=int, help='brand ids') 97 | @click.option('--serie', '-s', multiple=True, type=int, help='serie ids') 98 | @click.option('--check/--no-check', default=True) 99 | @click.option('--delay', type=float, help='delay between download') 100 | @click.option('--news', is_flag=True, default=False) 101 | @click.option('--days', type=int, default=14,help='save log by days') 102 | def start(verbose, debug, proxy, min, product, brand, serie, check, delay, news, days): 103 | def check_db(): 104 | from DuTracker.tsdb import influxdb 105 | try: 106 | influxdb.ping() 107 | except Exception as e: 108 | log.error(f'InfluxDB 连接错误') 109 | sys.exit(1) 110 | else: 111 | log.success(f'InfluxDB 连接成功') 112 | 113 | if check: check_db() 114 | 115 | # https://stackoverflow.com/questions/44228851/scrapy-on-a-schedule 116 | settings = get_project_settings() 117 | 118 | if verbose: log.setLevel(logging.DEBUG) 119 | if proxy: 120 | settings['DOWNLOADER_MIDDLEWARES'].update({ 121 | 'DuTracker.middlewares.RandomProxy': 760 122 | }) 123 | settings['PROXY_URL'] = proxy 124 | if debug: settings['LOG_ENABLED'] = True 125 | if delay: settings['DOWNLOAD_DELAY'] = delay 126 | 127 | process = CrawlerProcess(settings) 128 | sched = TwistedScheduler() 129 | 130 | if brand: 131 | sched.add_job(process.crawl, 'interval', args=[BrandSpider], kwargs={'auto': True, 'Ids': brand}, days=1) 132 | process.crawl(BrandSpider, auto=True, Ids=brand) 133 | if serie: 134 | sched.add_job(process.crawl, 'interval', args=[SerieSpider], kwargs={'auto': True, 'Ids': serie}, days=1) 135 | process.crawl(SerieSpider, auto=True, Ids=serie) 136 | if brand or serie: 137 | sched.add_job(process.crawl, 'interval', args=[ProductSpider], kwargs={'fromDB': True}, days=1) 138 | process.crawl(ProductSpider, fromDB=True) 139 | process.crawl(TrackerSpider, soldNum_min=min, Ids=product) 140 | 141 | sched.add_job(process.crawl, 'interval', args=[TrackerSpider], kwargs={'soldNum_min': min, 'Ids': product}, hours=6) 142 | if news: 143 | sched.add_job(process.crawl, 'interval', args=[TrackerSpider], kwargs={'newItem': True, 'days': days}, 144 | hours=1) 145 | 146 | sched.add_job(sched.print_jobs, 'interval', hours=6) 147 | 148 | log.info('开始商品价格追踪') 149 | sched.start() 150 | process.start(False) 151 | 152 | 153 | if __name__ == '__main__': 154 | cli() 155 | -------------------------------------------------------------------------------- /scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = DuTracker.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = DuTracker 12 | -------------------------------------------------------------------------------- /template.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 1, 19 | "iteration": 1560181131454, 20 | "links": [], 21 | "panels": [ 22 | { 23 | "aliasColors": {}, 24 | "bars": false, 25 | "cacheTimeout": null, 26 | "dashLength": 10, 27 | "dashes": false, 28 | "datasource": "InfluxDB", 29 | "fill": 1, 30 | "gridPos": { 31 | "h": 11, 32 | "w": 24, 33 | "x": 0, 34 | "y": 0 35 | }, 36 | "id": 2, 37 | "legend": { 38 | "alignAsTable": true, 39 | "avg": false, 40 | "current": false, 41 | "max": true, 42 | "min": true, 43 | "rightSide": false, 44 | "show": true, 45 | "total": false, 46 | "values": true 47 | }, 48 | "lines": true, 49 | "linewidth": 1, 50 | "links": [], 51 | "nullPointMode": "connected", 52 | "paceLength": 10, 53 | "percentage": false, 54 | "pointradius": 2, 55 | "points": false, 56 | "renderer": "flot", 57 | "seriesOverrides": [], 58 | "stack": false, 59 | "steppedLine": false, 60 | "targets": [ 61 | { 62 | "alias": "$title $size", 63 | "groupBy": [ 64 | { 65 | "params": [ 66 | "$__interval" 67 | ], 68 | "type": "time" 69 | }, 70 | { 71 | "params": [ 72 | "null" 73 | ], 74 | "type": "fill" 75 | } 76 | ], 77 | "measurement": "pHistory", 78 | "orderByTime": "ASC", 79 | "policy": "default", 80 | "refId": "A", 81 | "resultFormat": "time_series", 82 | "select": [ 83 | [ 84 | { 85 | "params": [ 86 | "price" 87 | ], 88 | "type": "field" 89 | }, 90 | { 91 | "params": [], 92 | "type": "mean" 93 | } 94 | ] 95 | ], 96 | "tags": [ 97 | { 98 | "key": "title", 99 | "operator": "=~", 100 | "value": "/^$title$/" 101 | }, 102 | { 103 | "condition": "AND", 104 | "key": "size", 105 | "operator": "=~", 106 | "value": "/^$size$/" 107 | } 108 | ] 109 | }, 110 | { 111 | "alias": "$title 40", 112 | "groupBy": [ 113 | { 114 | "params": [ 115 | "$__interval" 116 | ], 117 | "type": "time" 118 | }, 119 | { 120 | "params": [ 121 | "null" 122 | ], 123 | "type": "fill" 124 | } 125 | ], 126 | "measurement": "pHistory", 127 | "orderByTime": "ASC", 128 | "policy": "default", 129 | "refId": "B", 130 | "resultFormat": "time_series", 131 | "select": [ 132 | [ 133 | { 134 | "params": [ 135 | "price" 136 | ], 137 | "type": "field" 138 | }, 139 | { 140 | "params": [], 141 | "type": "mean" 142 | } 143 | ] 144 | ], 145 | "tags": [ 146 | { 147 | "key": "title", 148 | "operator": "=~", 149 | "value": "/^$title$/" 150 | }, 151 | { 152 | "condition": "AND", 153 | "key": "size", 154 | "operator": "=", 155 | "value": "40" 156 | } 157 | ] 158 | } 159 | ], 160 | "thresholds": [], 161 | "timeFrom": null, 162 | "timeRegions": [], 163 | "timeShift": null, 164 | "title": "$title 价格走势", 165 | "tooltip": { 166 | "shared": true, 167 | "sort": 0, 168 | "value_type": "individual" 169 | }, 170 | "type": "graph", 171 | "xaxis": { 172 | "buckets": null, 173 | "mode": "time", 174 | "name": null, 175 | "show": true, 176 | "values": [] 177 | }, 178 | "yaxes": [ 179 | { 180 | "decimals": null, 181 | "format": "none", 182 | "label": null, 183 | "logBase": 1, 184 | "max": null, 185 | "min": null, 186 | "show": true 187 | }, 188 | { 189 | "format": "short", 190 | "label": "", 191 | "logBase": 1, 192 | "max": null, 193 | "min": null, 194 | "show": true 195 | } 196 | ], 197 | "yaxis": { 198 | "align": false, 199 | "alignLevel": null 200 | } 201 | }, 202 | { 203 | "aliasColors": {}, 204 | "bars": true, 205 | "cacheTimeout": null, 206 | "dashLength": 10, 207 | "dashes": false, 208 | "description": "", 209 | "fill": 1, 210 | "gridPos": { 211 | "h": 10, 212 | "w": 24, 213 | "x": 0, 214 | "y": 11 215 | }, 216 | "id": 4, 217 | "legend": { 218 | "alignAsTable": true, 219 | "avg": true, 220 | "current": false, 221 | "hideEmpty": false, 222 | "max": true, 223 | "min": true, 224 | "rightSide": false, 225 | "show": true, 226 | "total": true, 227 | "values": true 228 | }, 229 | "lines": false, 230 | "linewidth": 1, 231 | "links": [], 232 | "nullPointMode": "null", 233 | "paceLength": 10, 234 | "percentage": false, 235 | "pointradius": 2, 236 | "points": false, 237 | "renderer": "flot", 238 | "seriesOverrides": [], 239 | "stack": false, 240 | "steppedLine": false, 241 | "targets": [ 242 | { 243 | "alias": "交易量", 244 | "groupBy": [ 245 | { 246 | "params": [ 247 | "$__interval" 248 | ], 249 | "type": "time" 250 | }, 251 | { 252 | "params": [ 253 | "null" 254 | ], 255 | "type": "fill" 256 | } 257 | ], 258 | "measurement": "pHistory", 259 | "orderByTime": "ASC", 260 | "policy": "default", 261 | "refId": "A", 262 | "resultFormat": "time_series", 263 | "select": [ 264 | [ 265 | { 266 | "params": [ 267 | "soldNum" 268 | ], 269 | "type": "field" 270 | }, 271 | { 272 | "params": [], 273 | "type": "mean" 274 | }, 275 | { 276 | "params": [], 277 | "type": "difference" 278 | } 279 | ] 280 | ], 281 | "tags": [ 282 | { 283 | "key": "title", 284 | "operator": "=~", 285 | "value": "/^$title$/" 286 | }, 287 | { 288 | "condition": "AND", 289 | "key": "size", 290 | "operator": "=~", 291 | "value": "/^$size$/" 292 | } 293 | ] 294 | } 295 | ], 296 | "thresholds": [], 297 | "timeFrom": null, 298 | "timeRegions": [], 299 | "timeShift": null, 300 | "title": "$title 每日销量", 301 | "tooltip": { 302 | "shared": true, 303 | "sort": 0, 304 | "value_type": "individual" 305 | }, 306 | "type": "graph", 307 | "xaxis": { 308 | "buckets": null, 309 | "mode": "time", 310 | "name": null, 311 | "show": true, 312 | "values": [] 313 | }, 314 | "yaxes": [ 315 | { 316 | "format": "none", 317 | "label": null, 318 | "logBase": 1, 319 | "max": null, 320 | "min": null, 321 | "show": true 322 | }, 323 | { 324 | "format": "short", 325 | "label": null, 326 | "logBase": 1, 327 | "max": null, 328 | "min": null, 329 | "show": true 330 | } 331 | ], 332 | "yaxis": { 333 | "align": false, 334 | "alignLevel": null 335 | } 336 | } 337 | ], 338 | "refresh": false, 339 | "schemaVersion": 18, 340 | "style": "dark", 341 | "tags": [], 342 | "templating": { 343 | "list": [ 344 | { 345 | "allValue": null, 346 | "current": { 347 | "tags": [], 348 | "text": "Converse Chuck Taylor All Star 1970s High Navy 海军蓝", 349 | "value": "Converse Chuck Taylor All Star 1970s High Navy 海军蓝" 350 | }, 351 | "datasource": "InfluxDB", 352 | "definition": "SHOW TAG VALUES WITH KEY = \"title\"", 353 | "hide": 0, 354 | "includeAll": false, 355 | "label": null, 356 | "multi": false, 357 | "name": "title", 358 | "options": [], 359 | "query": "SHOW TAG VALUES WITH KEY = \"title\"", 360 | "refresh": 1, 361 | "regex": "", 362 | "skipUrlSync": false, 363 | "sort": 0, 364 | "tagValuesQuery": "", 365 | "tags": [], 366 | "tagsQuery": "", 367 | "type": "query", 368 | "useTags": false 369 | }, 370 | { 371 | "allValue": null, 372 | "current": { 373 | "tags": [], 374 | "text": "43", 375 | "value": "43" 376 | }, 377 | "datasource": "InfluxDB", 378 | "definition": "SHOW TAG VALUES WITH KEY = \"size\" WHERE title =~ /$title/", 379 | "hide": 0, 380 | "includeAll": false, 381 | "label": null, 382 | "multi": false, 383 | "name": "size", 384 | "options": [], 385 | "query": "SHOW TAG VALUES WITH KEY = \"size\" WHERE title =~ /$title/", 386 | "refresh": 1, 387 | "regex": "", 388 | "skipUrlSync": false, 389 | "sort": 0, 390 | "tagValuesQuery": "", 391 | "tags": [], 392 | "tagsQuery": "", 393 | "type": "query", 394 | "useTags": false 395 | } 396 | ] 397 | }, 398 | "time": { 399 | "from": "now-90d", 400 | "to": "now" 401 | }, 402 | "timepicker": { 403 | "refresh_intervals": [ 404 | "5s", 405 | "10s", 406 | "30s", 407 | "1m", 408 | "5m", 409 | "15m", 410 | "30m", 411 | "1h", 412 | "2h", 413 | "1d" 414 | ], 415 | "time_options": [ 416 | "5m", 417 | "15m", 418 | "1h", 419 | "6h", 420 | "12h", 421 | "24h", 422 | "2d", 423 | "7d", 424 | "30d" 425 | ] 426 | }, 427 | "timezone": "", 428 | "title": "DuTracker", 429 | "uid": "VEnmhNqmk", 430 | "version": 14 431 | } --------------------------------------------------------------------------------