├── .gitignore ├── LICENSE ├── README.md ├── date_utils.py ├── screenshots ├── app.png ├── csv.png ├── requirement0.png ├── requirement1.png └── requirement2.png ├── search.py ├── searchDeal.py ├── sycm.py ├── topkeywords_deal.py └── topkeywords_stream.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 iOS Dev Log 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sycm 2 | --- 3 | 4 | 生意参谋 5 | 6 | # 需求 7 | --- 8 | 9 | 帮朋友做一个生意参谋的软件，需求如下。 10 | 11 | ![](screenshots/requirement0.png) 12 | 13 | ![](screenshots/requirement1.png) 14 | 15 | ![](screenshots/requirement2.png) 16 | 17 | 1. 首先选中两个竞品 18 | 2. 再把更新时间选为日，获取每一天的数据再合并 19 | 3. 竞品入店搜索词 后面选择 **无线端** 20 | 4. 需要把 **引流关键词** 和 **成交关键词** 的数据导出 Excel 21 | 22 | # 分析 23 | --- 24 | 25 | 第 4 点只要导出 csv 文件应该就可以了。 26 | 27 | 主要任务就是模拟一天一天的获取数据。 28 | 29 | 首先想到的解决方案是用爬虫，不过要动态获取。 X 30 | 31 | 再想要用 **selenium** 等模拟浏览器行为： 麻烦。 X 32 | 33 | 最后还是抓包再说。 O 34 | 35 | 用 **Charles** 抓取数据，对比后发现返回的是 *json* 模式数据。O O 36 | 37 | 好了，这下方便多了，分析一下 *url* 及返回数据格式。 38 | 39 | 还有一个登录的问题，我这里用最简单的，获取 **Chrome** 的 *cookie*，放到请求头中。 40 | 41 | # 编码 42 | --- 43 | 44 | 花了 2 天时间，数据可以获取成功了，其中遇到不少坑。 45 | 46 | 比如 **引流关键词** 和 **成交关键词** 返回数据大概相同，我就用同一数据，结果问题报错。 47 | 48 | 后来才发现，里面的有一个字段不一样，郁闷。 49 | 50 | cookie 本来想用 browsercookie 处理，后来发现不行，最后还是写到 cookie String 里面。 51 | 52 | # GUI 53 | --- 54 | 55 | 选择官方自带的 *tkinter*。参考 最简单的例子。 56 | 57 | 也遇到不能显示界面，又是一堆的配置问题，安装 `python` 还是用网络安装的方式最好。 58 | 59 | ![](screenshots/app.png) 60 | 61 | 编码可以了，不过人家要可以执行的软件，于是用 **pyinstaller** 生成 *exe* 文件，这才是一个天大的坑。 62 | 63 | 没有看说明文档： 64 | 65 | ``` 66 | Downloads 67 | The latest stable release of PyInstaller is 3.3.1 (Change Log). 68 | 69 | Release 3.3.1: stable, supports Python 2.7, 3.3–3.6 70 | 71 | PyInstaller 3.3.1 (tar.gz) (pgp, sha-256: 715f81f24b1ef0e5fe3b3c71e7540551838e46e9de30882aa7c0a521147fd1ce) 72 | Development: unstable, supports Python 2.7, 3.4–3.6 73 | 74 | ``` 75 | 76 | 在 *Windows* 下打包总是出错，因为我下载的 **Python 3.7.0**。 cry... 77 | 78 | # 交付 79 | --- 80 | 81 | 数据可以获取了，也有一个丑陋的界面，也可以导出数据了。 82 | 83 | 可以交付了吧，当然不行！ 84 | 85 | 导出数据想要 WPS 能打开。嗯，csv 可以另存为 *xlsx*，不算问题。 86 | 87 | 数据想要合并，不要日期，不要其它说明，只要数据汇总，好，可以。 88 | 89 | ![](screenshots/csv.png) 90 | 91 | # Licence 92 | ---- 93 | 94 | `sycm` is released under the MIT license. See [LICENSE](LICENSE) for details. 95 | -------------------------------------------------------------------------------- /date_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from datetime import datetime 3 | 4 | def stringToDate(string): 5 | #example '2018-07-22' 6 | dt = datetime.strptime(string, "%Y-%m-%d") 7 | #print dt 8 | return dt 9 | 10 | ''''' Date(datetime) to String ''' 11 | 12 | def dateToString(date): 13 | ds = date.strftime('%Y-%m-%d') 14 | return ds 15 | -------------------------------------------------------------------------------- /screenshots/app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iOSDevLog/sycm/a456952e1c8a146220f2d811a01f8f0731520a9a/screenshots/app.png -------------------------------------------------------------------------------- /screenshots/csv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iOSDevLog/sycm/a456952e1c8a146220f2d811a01f8f0731520a9a/screenshots/csv.png -------------------------------------------------------------------------------- /screenshots/requirement0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iOSDevLog/sycm/a456952e1c8a146220f2d811a01f8f0731520a9a/screenshots/requirement0.png -------------------------------------------------------------------------------- /screenshots/requirement1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iOSDevLog/sycm/a456952e1c8a146220f2d811a01f8f0731520a9a/screenshots/requirement1.png -------------------------------------------------------------------------------- /screenshots/requirement2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iOSDevLog/sycm/a456952e1c8a146220f2d811a01f8f0731520a9a/screenshots/requirement2.png -------------------------------------------------------------------------------- /search.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # To use this code, make sure you 3 | # 4 | # import json 5 | # 6 | # and then, to convert JSON from a string, do 7 | # 8 | # result = search_from_dict(json.loads(json_string)) 9 | 10 | from typing import Any, List, TypeVar, Type, cast, Callable 11 | 12 | 13 | T = TypeVar("T") 14 | 15 | 16 | def from_str(x: Any) -> str: 17 | assert isinstance(x, str) 18 | return x 19 | 20 | 21 | def from_int(x: Any) -> int: 22 | assert isinstance(x, int) and not isinstance(x, bool) 23 | return x 24 | 25 | 26 | def to_class(c: Type[T], x: Any) -> dict: 27 | assert isinstance(x, c) 28 | return cast(Any, x).to_dict() 29 | 30 | 31 | def from_list(f: Callable[[Any], T], x: Any) -> List[T]: 32 | assert isinstance(x, list) 33 | return [f(y) for y in x] 34 | 35 | 36 | class Keyword: 37 | value: str 38 | 39 | def __init__(self, value: str) -> None: 40 | self.value = value 41 | 42 | @staticmethod 43 | def from_dict(obj: Any) -> 'Keyword': 44 | assert isinstance(obj, dict) 45 | value = from_str(obj.get("value")) 46 | return Keyword(value) 47 | 48 | def to_dict(self) -> dict: 49 | result: dict = {} 50 | result["value"] = from_str(self.value) 51 | return result 52 | 53 | 54 | class Uv: 55 | value: int 56 | 57 | def __init__(self, value: int) -> None: 58 | self.value = value 59 | 60 | @staticmethod 61 | def from_dict(obj: Any) -> 'Uv': 62 | assert isinstance(obj, dict) 63 | value = from_int(obj.get("value")) 64 | return Uv(value) 65 | 66 | def to_dict(self) -> dict: 67 | result: dict = {} 68 | result["value"] = from_int(self.value) 69 | return result 70 | 71 | 72 | class Rival: 73 | uv: Uv 74 | keyword: Keyword 75 | 76 | def __init__(self, uv: Uv, keyword: Keyword) -> None: 77 | self.uv = uv 78 | self.keyword = keyword 79 | 80 | @staticmethod 81 | def from_dict(obj: Any) -> 'Rival': 82 | assert isinstance(obj, dict) 83 | uv = Uv.from_dict(obj.get("uv")) 84 | keyword = Keyword.from_dict(obj.get("keyword")) 85 | return Rival(uv, keyword) 86 | 87 | def to_dict(self) -> dict: 88 | result: dict = {} 89 | result["uv"] = to_class(Uv, self.uv) 90 | result["keyword"] = to_class(Keyword, self.keyword) 91 | return result 92 | 93 | 94 | class Data: 95 | rival1: List[Rival] 96 | rival2: List[Rival] 97 | 98 | def __init__(self, rival1: List[Rival], rival2: List[Rival]) -> None: 99 | self.rival1 = rival1 100 | self.rival2 = rival2 101 | 102 | @staticmethod 103 | def from_dict(obj: Any) -> 'Data': 104 | assert isinstance(obj, dict) 105 | rival1 = from_list(Rival.from_dict, obj.get("rival1")) 106 | rival2 = from_list(Rival.from_dict, obj.get("rival2")) 107 | return Data(rival1, rival2) 108 | 109 | def to_dict(self) -> dict: 110 | result: dict = {} 111 | result["rival1"] = from_list(lambda x: to_class(Rival, x), self.rival1) 112 | result["rival2"] = from_list(lambda x: to_class(Rival, x), self.rival2) 113 | return result 114 | 115 | 116 | class Search: 117 | trace_id: str 118 | code: int 119 | data: Data 120 | message: str 121 | 122 | def __init__(self, trace_id: str, code: int, data: Data, message: str) -> None: 123 | self.trace_id = trace_id 124 | self.code = code 125 | self.data = data 126 | self.message = message 127 | 128 | @staticmethod 129 | def from_dict(obj: Any) -> 'Search': 130 | assert isinstance(obj, dict) 131 | trace_id = from_str(obj.get("traceId")) 132 | code = from_int(obj.get("code")) 133 | data = Data.from_dict(obj.get("data")) 134 | message = from_str(obj.get("message")) 135 | return Search(trace_id, code, data, message) 136 | 137 | def to_dict(self) -> dict: 138 | result: dict = {} 139 | result["traceId"] = from_str(self.trace_id) 140 | result["code"] = from_int(self.code) 141 | result["data"] = to_class(Data, self.data) 142 | result["message"] = from_str(self.message) 143 | return result 144 | 145 | 146 | def search_from_dict(s: Any) -> Search: 147 | return Search.from_dict(s) 148 | 149 | 150 | def search_to_dict(x: Search) -> Any: 151 | return to_class(Search, x) 152 | 153 | -------------------------------------------------------------------------------- /searchDeal.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # To use this code, make sure you 3 | # 4 | # import json 5 | # 6 | # and then, to convert JSON from a string, do 7 | # 8 | # result = search_deal_from_dict(json.loads(json_string)) 9 | 10 | from typing import Any, List, TypeVar, Type, cast, Callable 11 | 12 | 13 | T = TypeVar("T") 14 | 15 | 16 | def from_str(x: Any) -> str: 17 | assert isinstance(x, str) 18 | return x 19 | 20 | 21 | def from_int(x: Any) -> int: 22 | assert isinstance(x, int) and not isinstance(x, bool) 23 | return x 24 | 25 | 26 | def to_class(c: Type[T], x: Any) -> dict: 27 | assert isinstance(x, c) 28 | return cast(Any, x).to_dict() 29 | 30 | 31 | def from_list(f: Callable[[Any], T], x: Any) -> List[T]: 32 | assert isinstance(x, list) 33 | return [f(y) for y in x] 34 | 35 | 36 | class Keyword: 37 | value: str 38 | 39 | def __init__(self, value: str) -> None: 40 | self.value = value 41 | 42 | @staticmethod 43 | def from_dict(obj: Any) -> 'Keyword': 44 | assert isinstance(obj, dict) 45 | value = from_str(obj.get("value")) 46 | return Keyword(value) 47 | 48 | def to_dict(self) -> dict: 49 | result: dict = {} 50 | result["value"] = from_str(self.value) 51 | return result 52 | 53 | 54 | class PayItemCnt: 55 | value: int 56 | 57 | def __init__(self, value: int) -> None: 58 | self.value = value 59 | 60 | @staticmethod 61 | def from_dict(obj: Any) -> 'PayItemCnt': 62 | assert isinstance(obj, dict) 63 | value = from_int(obj.get("value")) 64 | return PayItemCnt(value) 65 | 66 | def to_dict(self) -> dict: 67 | result: dict = {} 68 | result["value"] = from_int(self.value) 69 | return result 70 | 71 | 72 | class Rival: 73 | pay_item_cnt: PayItemCnt 74 | keyword: Keyword 75 | 76 | def __init__(self, pay_item_cnt: PayItemCnt, keyword: Keyword) -> None: 77 | self.pay_item_cnt = pay_item_cnt 78 | self.keyword = keyword 79 | 80 | @staticmethod 81 | def from_dict(obj: Any) -> 'Rival': 82 | assert isinstance(obj, dict) 83 | pay_item_cnt = PayItemCnt.from_dict(obj.get("payItemCnt")) 84 | keyword = Keyword.from_dict(obj.get("keyword")) 85 | return Rival(pay_item_cnt, keyword) 86 | 87 | def to_dict(self) -> dict: 88 | result: dict = {} 89 | result["payItemCnt"] = to_class(PayItemCnt, self.pay_item_cnt) 90 | result["keyword"] = to_class(Keyword, self.keyword) 91 | return result 92 | 93 | 94 | class Data: 95 | rival1: List[Rival] 96 | rival2: List[Rival] 97 | 98 | def __init__(self, rival1: List[Rival], rival2: List[Rival]) -> None: 99 | self.rival1 = rival1 100 | self.rival2 = rival2 101 | 102 | @staticmethod 103 | def from_dict(obj: Any) -> 'Data': 104 | assert isinstance(obj, dict) 105 | rival1 = from_list(Rival.from_dict, obj.get("rival1")) 106 | rival2 = from_list(Rival.from_dict, obj.get("rival2")) 107 | return Data(rival1, rival2) 108 | 109 | def to_dict(self) -> dict: 110 | result: dict = {} 111 | result["rival1"] = from_list(lambda x: to_class(Rival, x), self.rival1) 112 | result["rival2"] = from_list(lambda x: to_class(Rival, x), self.rival2) 113 | return result 114 | 115 | 116 | class SearchDeal: 117 | trace_id: str 118 | code: int 119 | data: Data 120 | message: str 121 | 122 | def __init__(self, trace_id: str, code: int, data: Data, message: str) -> None: 123 | self.trace_id = trace_id 124 | self.code = code 125 | self.data = data 126 | self.message = message 127 | 128 | @staticmethod 129 | def from_dict(obj: Any) -> 'SearchDeal': 130 | assert isinstance(obj, dict) 131 | trace_id = from_str(obj.get("traceId")) 132 | code = from_int(obj.get("code")) 133 | data = Data.from_dict(obj.get("data")) 134 | message = from_str(obj.get("message")) 135 | return SearchDeal(trace_id, code, data, message) 136 | 137 | def to_dict(self) -> dict: 138 | result: dict = {} 139 | result["traceId"] = from_str(self.trace_id) 140 | result["code"] = from_int(self.code) 141 | result["data"] = to_class(Data, self.data) 142 | result["message"] = from_str(self.message) 143 | return result 144 | 145 | 146 | def search_deal_from_dict(s: Any) -> SearchDeal: 147 | return SearchDeal.from_dict(s) 148 | 149 | 150 | def search_deal_to_dict(x: SearchDeal) -> Any: 151 | return to_class(SearchDeal, x) 152 | 153 | -------------------------------------------------------------------------------- /sycm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import time 3 | import os 4 | import csv 5 | from datetime import datetime,timedelta 6 | import tkinter as tk 7 | from tkinter import messagebox as msg 8 | from date_utils import * 9 | from topkeywords_stream import fetchStreamTopKeywords 10 | from topkeywords_deal import fetchDealTopKeywords 11 | 12 | class Timer(tk.Tk): 13 | def __init__(self): 14 | super().__init__() 15 | self.title("生意参谋") 16 | self.geometry("500x800") 17 | self.resizable(False, False) 18 | 19 | self.standard_font = (None, 16) 20 | 21 | self.main_frame = tk.Frame(self, width=500, height=800, bg="lightgrey") 22 | 23 | self.val1_label = tk.Label(self.main_frame, text="竞品1 ID", bg="lightgrey", fg="black", font=self.standard_font) 24 | self.val1_entry = tk.Entry(self.main_frame, bg="white", fg="black", font=self.standard_font) 25 | 26 | self.val2_label = tk.Label(self.main_frame, text="竞品2 ID", bg="lightgrey", fg="black", font=self.standard_font) 27 | self.val2_entry = tk.Entry(self.main_frame, bg="white", fg="black", font=self.standard_font) 28 | 29 | self.start_date_label = tk.Label(self.main_frame, text="开始日期", bg="lightgrey", fg="black", font=self.standard_font) 30 | self.start_date_entry = tk.Entry(self.main_frame, bg="white", fg="black", font=self.standard_font) 31 | 32 | self.end_date_label = tk.Label(self.main_frame, text="结束日期:", bg="lightgrey", fg="black", font=self.standard_font) 33 | self.end_date_entry = tk.Entry(self.main_frame, bg="white", fg="black", font=self.standard_font) 34 | 35 | self.start_button = tk.Button(self.main_frame, text="运行", bg="lightgrey", fg="black", command=self.start, font=self.standard_font) 36 | 37 | progress = tk.StringVar() 38 | self.progress_label = tk.Label(self.main_frame, text="当前处理日期:", bg="lightgrey", fg="black", font=self.standard_font) 39 | self.progress_entry = tk.Entry(self.main_frame, bg="white", fg="black", font=self.standard_font, textvariable=progress) 40 | 41 | self.main_frame.pack(fill=tk.BOTH, expand=1) 42 | 43 | self.val1_label.pack(fill=tk.X, pady=10) 44 | self.val1_entry.pack(fill=tk.X, padx=50, pady=(0,20)) 45 | self.val1_entry.insert(0,"25649816644") 46 | 47 | self.val2_label.pack(fill=tk.X, pady=10) 48 | self.val2_entry.pack(fill=tk.X, padx=50, pady=(0,20)) 49 | self.val2_entry.insert(0,"524007275779") 50 | 51 | self.start_date_label.pack(fill=tk.X, pady=10) 52 | self.start_date_entry.pack(fill=tk.X, padx=50, pady=(0,20)) 53 | self.start_date_entry.insert(0,"2018-07-01") 54 | 55 | self.end_date_label.pack(fill=tk.X, pady=10) 56 | self.end_date_entry.pack(fill=tk.X, padx=50, pady=(0,20)) 57 | self.end_date_entry.insert(0,"2018-07-29") 58 | 59 | self.start_button.pack(fill=tk.X, padx=50) 60 | 61 | self.progress_label.pack(fill=tk.X, pady=10) 62 | self.progress_entry.pack(fill=tk.X, padx=50, pady=(0,20)) 63 | 64 | def start(self): 65 | start_date_string = self.start_date_entry.get() 66 | end_date_string = self.end_date_entry.get() 67 | rival1Id = self.val1_entry.get() 68 | rival2Id = self.val2_entry.get() 69 | start_date = stringToDate(start_date_string) 70 | end_date = stringToDate(end_date_string) 71 | print(dateToString(start_date)) 72 | print(dateToString(end_date)) 73 | self.progress_entry.delete(0, tk.END) 74 | self.progress_entry.insert(0,"开始处理") 75 | 76 | stream1s = [] 77 | deal1s = [] 78 | stream2s = [] 79 | deal2s = [] 80 | 81 | date_string = start_date_string + "_" + end_date_string 82 | 83 | for i in range((end_date - start_date).days): 84 | 85 | day = start_date + timedelta(days=i) 86 | stringDay = dateToString(day) 87 | # print(day) 88 | progress = stringDay 89 | self.progress_entry.delete(0, tk.END) 90 | self.progress_entry.insert(0, progress) 91 | (stream1, stream2) = fetchStreamTopKeywords(stringDay, int(rival1Id), int(rival2Id)) 92 | (deal1, deal2) = fetchDealTopKeywords(stringDay, int(rival1Id), int(rival2Id)) 93 | stream1s.extend(stream1) 94 | stream2s.extend(stream2) 95 | deal1s.extend(deal1) 96 | deal2s.extend(deal2) 97 | 98 | with open('引流/' + str(rival1Id) + '_' + date_string + '.csv', 'w', newline='') as csvfile: 99 | writer = csv.writer(csvfile) 100 | for stream in stream1s: 101 | writer.writerow([stream.keyword.value , stream.uv.value]) 102 | 103 | with open('引流/' + str(rival2Id) + '_' + date_string + '.csv', 'w', newline='') as csvfile: 104 | writer = csv.writer(csvfile) 105 | for stream in stream2s: 106 | writer.writerow([stream.keyword.value , stream.uv.value]) 107 | 108 | with open('交易/' + str(rival1Id) + '_' + date_string + '.csv', 'w', newline='') as csvfile: 109 | writer = csv.writer(csvfile) 110 | for deal in deal1s: 111 | writer.writerow([deal.keyword.value , deal.pay_item_cnt.value]) 112 | 113 | with open('交易/' + str(rival2Id) + '_' + date_string + '.csv', 'w', newline='') as csvfile: 114 | writer = csv.writer(csvfile) 115 | for deal in deal2s: 116 | writer.writerow([deal.keyword.value , deal.pay_item_cnt.value]) 117 | 118 | self.progress_entry.delete(0, tk.END) 119 | self.progress_entry.insert(0,"完成") 120 | 121 | 122 | def mkdir(path): 123 | folder = os.path.exists(path) 124 | 125 | if not folder: 126 | os.makedirs(path) 127 | print("创建成功") 128 | else: 129 | print("文件夹已经存在") 130 | 131 | 132 | if __name__ == "__main__": 133 | file = "引流" 134 | mkdir(file) 135 | file = "交易" 136 | mkdir(file) 137 | 138 | timer = Timer() 139 | timer.mainloop() 140 | -------------------------------------------------------------------------------- /topkeywords_deal.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | import json 4 | import os 5 | import datetime 6 | from searchDeal import * 7 | import csv 8 | 9 | def fetchDealTopKeywords(date, rival1Id, rival2Id): 10 | dateRange = date + "|" + date 11 | refererUrl = "https://sycm.taobao.com/ci/monitor/itemcompare" 12 | 13 | TOP_TYPE_STREAM = 1 14 | TOP_TYPE_DEAL = 2 15 | topType = TOP_TYPE_DEAL 16 | url = "https://sycm.taobao.com/ci/item/compare/topkeywords.json" 17 | querystring = {"deviceType":2,"topType":topType,"seType":"taobao","dateType":"day","dateRange":dateRange,"needSelf":"false","rival1Id":rival1Id,"rival2Id":rival2Id,"_":1532857215509,"token":"baa876795"} 18 | 19 | # replace cookie 20 | cookie = '''cna=cna;t=t''' 21 | 22 | headers = { 23 | 'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", 24 | 'accept': "*/*", 25 | 'accept-encoding': "gzip, deflate, br", 26 | 'accept-language': "zh-CN,zh;q=0.9", 27 | 'Cache-Control': "no-cache", 28 | 'referer': refererUrl, 29 | 'cookie': cookie 30 | } 31 | 32 | response = requests.request("GET", url, headers=headers, params=querystring) 33 | 34 | json_string = response.text 35 | result = search_deal_from_dict(json.loads(json_string)) 36 | 37 | data = result.data 38 | 39 | rival1 = data.rival1 40 | rival2 = data.rival2 41 | 42 | return (rival1, rival2) 43 | 44 | # with open(date + '_' + str(rival1Id) + '_deal_key_words.csv', 'w+', newline='') as csvfile: 45 | # for rival in rival1: 46 | # print(rival.keyword.value + "," + str(rival.pay_item_cnt.value)) 47 | 48 | # writer = csv.writer(csvfile) 49 | # writer.writerow([rival.keyword.value , rival.pay_item_cnt.value]) 50 | 51 | # # print('\n\n') 52 | # # writer.writerow([rival.keyword.value , rival.uv.value]) 53 | 54 | # with open(date + '_' + str(rival2Id) + '_deal_key_words.csv', 'w+', newline='') as csvfile: 55 | # for rival in rival2: 56 | # print(rival.keyword.value + "," + str(rival.pay_item_cnt.value)) 57 | 58 | # writer = csv.writer(csvfile) 59 | # writer.writerow([rival.keyword.value , rival.pay_item_cnt.value]) 60 | 61 | 62 | 63 | def main(): 64 | begin = datetime.date(2018,7,1) 65 | end = datetime.date.today() 66 | for i in range((end - begin).days): 67 | day = begin + datetime.timedelta(days=i) 68 | # fetchDealTopKeywords(str(day)) 69 | 70 | if __name__ == '__main__': 71 | main() 72 | 73 | -------------------------------------------------------------------------------- /topkeywords_stream.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | import json 4 | import os 5 | import datetime 6 | from search import * 7 | import csv 8 | 9 | def fetchStreamTopKeywords(date, rival1Id, rival2Id): 10 | dateRange = date + "|" + date 11 | refererUrl = "https://sycm.taobao.com/ci/monitor/itemcompare" 12 | 13 | TOP_TYPE_STREAM = 1 14 | TOP_TYPE_DEAL = 2 15 | topType = TOP_TYPE_STREAM 16 | url = "https://sycm.taobao.com/ci/item/compare/topkeywords.json" 17 | querystring = {"deviceType":2,"topType":topType,"seType":"taobao","dateType":"day","dateRange":dateRange,"needSelf":"false","rival1Id":rival1Id,"rival2Id":rival2Id,"_":1532857215509,"token":"baa876795"} 18 | 19 | # replace cookie 20 | cookie = '''cna=cna;t=t''' 21 | 22 | headers = { 23 | 'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", 24 | 'accept': "*/*", 25 | 'accept-encoding': "gzip, deflate, br", 26 | 'accept-language': "zh-CN,zh;q=0.9", 27 | 'Cache-Control': "no-cache", 28 | 'referer': refererUrl, 29 | 'cookie': cookie 30 | } 31 | 32 | response = requests.request("GET", url, headers=headers, params=querystring) 33 | 34 | json_string = response.text 35 | result = search_from_dict(json.loads(json_string)) 36 | 37 | data = result.data 38 | 39 | rival1 = data.rival1 40 | rival2 = data.rival2 41 | 42 | return (rival1, rival2) 43 | 44 | # with open(date + '_' + str(rival1Id) + '_stream_key_words.csv', 'w+', newline='') as csvfile: 45 | # for rival in rival1: 46 | # print(rival.keyword.value + "," + str(rival.uv.value)) 47 | 48 | # writer = csv.writer(csvfile) 49 | # writer.writerow([rival.keyword.value , rival.uv.value]) 50 | 51 | # # print('\n\n') 52 | # # writer.writerow([rival.keyword.value , rival.uv.value]) 53 | 54 | # with open(date + '_' + str(rival2Id) + '_stream_key_words.csv', 'w+', newline='') as csvfile: 55 | # for rival in rival2: 56 | # print(rival.keyword.value + "," + str(rival.uv.value)) 57 | 58 | # writer = csv.writer(csvfile) 59 | # writer.writerow([rival.keyword.value , rival.uv.value]) 60 | 61 | 62 | 63 | def main(): 64 | begin = datetime.date(2018,7,1) 65 | end = datetime.date.today() 66 | for i in range((end - begin).days): 67 | day = begin + datetime.timedelta(days=i) 68 | # fetchStreamTopKeywords(str(day)) 69 | 70 | if __name__ == '__main__': 71 | main() 72 | 73 | --------------------------------------------------------------------------------