├── .gitignore
├── LICENSE
├── Makefile
├── README.md
├── README_ZH.md
├── example
├── lesson
│ ├── lesson1
│ │ └── lesson1.go
│ ├── lesson2
│ │ └── lesson2.go
│ ├── lesson3
│ │ └── lesson3.go
│ ├── lesson4
│ │ └── lesson4.go
│ └── lesson5
│ │ └── lesson5.go
└── practice
│ ├── helloworld
│ ├── README.md
│ └── main.go
│ ├── pictures
│ ├── README.md
│ └── main.go
│ └── upload
│ ├── README.md
│ ├── client
│ └── main.go
│ └── main.go
├── expert
├── query.go
└── query_test.go
├── go.mod
├── go.sum
├── logo.png
├── miner
├── client.go
├── config.go
├── config
│ └── ua.txt
├── cons.go
├── default.go
├── doc.go
├── log.go
├── multworker.go
├── ua.go
├── util.go
└── worker.go
└── util
├── crypto.go
├── file.go
├── io.go
├── json.go
├── time.go
└── util.go
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | /vendor
3 | /picture
4 | .DS_Store
5 | doc.go.xxxxxx
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: clean fmt
2 |
3 | default: fmt
4 |
5 | clean:
6 | rm -rf bin/*
7 |
8 | fmt:
9 | go fmt ./
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Project: Marmot | HTTP Download
2 |
3 | [](https://github.com/hunterhug/marmot/network)
4 | [](https://github.com/hunterhug/marmot/stargazers)
5 | [](https://github.com/hunterhug/marmot)
6 | [](https://goreportcard.com/report/github.com/hunterhug/marmot)
7 | [](https://github.com/hunterhug/marmot/issues)
8 | [](https://www.apache.org/licenses/LICENSE-2.0.html)
9 |
10 | [中文介绍](/README_ZH.md)
11 |
12 | HTTP Download Helper, Supports Many Features such as Cookie Persistence, HTTP(S) and SOCKS5 Proxy....
13 |
14 | 
15 |
16 | ## 1. Introduction
17 |
18 | World-Wide-Web robot, also known as spiders and crawlers. The principle is to falsify network data by constructing appointed HTTP protocol data packet, then request resource to the specified host, goal is to access the data returned.
19 | There are a large number of web information, human's hand movement such as `copy-paste data` from web page is `time-consuming` and `laborious`, thus inspired the data acquisition industry.
20 |
21 | Batch access to public network data does not break the law, but because there is no difference, no control, very violent means will lead to other services is not stable, therefore, most of the resources provider will filtering some data packets(falsify),
22 | in this context, batch small data acquisition has become a problem. Integrated with various requirements, such as various API development, automated software testing(all this have similar technical principle). So this project come into the world(very simple).
23 |
24 | The `Marmot` is very easy to understand, just like Python's library `requests`(Not yet Smile~ --| ). By enhancing native Golang HTTP library, help you deal with some trivial logic (such as collecting information, checking parameters), and add some fault-tolerant mechanisms (such as add lock, close time flow, ensure the high concurrent run without accident).
25 | It provides a human friendly API interface, you can reuse it often. Very convenient to support `Cookie Persistence`, `Crawler Proxy Settings`, as well as others general settings, such as `HTTP request header settings, timeout/pause settings, data upload/post settings`.
26 | It supports all the HTTP methods `POST/PUT/GET/DELETE/...` and has built-in spider pool and browser UA pool, easy to develop UA+Cookie persistence distributed spider.
27 |
28 | The library is simple and practical, just a few lines of code to replace the previous `Spaghetti code`, has been applied in some large projects.
29 |
30 | The main uses: `WeChat development`/ `API docking` / `Automated test` / `Rush Ticket Scripting` / `Vote Plug-in` / `Data Crawling`
31 |
32 | Now We support Default Worker, You can easy use:
33 |
34 | `lesson1.go`
35 |
36 | ```go
37 | package main
38 |
39 | import (
40 | "fmt"
41 | "github.com/hunterhug/marmot/miner"
42 | )
43 |
44 | func main() {
45 | miner.SetLogLevel(miner.DEBUG)
46 |
47 | // Use Default Worker, You can Also New One:
48 | //worker, _ := miner.New(nil)
49 | //worker = miner.NewWorkerWithNoProxy()
50 | //worker = miner.NewAPI()
51 | //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080")
52 | worker := miner.Clone()
53 | _, err := worker.SetUrl("https://www.bing.com").Go()
54 | if err != nil {
55 | fmt.Println(err.Error())
56 | } else {
57 | fmt.Println(worker.ToString())
58 | }
59 | }
60 | ```
61 |
62 | See the [example](example) dir. such as lesson or practice.
63 |
64 | ## 2. How To Use
65 |
66 | You can get it by:
67 |
68 | ```
69 | go get -v github.com/hunterhug/marmot/miner
70 | ```
71 |
72 | ### 2.1 The First Step
73 |
74 | There are four kinds of worker:
75 |
76 | 1. `worker, err := miner.NewWorker("http://xx:xxxx@103.118.111.26:808") ` // proxy worker, format: `protocol://user(optional):password(optional)@ip:port`, alias to`New()`, support http(s), socks5
77 | 2. `worker, err := miner.NewWorker(nil)` // normal worker, default keep Cookie, alias to `New()`
78 | 3. `worker := miner.NewAPI()` // API worker, will not keep Cookie
79 | 4. `worker, err := miner.NewWorkerByClient(&http.Client{})` // You can also pass a `http.Client` if you want
80 |
81 | if you want to use worker twice, you can call `Clone()` method to clone a new worker, it can isolate the request and response of http, otherwise, you should deal concurrent program carefully.
82 |
83 | ### 2.2 The Second Step
84 |
85 | Camouflage our worker:
86 |
87 | 1. `worker.SetUrl("https://www.bing.com")` // required: set url you want to
88 | 2. `worker.SetMethod(miner.GET)` // optional: set http method `POST/GET/PUT/POSTJSON` and so on
89 | 3. `worker.SetWaitTime(2)` // optional: set timeout of http request
90 | 4. `worker.SetUa(miner.RandomUa())` // optional: set http browser user agent, you can see miner/config/ua.txt
91 | 5. `worker.SetRefer("https://www.bing.com")` // optional: set http request Refer
92 | 6. `worker.SetHeaderParam("diyheader", "diy")` // optional: set http diy header
93 | 7. `worker.SetBData([]byte("file data"))` // optional: set binary data for post or put
94 | 8. `worker.SetFormParam("username","hunterhug")` // optional: set form data for post or put
95 | 9. `worker.SetCookie("xx=dddd")` // optional: you can set a init cookie, some website you can login and F12 copy the cookie
96 | 10. `worker.SetCookieByFile("/root/cookie.txt")` // optional: set cookie which store in a file
97 |
98 | ### 2.3 The Third Step
99 |
100 | Run our worker:
101 |
102 | 1. `body, err := worker.Go()` // if you use SetMethod(), auto use following ways, otherwise use Get()
103 | 2. `body, err := worker.Get()` // default
104 | 3. `body, err := worker.Post()` // post form request, data fill by SetFormParam()
105 | 4. `body, err := worker.PostJSON()` // post JSON request, data fill by SetBData()
106 | 5. `body, err := worker.PostXML()` // post XML request, data fill by SetBData()
107 | 6. `body, err := worker.PostFILE()` // upload file, data fill by SetBData(), and should set SetFileInfo(fileName, fileFormName string)
108 | 7. `body, err := worker.Delete()` // you know!
109 | 8. `body, err := worker.Put()` // ones http method...
110 | 9. `body, err := worker.PutJSON()` // put JSON request
111 | 10. `body, err := worker.PutXML()`
112 | 11. `body, err := worker.PutFILE()`
113 | 12. `body, err := worker.OtherGo("OPTIONS", "application/x-www-form-urlencoded")` // Other http method, Such as OPTIONS etc., can not sent binary.
114 | 13. `body, err := worker.OtherGoBinary("OPTIONS", "application/x-www-form-urlencoded")` // Other http method, Such as OPTIONS etc., just sent binary.
115 | 14. `body, err := worker.GoByMethod("POST")` // you can override SetMethod() By this, equal SetMethod() then Go()
116 |
117 | ### 2.4 The Fourth Step
118 |
119 | Deal the return data, all data will be return as binary, You can immediately store it into a new variable:
120 |
121 | 1. `fmt.Println(string(html))` // type change directly
122 | 2. `fmt.Println(worker.ToString())` // use spider method, after http response, data will keep in the field `Raw`, just use ToString
123 | 3. `fmt.Println(worker.JsonToString())` // some json data will include chinese and other multibyte character, such as `我爱你,我的小绵羊`,`사랑해`
124 |
125 | Attention:
126 |
127 | After every request for an url, the next request you should cover your http request header, otherwise http header you set still exist,
128 | if just want clear post data, use `Clear()`, and want clear HTTP header too please use `ClearAll()`, but I suggest use `Clone()` to avoid this.
129 |
130 | ### 2.5 Other
131 |
132 | Hook:
133 |
134 | 1. `SetBeforeAction(fc func(context.Context, *Worker))`
135 | 2. `SetAfterAction(fc func(context.Context, *Worker))`
136 |
137 | # License
138 |
139 | ```
140 | Copyright [2016-2022] [github.com/hunterhug]
141 |
142 | Licensed under the Apache License, Version 2.0 (the "License");
143 | you may not use this file except in compliance with the License.
144 | You may obtain a copy of the License at
145 |
146 | https://www.apache.org/licenses/LICENSE-2.0
147 |
148 | Unless required by applicable law or agreed to in writing, software
149 | distributed under the License is distributed on an "AS IS" BASIS,
150 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
151 | See the License for the specific language governing permissions and
152 | limitations under the License.
153 | ```
--------------------------------------------------------------------------------
/README_ZH.md:
--------------------------------------------------------------------------------
1 | # 项目代号：土拨鼠
2 |
3 | [](https://github.com/hunterhug/marmot/network)
4 | [](https://github.com/hunterhug/marmot/stargazers)
5 | [](https://github.com/hunterhug/marmot)
6 | [](https://goreportcard.com/report/github.com/hunterhug/marmot)
7 | [](https://github.com/hunterhug/marmot/issues)
8 | [](https://www.apache.org/licenses/LICENSE-2.0.html)
9 |
10 | [English README](/README.md)
11 |
12 | 
13 |
14 | >万维网网络机器人，又称蜘蛛，爬虫，原理主要是通过构造符合HTTP协议的网络数据包，向指定主机请求资源，获取返回的数据.万维网有大量的公开信息，人力采集数据费时费力，故激发了爬虫的产业化.
15 | 批量获取公开网络数据并不违反，但由于无差别性，无节制，十分暴力的手段会导致对方服务的不稳定，因此，大部分资源提供商对数据包进行了某些过滤，在此背景下，小批量数据获取成为了难题.
16 | 综合各种需求，如各种API对接，自动化测试等原理均一样，故开发了此爬虫库.
17 |
18 | >土拨鼠项目是一个人类友好姿势的代码库，开发采用面向对象的方式，易于理解.通过对Golang原生HTTP库的封装，帮用户处理了一些琐碎逻辑(如收集信息，检测参数)，并加入了一些容错机制(如加锁，及时关闭流)，保证了爬虫高并发的安全.此库提供了大量优美的API接口，复用率高，十分方便地支持Cookie接力，爬虫代理设置，以及一般的HTTP请求设置如头部设置，超时，暂停设置，数据设置等，支持全部的HTTP方法如POST/PUT/GET/DELETE等，内置爬虫池和浏览器UA池，易于开发多UA多Cookie分布式爬虫.
19 |
20 | >该库简单实用，短短几行代码即可取代以往杂乱无章的面包条代码片段，已经应用在某些大项目中。
21 |
22 | >该库主要用途：微信开发/API对接/自动化测试/抢票脚本/网站监控/点赞插件/数据爬取
23 |
24 | ## 一. 下载
25 |
26 | 自己封装的 `Golang` 爬虫下载库，支持各种代理模式和伪装功能，你只需通过该方式获取库：
27 |
28 | ```
29 | go get -v github.com/hunterhug/marmot/miner
30 | ```
31 |
32 | 代码结构：
33 |
34 | ```
35 | ├── miner 核心库（HTTP请求封装）
36 | ├── expert 解析库（HTML解析封装）
37 | ├── example Example示例库
38 | ├── lession 示例
39 | ├── practice 练习
40 | ```
41 |
42 | 以下是几个实战例子：
43 |
44 | 1. [多线程批量抓图片](/example/practice/pictures/README.md)。
45 | 2. [模拟上传文件](/example/practice/upload/README.md)。
46 |
47 | ## 二. 使用
48 |
49 | 此库可模拟上传文件，模拟表单提交，模拟各种各样的操作。
50 |
51 | `lesson1.go`
52 |
53 | ```go
54 | package main
55 |
56 | import (
57 | "fmt"
58 | "github.com/hunterhug/marmot/miner"
59 | )
60 |
61 | func main() {
62 | miner.SetLogLevel(miner.DEBUG)
63 |
64 | // 使用默认的全局矿工，当然你可以自己建一个
65 | //worker, _ := miner.New(nil)
66 | //worker = miner.NewWorkerWithNoProxy()
67 | //worker = miner.NewAPI()
68 | //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080")
69 | worker := miner.Clone()
70 | _, err := worker.SetUrl("https://www.bing.com").Go()
71 | if err != nil {
72 | fmt.Println(err.Error())
73 | } else {
74 | fmt.Println(worker.ToString())
75 | }
76 | }
77 | ```
78 |
79 | 官方部分示例已经合进本库，参见 [example](example) 文件夹。
80 |
81 | ### 第一步
82 |
83 | 矿工有四种类型：
84 |
85 | 1. `miner.NewWorker("http://user:password@103.118.111.26:808") ` // 代理矿工，默认自动化Cookie接力 格式:`协议://代理帐号(可选):代理密码(可选)@ip:port`，支持http(s)，socks5，别名函数 `New()`
86 | 2. `miner.NewWorker(nil)` // 正常矿工，默认自动化Cookie接力，别名函数`New()`
87 | 3. `miner.NewAPI()` // API矿工，默认Cookie不接力，主要用来对接服务端 API
88 | 4. `miner.NewWorkerByClient(&http.Client{})` // 可自定义客户端
89 |
90 | 如果你想使用一个矿工多次，使用前请先调用 `Clone()` 方法，它会创建新的矿工来进行 HTTP 请求的数据隔离，避免你并发调用多次导致的混乱。
91 |
92 | ### 第二步
93 |
94 | 模拟矿工设置头部:
95 |
96 | 1. `worker.SetUrl("https://www.bing.com")` // 设置HTTP请求要抓取的网址，**必须**
97 | 2. `worker.SetMethod(miner.GET)` // 设置HTTP请求的方法:`POST/GET/PUT/POSTJSON`等
98 | 3. `worker.SetWaitTime(2)` // 设置HTTP请求超时时间
99 | 4. `worker.SetUa(miner.RandomUa())` // 设置HTTP请求浏览器标志，本项目提供445个浏览器标志，可选设置
100 | 5. `worker.SetRefer("http://www.baidu.com")` // 设置HTTP请求Refer头
101 | 6. `worker.SetHeaderParam("diyheader", "diy")` // 设置HTTP请求自定义头部
102 | 7. `worker.SetBData([]byte("file data"))` // HTTP请求需要上传数据
103 | 8. `worker.SetFormParam("username","hunterhug")` // HTTP请求需要提交表单
104 | 9. `worker.SetCookie("xx=dddd")` // HTTP请求设置cookie，某些网站需要登录后F12复制cookie
105 |
106 | ### 第三步
107 |
108 | 矿工启动方式有：
109 |
110 | 1. `body, err := worker.Go()` // 如果设置SetMethod()，会调用下方对应的方法，否则使用Get()
111 | 2. `body, err := worker.Get()` // 默认
112 | 3. `body, err := worker.Post()` // POST表单请求，数据在SetFormParam()
113 | 4. `body, err := worker.PostJSON()` // 提交JSON请求，数据在SetBData()
114 | 5. `body, err := worker.PostXML()` // 提交XML请求，数据在SetBData()
115 | 6. `body, err := worker.PostFILE()` // 提交文件上传请求，文件二进制数据通过SetBData()设置，然后设置SetFileInfo(fileName，fileFormName string) 表明文件名和表单field Name
116 | 7. `body, err := worker.Delete()`
117 | 8. `body, err := worker.Put()`
118 | 9. `body, err := worker.PutJSON()`
119 | 10. `body, err := worker.PutXML()`
120 | 11. `body, err := worker.PutFILE()`
121 | 12. `body, err := worker.OtherGo("OPTIONS", "application/x-www-form-urlencoded")` // 其他自定义的HTTP方法，不能模拟二进制
122 | 13. `body, err := worker.OtherGoBinary("OPTIONS", "application/x-www-form-urlencoded")` // 其他自定义的HTTP方法，模拟二进制
123 | 14. `body, err := worker.GoByMethod("POST")` // 等同于 SetMethod() 然后 Go()
124 |
125 | ### 第四步
126 |
127 | 每次下载会返回 `[]byte`，请自行解析，调试时可以使用以下方法：
128 |
129 | 1. `fmt.Println(string(html))` // 每次抓取后会返回二进制数据，直接类型转化
130 | 2. `fmt.Println(worker.ToString())` // http响应后二进制数据也会保存在矿工对象的Raw字段中，使用ToString可取出来
131 | 3. `fmt.Println(worker.JsonToString())` // 如果获取到的是JSON数据，请采用此方法转义回来，不然字符串会乱码
132 |
133 | 注意：每次下载后，需要使用以下方法将表单等数据重置（我推荐每次使用 `Clone()` 来避免这种做法，也就是克隆一个新的矿工来进行下一次操作）：
134 |
135 | 1. `Clear()` // 清除表单和二进制数据
136 | 2. `ClearAll()` // 清除表单和二进制数据，还清除全部HTTP头部
137 | 3. `ClearCookie()` // 可清除Cookie
138 |
139 | ### 其他
140 |
141 | 勾子:
142 |
143 | 1. `SetBeforeAction(fc func(context.Context, *Worker))` 爬虫动作前可AOP注入。
144 | 2. `SetAfterAction(fc func(context.Context, *Worker))` 爬虫动作完成后。
145 |
146 | # License
147 |
148 | ```
149 | Copyright [2016-2022] [github.com/hunterhug]
150 |
151 | Licensed under the Apache License, Version 2.0 (the "License");
152 | you may not use this file except in compliance with the License.
153 | You may obtain a copy of the License at
154 |
155 | https://www.apache.org/licenses/LICENSE-2.0
156 |
157 | Unless required by applicable law or agreed to in writing, software
158 | distributed under the License is distributed on an "AS IS" BASIS,
159 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
160 | See the License for the specific language governing permissions and
161 | limitations under the License.
162 | ```
--------------------------------------------------------------------------------
/example/lesson/lesson1/lesson1.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "github.com/hunterhug/marmot/miner"
6 | )
7 |
8 | /*
9 | Most Simple: Use Default Worker!
10 | */
11 | func main() {
12 | miner.SetLogLevel(miner.DEBUG)
13 |
14 | // Use Default Worker, You can Also New One:
15 | //worker, _ := miner.New(nil)
16 | //worker = miner.NewWorkerWithNoProxy()
17 | //worker = miner.NewAPI()
18 | //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080")
19 | worker := miner.Clone()
20 | _, err := worker.SetUrl("https://www.bing.com").Go()
21 | if err != nil {
22 | fmt.Println(err.Error())
23 | } else {
24 | fmt.Println(worker.ToString())
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/example/lesson/lesson2/lesson2.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "github.com/hunterhug/marmot/miner"
6 | )
7 |
8 | /*
9 | New A Worker!
10 | */
11 | func main() {
12 | // 1. New a worker
13 | worker, _ := miner.New(nil)
14 |
15 | // 2. Set a URL And Fetch
16 | html, err := worker.SetUrl("https://www.bing.com").SetUa(miner.RandomUa()).SetMethod(miner.GET).Go()
17 | if err != nil {
18 | fmt.Println(err.Error())
19 | }
20 |
21 | // 3.Print content equal to fmt.Println(worker.ToString())
22 | fmt.Println(string(html))
23 | }
24 |
--------------------------------------------------------------------------------
/example/lesson/lesson3/lesson3.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | /*
4 | More detail Example
5 | */
6 | import (
7 | // 1:import package
8 | "github.com/hunterhug/marmot/miner"
9 | "github.com/hunterhug/marmot/util"
10 | )
11 |
12 | func init() {
13 | // 2:Optional global setting
14 | miner.SetLogLevel(miner.DEBUG) // optional, set log to debug
15 | miner.SetGlobalTimeout(3) // optional, http request timeout time
16 |
17 | }
18 |
19 | func main() {
20 | log := miner.Log() // optional, miner log you can choose to use
21 |
22 | // 3: Must new a Worker object, three ways
23 | //worker, err := miner.NewWorker("http://xx:xxxx@103.118.111.26:808") // proxy format: protocol://user(optional):password(optional)@ip:port
24 | //worker, err := miner.NewWorker(nil) // normal worker, default keep Cookie
25 | //worker := miner.NewAPI() // API worker, not keep Cookie
26 | worker, err := miner.New(nil) // NewWorker alias
27 | if err != nil {
28 | panic(err)
29 | }
30 |
31 | // 4: Set the request Method/URL and some others, can chain set, only SetUrl is required.
32 | // SetUrl: required, the Url
33 | // SetMethod: optional, HTTP method: POST/GET/..., default GET
34 | // SetWaitTime: optional, HTTP request wait/pause time
35 | worker.SetUrl("https://www.bing.com").SetMethod(miner.GET).SetWaitTime(2)
36 | worker.SetUa(miner.RandomUa()) // optional, browser user agent: IE/Firefox...
37 | worker.SetRefer("https://www.bing.com") // optional, url refer
38 | worker.SetHeaderParam("diy_header", "diy") // optional, some other diy http header
39 | //worker.SetBData([]byte("file data")) // optional, if you want post JSON data or upload file
40 | //worker.SetFormParam("username","root") // optional: if you want post form
41 | //worker.SetFormParam("password","123")
42 |
43 | // 5: Start Run
44 | //worker.Get() // default GET
45 | //worker.Post() // POST form request data, data can fill by SetFormParam()
46 | //worker.PostJSON() // POST JSON dara, use SetBData()
47 | //worker.PostXML() // POST XML, use SetBData()
48 | //worker.PostFILE() // POST to Upload File, data in SetBData() too
49 | //worker.OtherGo("OPTIONS", "application/x-www-form-urlencoded") // Other http method, Such as OPTIONS etcd
50 | body, err := worker.Go() // if you use SetMethod(), otherwise equal to Get()
51 | if err != nil {
52 | log.Error(err.Error())
53 | } else {
54 | log.Infof("%s", string(body)) // Print return data
55 | }
56 |
57 | log.Debugf("%#v", worker.GetCookies) // if you not set log as debug, it will not appear
58 |
59 | // You must Clear it! If you want to POST Data by SetFormParam()/SetBData() again
60 | // After get the return data by post data, you can clear the data you fill
61 | // I suggest use Clone()
62 | worker.Clear()
63 | //worker.ClearAll() // you can also want to clear all, include http header you set
64 |
65 | // Worker pool for concurrent, every Worker Object is serial as the browser. if you want collateral execution, use this.
66 | miner.Pool.Set("myfirstworker", worker)
67 | if w, ok := miner.Pool.Get("myfirstworker"); ok {
68 | go func() {
69 | data, _ := w.Clone().SetUrl("https://www.bing.com").Get()
70 | log.Info(string(data))
71 | }()
72 | util.Sleep(10)
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/example/lesson/lesson4/lesson4.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "github.com/hunterhug/marmot/util"
7 | "strings"
8 |
9 | "github.com/hunterhug/marmot/expert"
10 | "github.com/hunterhug/marmot/miner"
11 | )
12 |
13 | func main() {
14 | // We can debug, to see whether SetBeforeAction make sense
15 | miner.SetLogLevel(miner.DEBUG)
16 |
17 | // The url we want
18 | url := "https://www.bing.com"
19 |
20 | // IAM we can NewAPI
21 | worker := miner.NewAPI()
22 |
23 | // We can aop by context
24 | ctx, cancel := context.WithCancel(context.Background())
25 | //ctx := context.TODO()
26 | worker.SetContext(ctx)
27 |
28 | // we cancel it after 5 second
29 | go func() {
30 | fmt.Println("I stop and sleep 5")
31 | util.Sleep(5)
32 | fmt.Println("I wake up after sleep 5")
33 | cancel()
34 | }()
35 |
36 | // Before we make some change, And every GET Or POST it will action
37 | worker.SetBeforeAction(func(ctx context.Context, this *miner.Worker) {
38 | fmt.Println("Before Action, I will add a HTTP header, then sleep wait cancel")
39 | this.SetHeaderParam("Marmot", "v2")
40 | this.SetHeaderParam("DUDUDUU", "DUDU")
41 | select {
42 | case <-ctx.Done(): // block in here util cancel()
43 | //fmt.Println(ctx.Err())
44 | fmt.Println("after sleep, i do action.")
45 | }
46 | })
47 |
48 | worker.SetAfterAction(func(ctx context.Context, this *miner.Worker) {
49 | fmt.Println("After Action, I just print this sentence")
50 | })
51 |
52 | // Let's Go
53 | body, err := worker.SetUrl(url).GoByMethod(miner.GET)
54 | if err != nil {
55 | fmt.Println(err.Error())
56 | } else {
57 | // Parse We want
58 | fmt.Printf("Output:\n %s\n", MyParse(body))
59 | }
60 |
61 | }
62 |
63 | // MyParse Parse HTML page
64 | func MyParse(data []byte) string {
65 | doc, err := expert.QueryBytes(data)
66 | if err != nil {
67 | fmt.Println(err.Error())
68 | }
69 | return strings.TrimSpace(doc.Find("title").Text())
70 | }
71 |
--------------------------------------------------------------------------------
/example/lesson/lesson5/lesson5.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | /*
4 | Proxy Worker!
5 | You first should own a remote machine, Then in your local tap:
6 | `ssh -ND 1080 ubuntu@remoteIp`
7 | It will generate socks5 proxy client in your local, which port is 1080
8 | */
9 | import (
10 | "fmt"
11 | "os"
12 |
13 | "github.com/hunterhug/marmot/expert"
14 | "github.com/hunterhug/marmot/miner"
15 | )
16 |
17 | func init() {
18 | miner.SetLogLevel(miner.DEBUG)
19 | }
20 |
21 | func main() {
22 | // You can use a lot of proxy ip such "https/http/socks5"
23 | proxyIp := "socks5://127.0.0.1:1080"
24 |
25 | url := "https://www.google.com"
26 |
27 | worker, err := miner.New(proxyIp)
28 | if err != nil {
29 | fmt.Println(err.Error())
30 | os.Exit(1)
31 | }
32 |
33 | body, err := worker.SetUa(miner.RandomUa()).SetUrl(url).SetMethod(miner.GET).Go()
34 | if err != nil {
35 | fmt.Println(err.Error())
36 | } else {
37 | fmt.Println(parse(body))
38 | }
39 | }
40 |
41 | // Parse HTML page
42 | func parse(data []byte) string {
43 | doc, err := expert.QueryBytes(data)
44 | if err != nil {
45 | fmt.Println(err.Error())
46 | return ""
47 | }
48 | return doc.Find("title").Text()
49 | }
50 |
--------------------------------------------------------------------------------
/example/practice/helloworld/README.md:
--------------------------------------------------------------------------------
1 | # 简单示例
2 |
3 | 1. simple.go 最简单的一个
--------------------------------------------------------------------------------
/example/practice/helloworld/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | // 第一步：引入库
5 | "github.com/hunterhug/marmot/miner"
6 | "github.com/hunterhug/marmot/util"
7 | )
8 |
9 | func init() {
10 | // 第二步：可选设置全局
11 | miner.SetLogLevel(miner.DEBUG) // 设置全局矿工日志,可不设置,设置debug可打印出http请求轨迹
12 | miner.SetGlobalTimeout(3) // 矿工超时时间,可不设置
13 |
14 | }
15 |
16 | func main() {
17 |
18 | log := miner.Log() // 矿工为你提供的日志工具,可不用
19 |
20 | // 第三步： 必须新建一个矿工对象
21 | // 代理IP格式: 协议://代理帐号(可选):代理密码(可选)@ip:port
22 | //worker, err := miner.NewWorker("http://xx:xxxx@103.118.111.26:808")
23 | //worker, err := miner.NewWorker(nil) // 正常矿工 默认带Cookie
24 | //worker := miner.NewAPI() // API矿工 默认不带Cookie
25 | worker, err := miner.New(nil) // NewWorker同名函数
26 | if err != nil {
27 | panic(err)
28 | }
29 |
30 | // 第四步：设置抓取方式和网站,可链式结构设置,只有SetUrl是必须的
31 | // SetUrl:Url必须设置
32 | // SetMethod:HTTP方法可以是POST或GET,可不设置,默认GET,传错值默认为GET
33 | // SetWaitTime:暂停时间,可不设置,默认不暂停
34 | worker.SetUrl("https://www.bing.com").SetMethod(miner.GET).SetWaitTime(2)
35 | worker.SetUa(miner.RandomUa()) //设置随机浏览器标志
36 | worker.SetRefer("https://www.bing.com") // 设置Refer头
37 | worker.SetHeaderParam("diyheader", "diy") // 自定义头部
38 | //worker.SetBData([]byte("file data")) // 如果你要提交JSON数据/上传文件
39 | //worker.SetFormParam("username","hunterhug") // 提交表单
40 | //worker.SetFormParam("password","123")
41 |
42 | // 第五步：开始爬
43 | //worker.Get() // 默认GET
44 | //worker.Post() // POST表单请求,数据在SetFormParam()
45 | //worker.PostJSON() // 提交JSON请求,数据在SetBData()
46 | //worker.PostXML() // 提交XML请求,数据在SetBData()
47 | //worker.PostFILE() // 提交文件上传请求,数据在SetBData()
48 | body, err := worker.Go() // 如果设置SetMethod(),采用,否则Get()
49 | if err != nil {
50 | log.Error(err.Error())
51 | } else {
52 | log.Infof("%s", string(body)) // 打印获取的数据
53 | }
54 |
55 | log.Debugf("%#v", worker.GetCookies()) // 不设置全局log为debug是不会出现这个东西的
56 |
57 | // 爬取完毕后可以清除POST的表单数据/文件数据/JSON数据
58 | // 我建议使用 Clone()，这样也可以不用清除
59 | worker.Clear()
60 |
61 | // 爬取完毕后可以清除设置的Http头部和POST的表单数据/文件数据/JSON数据
62 | //worker.ClearAll()
63 |
64 | // 矿工池子
65 | miner.Pool.Set("worker1", worker)
66 | if w, ok := miner.Pool.Get("worker1"); ok {
67 | go func() {
68 | data, _ := w.Clone().SetUrl("https://www.bing.com").Get()
69 | log.Info(string(data))
70 | }()
71 | util.Sleep(10)
72 | }
73 |
74 | }
75 |
--------------------------------------------------------------------------------
/example/practice/pictures/README.md:
--------------------------------------------------------------------------------
1 | # 多线程批量抓图片
2 |
3 | 直接运行程序即可。
4 |
5 | ```go
6 | go run main.go
7 |
8 | Welcome: Input "url" and picture keep "dir"
9 | ---------------------------------------------
10 | URL(Like: "https://www.bizhizj.com")
11 |
12 | DIR(Default: "./picture")
13 | ```
--------------------------------------------------------------------------------
/example/practice/pictures/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "github.com/hunterhug/marmot/expert"
7 | "github.com/hunterhug/marmot/miner"
8 | "github.com/hunterhug/marmot/util"
9 | "net/url"
10 | "strings"
11 | )
12 |
13 | // MinerNum Num of miner, We can run it at the same time to crawl data fast
14 | var MinerNum = 5
15 |
16 | // ProxyAddress You can update this decides whether to proxy
17 | var ProxyAddress interface{}
18 |
19 | func main() {
20 | // You can Proxy!
21 | // ProxyAddress = "socks5://127.0.0.1:1080"
22 |
23 | fmt.Println(`Welcome: Input "url" and picture keep "dir"`)
24 | fmt.Println("---------------------------------------------")
25 | urlPicture := util.Input(`URL(Like: "https://www.bizhizj.com")`, "https://www.bizhizj.com/dongman")
26 | dir := util.Input(`DIR(Default: "./picture")`, "./picture")
27 | fmt.Printf("You will keep %s picture in dir %s\n", urlPicture, dir)
28 | fmt.Println("---------------------------------------------")
29 |
30 | // Start Catch
31 | err := DownloadHTMLPictures(urlPicture, dir, MinerNum, ProxyAddress)
32 | if err != nil {
33 | fmt.Println("Error:" + err.Error())
34 | }
35 | }
36 |
37 | // DownloadHTMLPictures Download one HTML page's all pictures
38 | // @URL: http://image.baidu.com
39 | // @SaveDir /home/images
40 | // @ProxyAddress : "socks5://127.0.0.1:1080"
41 | func DownloadHTMLPictures(URL string, SaveDir string, MinerNum int, ProxyAddress interface{}) error {
42 |
43 | // Check valid
44 | _, err := url.Parse(URL)
45 | if err != nil {
46 | return err
47 | }
48 |
49 | // New a worker to get url
50 | worker, err := miner.New(ProxyAddress)
51 | if err != nil {
52 | return err
53 | }
54 |
55 | result, err := worker.SetUrl(URL).SetUa(miner.RandomUa()).Get()
56 | if err != nil {
57 | return err
58 | }
59 |
60 | // Find all picture
61 | pictures := expert.FindPicture(string(result))
62 |
63 | return DownloadURLPictures(pictures, SaveDir, MinerNum, worker)
64 | }
65 |
66 | // DownloadURLPictures Download pictures faster!
67 | func DownloadURLPictures(PictureUrls []string, SaveDir string, MinerNum int, initWorker *miner.Worker) error {
68 | // Empty, What a pity!
69 | if len(PictureUrls) == 0 {
70 | return errors.New("empty")
71 | }
72 |
73 | // Make dir!
74 | err := util.MakeDir(SaveDir)
75 | if err != nil {
76 | return err
77 | }
78 |
79 | // Divide pictures into several worker
80 | xxx, _ := util.DivideStringList(PictureUrls, MinerNum)
81 |
82 | // Chanel to info exchange
83 | chs := make(chan int, len(PictureUrls))
84 |
85 | // Go at the same time
86 | for num, pictureList := range xxx {
87 | // Clone new worker
88 | workerPicture := initWorker.Clone()
89 | workerPicture.SetUa(miner.RandomUa())
90 |
91 | // Go save picture!
92 | go func(images []string, worker *miner.Worker, num int) {
93 | for _, img := range images {
94 |
95 | // Check, May be Pass
96 | _, err := url.Parse(img)
97 | if err != nil {
98 | continue
99 | }
100 |
101 | // Change Name of our picture
102 | filename := strings.Replace(util.ValidFileName(img), "#", "_", -1)
103 |
104 | // Exist?
105 | if util.FileExist(SaveDir + "/" + filename) {
106 | fmt.Println("File Exist：" + SaveDir + "/" + filename)
107 | chs <- 0
108 | } else {
109 |
110 | // Not Exsit?
111 | imgsrc, e := worker.SetUrl(img).Get()
112 | if e != nil {
113 | fmt.Println("Download " + img + " error:" + e.Error())
114 | chs <- 0
115 | return
116 | }
117 |
118 | // Save it!
119 | e = util.SaveToFile(SaveDir+"/"+filename, imgsrc)
120 | if e == nil {
121 | fmt.Printf("SP%d: Keep in %s/%s\n", num, SaveDir, filename)
122 | }
123 | chs <- 1
124 | }
125 | }
126 | }(pictureList, workerPicture, num)
127 | }
128 |
129 | // Every picture should return
130 | for i := 0; i < len(PictureUrls); i++ {
131 | <-chs
132 | }
133 |
134 | return nil
135 | }
136 |
--------------------------------------------------------------------------------
/example/practice/upload/README.md:
--------------------------------------------------------------------------------
1 | # 模拟上传文件
2 |
3 | ## 启动服务端
4 |
5 | ```
6 | go run main.go
7 | ```
8 |
9 | 打开 127.0.0.1:1789 可以看到上传文件的界面。
10 |
11 | 但我们使用程序模拟上传。
12 |
13 | ## 启动模拟上传程序
14 |
15 | ```
16 | cd client
17 | go run main.go
18 | ```
19 |
20 | 将会看到出现一个 `doc.go.xxxx` 的文件出现在目录下。
--------------------------------------------------------------------------------
/example/practice/upload/client/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "github.com/hunterhug/marmot/miner"
6 | )
7 |
8 | func postFile(filename string, targetUrl string) {
9 | worker, _ := miner.New(nil)
10 | result, err := worker.SetUrl(targetUrl).SetBData([]byte("dddd")).SetFileInfo(filename+".xxxxxx", "uploadfile").SetFormParam("xxxx", "xxx").PostFILE()
11 | if err != nil {
12 | fmt.Println(err.Error())
13 | } else {
14 | fmt.Println(string(result))
15 | }
16 | }
17 |
18 | // sample usage
19 | func main() {
20 | targetUrl := "http://127.0.0.1:1789/upload"
21 | filename := "./doc.go"
22 | postFile(filename, targetUrl)
23 | }
24 |
--------------------------------------------------------------------------------
/example/practice/upload/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "net/http"
7 | "os"
8 | )
9 |
10 | func main() {
11 | fmt.Println("max upload size", 32<<20)
12 | fmt.Println("web url: 127.0.0.1:1789")
13 | http.HandleFunc("/", index)
14 | http.HandleFunc("/upload", upload)
15 | http.ListenAndServe(":1789", nil)
16 | }
17 |
18 | func upload(w http.ResponseWriter, r *http.Request) {
19 | r.ParseMultipartForm(32 << 20)
20 | file, handler, err := r.FormFile("uploadfile")
21 | if err != nil {
22 | fmt.Println(err)
23 | return
24 | }
25 | defer file.Close()
26 | f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
27 | if err != nil {
28 | fmt.Println(err)
29 | return
30 | }
31 | defer f.Close()
32 | io.Copy(f, file)
33 | fmt.Fprintln(w, "upload ok!")
34 | }
35 |
36 | func index(w http.ResponseWriter, r *http.Request) {
37 | w.Write([]byte(tpl))
38 | }
39 |
40 | const tpl = `
41 |
42 | 上传文件
43 |
44 |
45 |
50 |
51 | `
52 |
--------------------------------------------------------------------------------
/expert/query.go:
--------------------------------------------------------------------------------
1 | package expert
2 |
3 | // Package expert is use to parse content
4 | import (
5 | "github.com/PuerkitoBio/goquery"
6 | "regexp"
7 | "strings"
8 | )
9 |
10 | func QueryBytes(content []byte) (*goquery.Document, error) {
11 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(content)))
12 | return doc, err
13 | }
14 |
15 | func QueryString(content string) (*goquery.Document, error) {
16 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
17 | return doc, err
18 | }
19 |
20 | // FindPicture Find All picture. Must prefix with http(s)
21 | func FindPicture(s string) []string {
22 | picList := make([]string, 0)
23 | re, _ := regexp.Compile(`src\s*=\s*["'](http[s]?:\/\/.*?\.(jpg|jpeg|png|gif))["']`)
24 | output := re.FindAllStringSubmatch(s, -1)
25 | for _, o := range output {
26 | picList = append(picList, o[1])
27 | }
28 | return picList
29 | }
30 |
--------------------------------------------------------------------------------
/expert/query_test.go:
--------------------------------------------------------------------------------
1 | package expert
2 |
3 | import (
4 | "fmt"
5 | "testing"
6 | )
7 |
8 | func TestFindPicture(t *testing.T) {
9 | data := `
10 |
11 | https://imgsa.baidu.com/forum/w%3D580/sign=294db374d462853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpg
12 |
13 | https://imgsa.baidu.com/forum/whttps:/4d462853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpg
14 | https://img1.jpg
15 | http://im62853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpgsfsadfsda
16 | httpdb374://aafa40f0611.jpg
17 |
18 | src="http://s.jpg"
19 |
20 | src="https://s.jpg"
21 |
22 | src = "http://s.jpg"
23 | src="https:s.jpg"
24 | "https://img1.jpg" "https://img1.jpgsss","https://img1.jpgss","https://img1.jpgss"
25 |
26 | src="https://up.enterdesk.com/edpic_360_360/bf/53/52/bf5352f28a839e01dd3fcb811669102c.jpg"
27 | `
28 | result := FindPicture(data)
29 |
30 | fmt.Printf("%#v", result)
31 | }
32 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/hunterhug/marmot
2 |
3 | go 1.19
4 |
5 | require (
6 | github.com/PuerkitoBio/goquery v1.8.0
7 | github.com/hunterhug/golog v1.0.3
8 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
9 | )
10 |
11 | require (
12 | github.com/andybalholm/cascadia v1.3.1 // indirect
13 | github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect
14 | github.com/lestrrat-go/strftime v1.0.6 // indirect
15 | github.com/pkg/errors v0.9.1 // indirect
16 | go.uber.org/atomic v1.10.0 // indirect
17 | go.uber.org/multierr v1.8.0 // indirect
18 | go.uber.org/zap v1.23.0 // indirect
19 | )
20 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
2 | github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
3 | github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
4 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
5 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9 | github.com/hunterhug/golog v1.0.3 h1:GoHnqdhssW5CuXky5j8gaYxeIOEdxXap001LzYBfyrA=
10 | github.com/hunterhug/golog v1.0.3/go.mod h1:tW9dEcCYbQIdLRC2orY7lbZktWV9+COY0pH2V7nzdUM=
11 | github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
12 | github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
13 | github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
14 | github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
15 | github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
16 | github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
17 | github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
18 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
19 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
20 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
21 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
23 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
24 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
25 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
26 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
27 | go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
28 | go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
29 | go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
30 | go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
31 | go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
32 | go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
33 | go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
34 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
35 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
36 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
37 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
38 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
39 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
40 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
41 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
42 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
43 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
44 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
45 |
--------------------------------------------------------------------------------
/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hunterhug/marmot/cea47e15666542faca08210eea6a83e0f408791c/logo.png
--------------------------------------------------------------------------------
/miner/client.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "crypto/tls"
5 | "errors"
6 | "golang.org/x/net/proxy"
7 | "net/http"
8 | "net/http/cookiejar"
9 | "net/url"
10 | "strings"
11 |
12 | "github.com/hunterhug/marmot/util"
13 | )
14 |
15 | // NewJar Cookie record Jar
16 | func NewJar() *cookiejar.Jar {
17 | cookieJar, _ := cookiejar.New(nil)
18 | return cookieJar
19 | }
20 |
21 | // Default Client
22 | var (
23 | // Client Save Cookie
24 | Client = &http.Client{
25 | CheckRedirect: func(req *http.Request, via []*http.Request) error {
26 | Logger.Debugf("[GoWorker] Redirect: %v", req.URL)
27 | return nil
28 | },
29 | Jar: NewJar(),
30 | Timeout: util.Second(DefaultTimeOut),
31 | Transport: &http.Transport{
32 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
33 | },
34 | }
35 |
36 | // NoCookieClient Not Save Cookie
37 | NoCookieClient = &http.Client{
38 | CheckRedirect: func(req *http.Request, via []*http.Request) error {
39 | Logger.Debugf("[GoWorker] Redirect: %v", req.URL)
40 | return nil
41 | },
42 | Timeout: util.Second(DefaultTimeOut),
43 | Transport: &http.Transport{
44 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
45 | },
46 | }
47 | )
48 |
49 | // NewProxyClient New a Proxy client, Default save cookie, Can timeout
50 | // We should support some proxy way such as http(s) or socks
51 | func NewProxyClient(proxyString string) (*http.Client, error) {
52 | proxyUrl, err := url.Parse(proxyString)
53 | if err != nil {
54 | return nil, err
55 | }
56 |
57 | prefix := strings.Split(proxyString, ":")[0]
58 |
59 | // setup a http transport
60 | httpTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
61 |
62 | // http://
63 | // https://
64 | // socks5://
65 | switch prefix {
66 | case "http", "https":
67 | httpTransport.Proxy = http.ProxyURL(proxyUrl)
68 | case "socks5":
69 | // create a socks5 dialer
70 | dialer, err := proxy.FromURL(proxyUrl, proxy.Direct)
71 | if err != nil {
72 | return nil, err
73 | }
74 | httpTransport.Dial = dialer.Dial
75 | default:
76 | return nil, errors.New("this proxy way not allow:" + prefix)
77 | }
78 |
79 | // This alone client, diff from global client.
80 | client := &http.Client{
81 | // Allow redirect
82 | CheckRedirect: func(req *http.Request, via []*http.Request) error {
83 | Logger.Debugf("[GoWorker] Redirect: %v", req.URL)
84 | return nil
85 | },
86 | // Allow proxy: http, https, socks5
87 | Transport: httpTransport,
88 | // Allow keep cookie
89 | Jar: NewJar(),
90 | // Allow Timeout
91 | Timeout: util.Second(DefaultTimeOut),
92 | }
93 | return client, nil
94 | }
95 |
96 | // NewClient New a client, diff from proxy client
97 | func NewClient() *http.Client {
98 | client := &http.Client{
99 | // Allow redirect
100 | CheckRedirect: func(req *http.Request, via []*http.Request) error {
101 | Logger.Debugf("[GoWorker] Redirect: %v", req.URL)
102 | return nil
103 | },
104 | Jar: NewJar(),
105 | Timeout: util.Second(DefaultTimeOut),
106 | Transport: &http.Transport{
107 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
108 | },
109 | }
110 |
111 | return client
112 | }
113 |
--------------------------------------------------------------------------------
/miner/config.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "context"
5 | "net/http"
6 | "net/url"
7 | "strings"
8 | "sync"
9 |
10 | "github.com/hunterhug/marmot/util"
11 | )
12 |
13 | // Worker is the main object to sent http request and return result of response
14 | type Worker struct {
15 | // In order fast chain func call I put the basic config below
16 | *Request
17 | *Response
18 |
19 | // Which url we want
20 | Url string
21 |
22 | // Get,Post method
23 | Method string
24 |
25 | // Our Client
26 | Client *http.Client
27 |
28 | // Wait Sleep Time
29 | Wait int
30 |
31 | // Worker proxy ip, just for user to record their proxy ip, default: localhost
32 | Ip string
33 |
34 | // AOP like Java
35 | Ctx context.Context
36 | BeforeAction func(context.Context, *Worker)
37 | AfterAction func(context.Context, *Worker)
38 |
39 | // Http header
40 | Header http.Header
41 |
42 | // Mux lock
43 | mux sync.RWMutex
44 | }
45 |
46 | type Request struct {
47 | Data url.Values // Sent by form data
48 | FileName string // FileName which sent to remote
49 | FileFormName string // File Form Name which sent to remote
50 | BData []byte // Sent by binary data, can together with File
51 | Request *http.Request // Debug
52 | }
53 |
54 | type Response struct {
55 | Response *http.Response // Debug
56 | Raw []byte // Raw data we get
57 | ResponseStatusCode int // The last url response code, such as 404
58 | }
59 |
60 | // SetHeader Java Bean Chain pattern
61 | func (worker *Worker) SetHeader(header http.Header) *Worker {
62 | worker.Header = header
63 | return worker
64 | }
65 |
66 | // SetHeader Default Worker SetHeader!
67 | func SetHeader(header http.Header) *Worker {
68 | return DefaultWorker.SetHeader(header)
69 | }
70 |
71 | func (worker *Worker) SetHeaderParam(k, v string) *Worker {
72 | worker.Header.Set(k, v)
73 | return worker
74 | }
75 |
76 | func SetHeaderParam(k, v string) *Worker {
77 | return DefaultWorker.SetHeaderParam(k, v)
78 | }
79 |
80 | func (worker *Worker) SetCookie(v string) *Worker {
81 | worker.SetHeaderParam("Cookie", v)
82 | return worker
83 | }
84 |
85 | func SetCookie(v string) *Worker {
86 | return DefaultWorker.SetCookie(v)
87 | }
88 |
89 | // SetCookieByFile Set Cookie by file.
90 | func (worker *Worker) SetCookieByFile(file string) (*Worker, error) {
91 | haha, err := util.ReadFromFile(file)
92 | if err != nil {
93 | return nil, err
94 | }
95 | cookie := string(haha)
96 | cookie = strings.Replace(cookie, " ", "", -1)
97 | cookie = strings.Replace(cookie, "\n", "", -1)
98 | cookie = strings.Replace(cookie, "\r", "", -1)
99 | return worker.SetCookie(cookie), nil
100 | }
101 |
102 | func SetCookieByFile(file string) (*Worker, error) {
103 | return DefaultWorker.SetCookieByFile(file)
104 | }
105 |
106 | func (worker *Worker) SetUa(ua string) *Worker {
107 | worker.Header.Set("User-Agent", ua)
108 | return worker
109 | }
110 |
111 | func SetUa(ua string) *Worker {
112 | return DefaultWorker.SetUa(ua)
113 | }
114 |
115 | func (worker *Worker) SetRefer(refer string) *Worker {
116 | worker.Header.Set("Referer", refer)
117 | return worker
118 | }
119 |
120 | func SetRefer(refer string) *Worker {
121 | return DefaultWorker.SetRefer(refer)
122 | }
123 |
124 | func (worker *Worker) SetHost(host string) *Worker {
125 | worker.Header.Set("Host", host)
126 | return worker
127 | }
128 |
129 | // SetUrl at the same time SetHost
130 | func (worker *Worker) SetUrl(url string) *Worker {
131 | worker.Url = url
132 | temp := strings.Split(url, "//")
133 | if len(temp) >= 2 {
134 | worker.SetHost(strings.Split(temp[1], "/")[0])
135 | }
136 | return worker
137 | }
138 |
139 | func SetUrl(url string) *Worker {
140 | return DefaultWorker.SetUrl(url)
141 | }
142 |
143 | func (worker *Worker) SetFileInfo(fileName, fileFormName string) *Worker {
144 | worker.FileName = fileName
145 | worker.FileFormName = fileFormName
146 | return worker
147 | }
148 |
149 | func SetFileInfo(fileName, fileFormName string) *Worker {
150 | return DefaultWorker.SetFileInfo(fileName, fileFormName)
151 | }
152 |
153 | func (worker *Worker) SetMethod(method string) *Worker {
154 | temp := GET
155 | switch strings.ToUpper(method) {
156 | case GET:
157 | temp = GET
158 | case POST:
159 | temp = POST
160 | case POSTFILE:
161 | temp = POSTFILE
162 | case POSTJSON:
163 | temp = POSTJSON
164 | case POSTXML:
165 | temp = POSTXML
166 | case PUT:
167 | temp = PUT
168 | case PUTFILE:
169 | temp = PUTFILE
170 | case PUTJSON:
171 | temp = PUTJSON
172 | case PUTXML:
173 | temp = PUTXML
174 | case DELETE:
175 | temp = DELETE
176 | default:
177 | temp = OTHER
178 | }
179 | worker.Method = temp
180 | return worker
181 | }
182 |
183 | func SetMethod(method string) *Worker {
184 | return DefaultWorker.SetMethod(method)
185 | }
186 |
187 | func (worker *Worker) SetWaitTime(num int) *Worker {
188 | if num <= 0 {
189 | num = 1
190 | }
191 | worker.Wait = num
192 | return worker
193 | }
194 |
195 | func SetWaitTime(num int) *Worker {
196 | return DefaultWorker.SetWaitTime(num)
197 | }
198 |
199 | func (worker *Worker) SetBData(data []byte) *Worker {
200 | worker.BData = data
201 | return worker
202 | }
203 |
204 | func SetBData(data []byte) *Worker {
205 | return DefaultWorker.SetBData(data)
206 | }
207 |
208 | func (worker *Worker) SetForm(form url.Values) *Worker {
209 | worker.Data = form
210 | return worker
211 | }
212 |
213 | func SetForm(form url.Values) *Worker {
214 | return DefaultWorker.SetForm(form)
215 | }
216 |
217 | func (worker *Worker) SetFormParam(k, v string) *Worker {
218 | worker.Data.Set(k, v)
219 | return worker
220 | }
221 |
222 | func SetFormParam(k, v string) *Worker {
223 | return DefaultWorker.SetFormParam(k, v)
224 | }
225 |
226 | // SetContext Set Context so Action can soft
227 | func (worker *Worker) SetContext(ctx context.Context) *Worker {
228 | worker.Ctx = ctx
229 | return worker
230 | }
231 |
232 | func SetContext(ctx context.Context) *Worker {
233 | return DefaultWorker.SetContext(ctx)
234 | }
235 |
236 | func (worker *Worker) SetBeforeAction(fc func(context.Context, *Worker)) *Worker {
237 | worker.BeforeAction = fc
238 | return worker
239 | }
240 |
241 | func SetBeforeAction(fc func(context.Context, *Worker)) *Worker {
242 | return DefaultWorker.SetBeforeAction(fc)
243 | }
244 |
245 | func (worker *Worker) SetAfterAction(fc func(context.Context, *Worker)) *Worker {
246 | worker.AfterAction = fc
247 | return worker
248 | }
249 |
250 | func SetAfterAction(fc func(context.Context, *Worker)) *Worker {
251 | return DefaultWorker.SetAfterAction(fc)
252 | }
253 |
254 | // Clear data we sent
255 | // I suggest use Clone() to avoid clear
256 | func (worker *Worker) Clear() *Worker {
257 | worker.Request = newRequest()
258 | worker.Response = new(Response)
259 | return worker
260 | }
261 |
262 | func Clear() *Worker {
263 | return DefaultWorker.Clear()
264 | }
265 |
266 | // ClearAll All clear include header
267 | func (worker *Worker) ClearAll() *Worker {
268 | worker.Clear()
269 | worker.Header = http.Header{}
270 | return worker
271 | }
272 |
273 | func ClearAll() *Worker {
274 | return DefaultWorker.ClearAll()
275 | }
276 |
277 | // ClearCookie Clear Cookie
278 | func (worker *Worker) ClearCookie() *Worker {
279 | worker.Header.Del("Cookie")
280 | return worker
281 | }
282 |
283 | func ClearCookie() *Worker {
284 | return DefaultWorker.ClearCookie()
285 | }
286 |
287 | // GetCookies Get Cookies
288 | func (worker *Worker) GetCookies() []*http.Cookie {
289 | if worker.Response != nil && worker.Response.Response != nil {
290 | return worker.Response.Response.Cookies()
291 | } else {
292 | return []*http.Cookie{}
293 | }
294 | }
295 |
296 | // GetResponseStatusCode Get ResponseStatusCode
297 | func (worker *Worker) GetResponseStatusCode() int {
298 | if worker.Response != nil && worker.Response.Response != nil {
299 | return worker.Response.ResponseStatusCode
300 | } else {
301 | return 0
302 | }
303 | }
304 |
305 | func GetCookies() []*http.Cookie {
306 | return DefaultWorker.GetCookies()
307 | }
308 |
309 | func GetResponseStatusCode() int {
310 | return DefaultWorker.GetResponseStatusCode()
311 | }
312 |
--------------------------------------------------------------------------------
/miner/config/ua.txt:
--------------------------------------------------------------------------------
1 | Mozilla/5.0 (SymbianOS/9.1; U; [en]; SymbianOS/91 Series60/3.0) AppleWebkit/413 (KHTML, like Gecko) Safari/413
2 | Mozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413
3 | Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413
4 | Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1/110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413
5 | Mozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 Nokia5800d-1/21.0.025; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413
6 | Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/12.0.024; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.12344
7 | Mozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95)
8 | Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
9 | Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; MSN 2.5; Windows 98)
10 | Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 95)
11 | Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 98)
12 | Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)
13 | Mozilla/4.0 (compatible; MSIE 4.01; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)
14 | Mozilla/4.0 (compatible; MSIE 4.01; Windows 98)
15 | Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; BIGLOBE)
16 | Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; canoncopyer)
17 | Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; Compaq)
18 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 95)
19 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)
20 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; i-CABLE)
21 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; ocnie5-1)
22 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)
23 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98)
24 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; CNETHomeBuild051099)
25 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)
26 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt; ocnie5-1)
27 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; wn_ie5_en_v1)
28 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.; DigExt)
29 | Mozilla/4.0 (compatible; MSIE 5.0; Windows NT)
30 | Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)
31 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)
32 | Mozilla/4.0 (compatible; MSIE 5.01; MSN 2.5; Windows 98)
33 | Mozilla/4.0 (compatible; MSIE 5.01; Windows 95)
34 | Mozilla/4.0 (compatible; MSIE 5.01; Windows 98)
35 | Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)
36 | Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DigExt)
37 | Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; TUCOWS)
38 | Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)
39 | Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; .NET CLR 1.1.4322)
40 | Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC)
41 | Mozilla/4.0 (compatible; MSIE 5.16; Mac_PowerPC)
42 | Mozilla/4.0 (compatible; MSIE 5.17; Mac_PowerPC)
43 | Mozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC)
44 | Mozilla/4.0 (compatible; MSIE 5.23; Mac_PowerPC)
45 | Mozilla/4.0 (compatible; MSIE 5.5; Windows 95)
46 | Mozilla/4.0 (compatible; MSIE 5.5; Windows 98)
47 | Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)
48 | Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)
49 | Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.1.4322)
50 | Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; by TSG)
51 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)
52 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)
53 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)
54 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)
55 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607)
56 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 1.0.3705)
57 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; NOKTURNAL KICKS ASS)
58 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FDM;
59 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.41115)
60 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)
61 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1
62 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)
63 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)
64 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1
65 | Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)
66 | Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; iebar; .NET CLR 1.0.3705)
67 | Mozilla/4.0 (compatible; MSIE 6.0; Win32);
68 | Mozilla/4.0 (compatible; MSIE 6.0; Win32); .NET CLR 1.0.3705)
69 | Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Hotbar 4.4.6.0)
70 | Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90)
71 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0)
72 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; BVG
73 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)
74 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322)
75 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)
76 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; FDM)
77 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; {FF0C8E09-3C86-44CB-834A-B8CEEC80A1D7}; iOpus-I-M)
78 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; i-Nav 3.0.1.0F; .NET CLR 1.0.3705; .NET CLR 1.1.4322)
79 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MathPlayer 2.0; .NET CLR 1.1.4322)
80 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Maxthon; .NET CLR 1.1.4322)
81 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)
82 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;)
83 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)
84 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1);
85 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705)
86 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)
87 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)
88 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)
89 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 2.0.40607)
90 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Alexa Toolbar)
91 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; BrowserBob)
92 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; DFO-MPO Internet Explorer 6.0)
93 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ENGINE; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT))
94 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ESB{F40811EE-DF17-4BC9-8785-B362ABF34098}; .NET CLR 1.1.4322)
95 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FDM)
96 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FTDv3 Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322)
97 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.40607)
98 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; AtHome033)
99 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; HCI0449; .NET CLR 1.0.3705)
100 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; i-NavFourF; .NET CLR 1.1.4322)
101 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon;
102 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; .NET CLR 1.1.4322)
103 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; .NET CLR 1.1.4322)
104 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; Maxthon; .NET CLR 1.1.4322)
105 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; FunWebProducts; .NET CLR 1.1.4322)
106 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Woningstichting Den Helder; .NET CLR 1.0.3705)
107 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)
108 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.41115)
109 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; .NET CLR 1.1.4322)
110 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; Maxthon; .NET CLR 1.1.4322)
111 | Mozilla/4.0 (compatible; MSIE 6.0; Windows XP)
112 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)
113 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322)
114 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)
115 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1)
116 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)
117 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)
118 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)
119 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.1)
120 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)
121 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)
122 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; InfoPath.1
123 | Mozilla/4.0 (compatible; GoogleToolbar 5.0.2124.2070; Windows 6.0; MSIE 8.0.6001.18241)
124 | Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits Go v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)
125 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)
126 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Sleipnir/2.9.8)
127 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)
128 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)
129 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)
130 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)
131 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)
132 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)
133 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/6.0)
134 | Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13
135 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19
136 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.33 Safari/530.5
137 | Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.38 Safari/532.0
138 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.55 Safari/533.4
139 | Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3
140 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.55 Safari/534.3
141 | Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.43 Safari/534.7
142 | Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10 ChromePlus/1.5.2.0
143 | Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14
144 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16
145 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24
146 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Iron/11.0.700.2 Chrome/11.0.700.2 Safari/534.24
147 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24
148 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30 ChromePlus/1.6.3.1
149 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30
150 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1
151 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) RockMelt/0.9.64.361 Chrome/13.0.782.218 Safari/535.1
152 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1
153 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1
154 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1
155 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1
156 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2
157 | Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/10.04 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2
158 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2
159 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7
160 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7
161 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; Lunascape 4.7.3)
162 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Lunascape 5.0 alpha2)
163 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Lunascape 5.0 alpha2)
164 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Tablet PC 2.0; Lunascape 5.0 alpha2)
165 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MAGW; .NET4.0C; Lunascape 6.5.8.24780)
166 | Mozilla/2.02 (Macintosh; I; PPC)
167 | Mozilla/3.01 (Macintosh; I; PPC)
168 | Mozilla/4.01 (Macintosh; I; PPC)
169 | Mozilla/4.03 [en]C-IMS (Win95; I)
170 | Mozilla/4.04 [en] (Win95; I ;Nav)
171 | Mozilla/4.04 [en] (Win95; I)
172 | Mozilla/4.04 [en] (WinNT; I ;Nav)
173 | Mozilla/4.04 [en] (WinNT; I)
174 | Mozilla/4.04 [en] (Macintosh; I; PPC Nav)
175 | Mozilla/4.04 [en] (X11; I; SunOS 5.5 sun4u)
176 | Mozilla/4.05 [en] (Win95; I)
177 | Mozilla/4.05 (Macintosh; I; PPC)
178 | Mozilla/4.06 [en] (Win98; I)
179 | Mozilla/4.06 [en] (Macintosh; I; PPC)
180 | Mozilla/4.08 (Macintosh; I; PPC)
181 | Mozilla/4.5 [en] (Win95; I)
182 | Mozilla/4.5 [en] (Win98; I)
183 | Mozilla/4.5 [en] (WinNT; I)
184 | Mozilla/4.5 (Macintosh; I; PPC)
185 | Mozilla/4.51 [en] (Win95; I)
186 | Mozilla/4.51 [en] (Win98; I)
187 | Mozilla/4.51 [en] (WinNT; I)
188 | Mozilla/4.51 [en] (X11; I; SunOS 5.8 sun4u)
189 | Mozilla/4.6 [en] (Win95; I)
190 | Mozilla/4.6 [en] (Win98; I)
191 | Mozilla/4.6 [en] (WinNT; I)
192 | Mozilla/4.6 [en] (WinNT; I)
193 | Mozilla/4.7 [en] (WinNT; I)
194 | Mozilla/4.7 [en] (Win95; I)
195 | Mozilla/4.7 [en] (Win98; I)
196 | Mozilla/4.7 [en] (WinNT; I)
197 | Mozilla/4.7 [en] (WinNT; I)
198 | Mozilla/4.7 [en] (WinNT; U)
199 | Mozilla/4.7 [en] (Macintosh; I; PPC)
200 | Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)
201 | Mozilla/4.76 [en] (X11; U; SunOS 5.8 sun4u)
202 | Mozilla/4.78 [en] (X11; U; SunOS 5.9 sun4u)
203 | Mozilla/4.8 [en] (X11; U; SunOS 5.7 sun4u)
204 | Mozilla/5.0 (Windows; U; Win98; en-JP; m18) Gecko/20001108 Netscape6/6.0
205 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; m18) Gecko/20010131 Netscape6/6.01
206 | Mozilla/5.0 (Windows; U; Win 9x 4.90; en-JP; rv:0.9.4) Gecko/20011128 Netscape6/6.2.1
207 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:0.9.4.1) Gecko/20020508 Netscape6/6.2.3
208 | Mozilla/5.0 (Macintosh; N; PPC; en-JP; macen-pub12) Gecko/20001108 Netscape6/6.0
209 | Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.2) Gecko/20010726 Netscape6/6.1
210 | Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.4) Gecko/20011022 Netscape6/6.2
211 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:0.9.4.1) Gecko/20020315 Netscape6/6.2.2
212 | Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02
213 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)
214 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)
215 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)
216 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.2) Gecko/20040805 Netscape/7.2
217 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-JP; rv:1.0.2) Gecko/20021120 Netscape/7.01
218 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1
219 | Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.0.1) Gecko/20020921 Netscape/7.0
220 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; rv:1.5) Gecko/20031007
221 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113
222 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040616
223 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040113
224 | Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.2.1) Gecko/20030225
225 | Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.4.1) Gecko/20031030
226 | Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.1) Gecko/20040805
227 | Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.4) Gecko/20040414
228 | Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20040913 Firefox/0.10
229 | Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20040913 Firefox/0.10
230 | Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20040913 Firefox/0.10
231 | Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20041001 Firefox/0.10.1
232 | Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1
233 | Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20041001 Firefox/0.10.1
234 | Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1
235 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.6) Gecko/20040206 Firefox/0.8
236 | Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.6) Gecko/20040206 Firefox/0.8
237 | Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.6) Gecko/20040207 Firefox/0.8
238 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040206 Firefox/0.8
239 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040614 Firefox/0.9
240 | Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.7) Gecko/20040708 Firefox/0.9
241 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040707 Firefox/0.9.2
242 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/0.9.2 StumbleUpon/1.994
243 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040803 Firefox/0.9.3
244 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3
245 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3
246 | Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3
247 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3
248 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20040803 Firefox/0.9.3
249 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)
250 | Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)
251 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
252 | Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0
253 | Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0
254 | Mozilla/5.0 (Windows; U; Win98; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0
255 | Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0
256 | Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
257 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041107 Firefox/1.0
258 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0
259 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
260 | Mozilla/5.0 (Windows; U; Windows NT 5.0; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0
261 | Mozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.5) Gecko/20041110 Firefox/1.0
262 | Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041107 Firefox/1.0
263 | Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0
264 | Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
265 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0
266 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
267 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 StumbleUpon/1.999
268 | Mozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.7.5) Gecko/20041210 Firefox/1.0
269 | Mozilla/5.0 (Windows; U; Windows NT 5.1; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0
270 | Mozilla/5.0 (Windows; U; Windows NT 5.1; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0
271 | Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.7.5) Gecko/20041108 Firefox/1.0
272 | Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
273 | Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8b) Gecko/20050212 Firefox/1.0+ (MOOX M3)
274 | Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
275 | Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+
276 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+
277 | Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.5) Gecko/20050103 Firefox/1.0
278 | Mozilla/5.0 (X11; U; Linux i386; en-US; rv:1.7.5) Gecko/20041109 Firefox/1.0
279 | Mozilla/5.0 (X11; U; Linux i686; de-DE; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)
280 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
281 | Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0
282 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0
283 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 (Debian package 1.0-2)
284 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041116 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0-2ubuntu3)
285 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041119 Firefox/1.0
286 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041123 Firefox/1.0
287 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)
288 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041130 Firefox/1.0
289 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041214 Firefox/1.0 StumbleUpon/1.999
290 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041219 Firefox/1.0 (Debian package 1.0+dfsg.1-1)
291 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20050110 Firefox/1.0 (Debian package 1.0+dfsg.1-2)
292 | Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0
293 | Mozilla/5.0 (X11; U; Linux i686; nl-NL; rv:1.7.5) Gecko/20050221 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0+dfsg.1-6ubuntu1)
294 | Mozilla/5.0 (X11; U; Linux i686; rv:1.8b) Gecko/20050124 Firefox/1.0+
295 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.6) Gecko/20050306 Firefox/1.0.1 (Debian package 1.0.1-2)
296 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050223 Firefox/1.0.1
297 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050225 Firefox/1.0.1
298 | Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1
299 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP-mac; rv:1.8) Gecko/20051111 Firefox/1.5
300 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8) Gecko/20051111 Firefox/1.5
301 | Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5
302 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1
303 | Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2
304 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3
305 | Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4
306 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20
307 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12
308 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-JP-mac; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20
309 | Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6
310 | Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)
311 | Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.17) Gecko/2009122116 Firefox/3.0.17 GTB6 (.NET CLR 3.5.30729)
312 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6
313 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 GTB5
314 | Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20100101 Firefox/4.0
315 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.0) Gecko/20100101 Firefox/4.0
316 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0
317 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0.1) Gecko/20100101 Firefox/5.0.1
318 | Mozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0
319 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2
320 | Mozilla/5.0 (Windows NT 6.0; rv:7.0.1) Gecko/20100101 Firefox/7.0.1
321 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:7.0.1) Gecko/20100101 Firefox/7.0.1
322 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:8.0.1) Gecko/20100101 Firefox/8.0.1
323 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
324 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0.1) Gecko/20100101 Firefox/9.0.1
325 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6
326 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.7
327 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8
328 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8.1
329 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.5 (KHTML, like Gecko) Safari/85.8
330 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1
331 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1
332 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8
333 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.4 (KHTML, like Gecko) Safari/125.9
334 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.11
335 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12
336 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12
337 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; nl-nl) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12
338 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.5.6 (KHTML, like Gecko) Safari/125.12
339 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312
340 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1.1 (KHTML, like Gecko) Safari/312
341 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3
342 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.5
343 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6
344 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412 (KHTML, like Gecko) Safari/412
345 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/125.11
346 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/412.2.2
347 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.7 (KHTML, like Gecko) Safari/412.5
348 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/416.11 (KHTML, like Gecko) Safari/416.12
349 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.2
350 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3
351 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1
352 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/523.10.3 (KHTML, like Gecko) Version/3.0.4 Safari/523.10
353 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6
354 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12 (KHTML, like Gecko) Version/3.0.4 Safari/523.12
355 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12.2 (KHTML, like Gecko) Version/3.0.4 Safari/523.12.2
356 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13
357 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_3; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20
358 | Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22
359 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-jp) AppleWebKit/525.26.2 (KHTML, like Gecko) Version/3.2 Safari/525.26.12
360 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1
361 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16
362 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-jp) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10
363 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/531.21.11 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10
364 | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16
365 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7
366 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2
367 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25
368 | Mozilla/4.0 (Windows 95;US) Opera 3.62 [en]
369 | Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 6.0 [en]
370 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.01 [en]
371 | Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 6.03 [en]
372 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.05 [en]
373 | Mozilla/4.0 (compatible; MSIE 5.0; Windows XP) Opera 6.06 [en]
374 | Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.06 [en]
375 | Opera 7.11
376 | Opera/7.23 (Windows NT 5.0; U) [en]
377 | Opera/7.52 (Windows NT 5.1; U) [en]
378 | Opera/7.53 (Windows NT 5.0; U) [en]
379 | Opera/7.54 (Windows NT 5.0; U) [en]
380 | Opera/7.54 (Windows NT 5.1; U) [en]
381 | Opera/7.54 (Windows NT 5.1; U)
382 | Opera/7.54 (X11; Linux i686; U) [en]
383 | Opera/7.54 (X11; Linux i686; U) [sv]
384 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.11 [en]
385 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.21 [pt-BR]
386 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.22 [en]
387 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]
388 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]
389 | Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.23 [en]
390 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.23 [fr]
391 | Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC) Opera 7.50 [en]
392 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.53 [en]
393 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.53 [en]
394 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]
395 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54 [en]
396 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]
397 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54u1 [en]
398 | Opera/7.54 (Windows 98; U) [en]
399 | Mozilla/4.78 (Windows NT 5.1; U) Opera 7.23 [en]
400 | Mozilla/5.0 (Windows NT 5.0; U) Opera 7.54 [en]
401 | Mozilla/4.0 (compatible; MSIE 6.0; X11; OpenBSD i386) Opera 7.54 [en]
402 | Opera/8.0 (X11; Linux i686; U; en)
403 | Opera/8.01 (Windows ME; U; en)
404 | Opera/8.01 (Windows NT 5.1; U; en)
405 | Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.01
406 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.01
407 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; tr) Opera 8.02
408 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.02
409 | Opera/8.5 (Windows NT 5.0; U; en)
410 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.5
411 | Opera/8.51 (Windows NT 5.1; U; en)
412 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.51
413 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; de) Opera 8.52
414 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; pl) Opera 8.52
415 | Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.52
416 | Opera/8.53 (Windows NT 5.1; U; en)
417 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.53
418 | Opera/8.54 (Windows NT 5.1; U; en)
419 | Opera/8.54 (Windows NT 5.0; U; en)
420 | Mozilla/5.0 (X11; Linux i686; U; cs) Opera 8.54
421 | Mozilla/4.0 (compatible; MSIE 6.0; KDDI-SA39) Opera 8.60 [en]
422 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 9.00
423 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.00
424 | Opera/9.00 (Windows NT 5.1; U; en)
425 | Opera/9.0 (Windows NT 5.1; U; en)
426 | Opera/9.00 (Macintosh; PPC Mac OS X; U; en)
427 | Opera/9.02 (Macintosh; PPC Mac OS X; U; en)
428 | Opera/9.02 (Windows NT 5.1; U; zh-tw)
429 | Opera/9.10 (Windows NT 6.0; U; en)
430 | Opera/9.21 (Windows NT 6.0; U; en)
431 | Opera/9.22 (Windows NT 5.1; U; en)
432 | Opera/9.23 (Windows NT 5.1; U; en)
433 | Opera/9.23 (Windows ME; U; en)
434 | Opera/9.26 (Windows NT 5.1; U; en)
435 | Opera/9.51 (Windows NT 5.1; U; en)
436 | Opera/9.52 (Macintosh; Intel Mac OS X; U; en)
437 | Opera/9.52 (Windows NT 5.1; U; en)
438 | Opera/9.60 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1
439 | Opera/9.60 (Windows NT 5.1; U; en) Presto/2.1.1
440 | Opera/9.61 (Windows NT 5.1; U; en) Presto/2.1.1
441 | Opera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1
442 | Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; en) Opera 10.10
443 | Opera/9.80 (Windows NT 6.1; U; en) Presto/2.9.168 Version/11.50
444 | Opera/9.80 (Windows NT 6.1; U; en) Presto/2.10.229 Version/11.60
445 | Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.10.289 Version/12.00
--------------------------------------------------------------------------------
/miner/cons.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import "net/http"
4 |
5 | const (
6 | VERSION = "1.0.13"
7 |
8 | // GET HTTP method
9 | GET = "GET"
10 | POST = "POST"
11 | POSTJSON = "POSTJSON"
12 | POSTXML = "POSTXML"
13 | POSTFILE = "POSTFILE"
14 | PUT = "PUT"
15 | PUTJSON = "PUTJSON"
16 | PUTXML = "PUTXML"
17 | PUTFILE = "PUTFILE"
18 | DELETE = "DELETE"
19 | OTHER = "OTHER" // this stand for you can use other method this lib not own.
20 |
21 | // HTTPFORMContentType HTTP content type
22 | HTTPFORMContentType = "application/x-www-form-urlencoded"
23 | HTTPJSONContentType = "application/json"
24 | HTTPXMLContentType = "text/xml"
25 | HTTPFILEContentType = "multipart/form-data"
26 | )
27 |
28 | var (
29 | // Browser User-Agent, Our default Http ua header!
30 | ourLoveUa = "Marmot+" + VERSION + "+github:hunterhug"
31 |
32 | DefaultHeader = map[string][]string{
33 | "User-Agent": {
34 | ourLoveUa,
35 | },
36 | }
37 |
38 | // DefaultTimeOut http get and post No timeout
39 | DefaultTimeOut = 0
40 | )
41 |
42 | // SetGlobalTimeout Set global timeout, it can only by this way!
43 | func SetGlobalTimeout(num int) {
44 | DefaultTimeOut = num
45 | }
46 |
47 | func SetDefaultTimeOut(num int) {
48 | DefaultTimeOut = num
49 | }
50 |
51 | // MergeCookie Merge Cookie, not use
52 | func MergeCookie(before []*http.Cookie, after []*http.Cookie) []*http.Cookie {
53 | cs := make(map[string]*http.Cookie)
54 |
55 | for _, b := range before {
56 | cs[b.Name] = b
57 | }
58 |
59 | for _, a := range after {
60 | if a.Value != "" {
61 | cs[a.Name] = a
62 | }
63 | }
64 |
65 | res := make([]*http.Cookie, 0, len(cs))
66 |
67 | for _, q := range cs {
68 | res = append(res, q)
69 |
70 | }
71 |
72 | return res
73 |
74 | }
75 |
76 | // CloneHeader Clone a header, If not exist Ua, Set our Ua!
77 | func CloneHeader(h map[string][]string) map[string][]string {
78 | if h == nil || len(h) == 0 {
79 | h = DefaultHeader
80 | return h
81 | }
82 |
83 | if len(h["User-Agent"]) == 0 {
84 | h["User-Agent"] = []string{ourLoveUa}
85 | }
86 | return CopyM(h)
87 | }
88 |
--------------------------------------------------------------------------------
/miner/default.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import "net/http"
4 |
5 | // DefaultWorker Global Worker
6 | var DefaultWorker *Worker
7 |
8 | func init() {
9 | UaInit()
10 |
11 | // New a Worker
12 | worker := new(Worker)
13 | worker.Request = newRequest()
14 | worker.Header = http.Header{}
15 | worker.Response = new(Response)
16 | worker.Client = Client
17 |
18 | // Global Worker!
19 | DefaultWorker = worker
20 | }
21 |
22 | // ToString This make effect only your Worker exec serial! Attention!
23 | // Change Your Raw data To string
24 | func ToString() string {
25 | return DefaultWorker.ToString()
26 | }
27 |
28 | // JsonToString This make effect only your Worker exec serial! Attention!
29 | // Change Your JSON like Raw data to string
30 | func JsonToString() (string, error) {
31 | return DefaultWorker.JsonToString()
32 | }
33 |
34 | func Clone() *Worker {
35 | return DefaultWorker.Clone()
36 | }
37 |
38 | func Get() (body []byte, e error) {
39 | return DefaultWorker.Get()
40 | }
41 |
42 | func Delete() (body []byte, e error) {
43 | return DefaultWorker.Delete()
44 | }
45 |
46 | func Go() (body []byte, e error) {
47 | return DefaultWorker.Go()
48 | }
49 |
50 | func GoByMethod(method string) (body []byte, e error) {
51 | return DefaultWorker.SetMethod(method).Go()
52 | }
53 |
54 | func OtherGo(method, contentType string) (body []byte, e error) {
55 | return DefaultWorker.OtherGo(method, contentType)
56 | }
57 |
58 | func Post() (body []byte, e error) {
59 | return DefaultWorker.Post()
60 | }
61 |
62 | func PostJSON() (body []byte, e error) {
63 | return DefaultWorker.PostJSON()
64 | }
65 |
66 | func PostFILE() (body []byte, e error) {
67 | return DefaultWorker.PostFILE()
68 | }
69 |
70 | func PostXML() (body []byte, e error) {
71 | return DefaultWorker.PostXML()
72 | }
73 |
74 | func Put() (body []byte, e error) {
75 | return DefaultWorker.Put()
76 | }
77 | func PutJSON() (body []byte, e error) {
78 | return DefaultWorker.PutJSON()
79 | }
80 |
81 | func PutFILE() (body []byte, e error) {
82 | return DefaultWorker.PutFILE()
83 | }
84 |
85 | func PutXML() (body []byte, e error) {
86 | return DefaultWorker.PutXML()
87 | }
88 |
--------------------------------------------------------------------------------
/miner/doc.go:
--------------------------------------------------------------------------------
1 | /*
2 | All right reserved https://github.com/hunterhug/marmot at 2016-2022
3 | Attribution-NonCommercial-NoDerivatives 4.0 International
4 | Notice: The following code's copyright by hunterhug, Please do not spread and modify.
5 | You can use it for education only but can't make profits for any companies and individuals!
6 | */
7 |
8 | /*
9 | Package miner is the core of this project, use to request for http api.
10 |
11 | Example:
12 |
13 | package main
14 |
15 | import (
16 | "fmt"
17 | "github.com/hunterhug/marmot/miner"
18 | )
19 |
20 | func main() {
21 | miner.SetLogLevel(miner.DEBUG)
22 |
23 | // Use Default Worker, You can Also New One:
24 | //worker, _ := miner.New(nil)
25 | //worker = miner.NewWorkerWithNoProxy()
26 | //worker = miner.NewAPI()
27 | //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080")
28 | worker := miner.Clone()
29 | _, err := worker.SetUrl("https://www.bing.com").Go()
30 | if err != nil {
31 | fmt.Println(err.Error())
32 | } else {
33 | fmt.Println(worker.ToString())
34 | }
35 | }
36 | */
37 | package miner
38 |
--------------------------------------------------------------------------------
/miner/log.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "github.com/hunterhug/golog"
5 | )
6 |
7 | type Level = golog.Level
8 |
9 | const (
10 | DEBUG = golog.DebugLevel
11 | INFO = golog.InfoLevel
12 | WARN = golog.WarnLevel
13 | ERROR = golog.ErrorLevel
14 | )
15 |
16 | // Logger Global logger config for debug
17 | var (
18 | Logger = golog.New()
19 | )
20 |
21 | func init() {
22 | Logger.InitLogger()
23 | }
24 |
25 | // SetLogLevel Set log level
26 | func SetLogLevel(level Level) {
27 | Logger.SetLevel(level)
28 | Logger.InitLogger()
29 | }
30 |
31 | // Log Return global log
32 | func Log() golog.LoggerInterface {
33 | return Logger
34 | }
35 |
--------------------------------------------------------------------------------
/miner/multworker.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "sync"
5 | )
6 |
7 | // Pool for many Worker, every Worker can only serial execution
8 | var Pool = &_Workers{ws: make(map[string]*Worker)}
9 |
10 | type _Workers struct {
11 | mux sync.RWMutex
12 | ws map[string]*Worker
13 | }
14 |
15 | func (pool *_Workers) Get(name string) (b *Worker, ok bool) {
16 | pool.mux.RLock()
17 | b, ok = pool.ws[name]
18 | pool.mux.RUnlock()
19 | return
20 | }
21 |
22 | func (pool *_Workers) Set(name string, b *Worker) {
23 | pool.mux.Lock()
24 | pool.ws[name] = b
25 | pool.mux.Unlock()
26 | return
27 | }
28 |
29 | func (pool *_Workers) Delete(name string) {
30 | pool.mux.Lock()
31 | delete(pool.ws, name)
32 | pool.mux.Unlock()
33 | return
34 | }
35 |
--------------------------------------------------------------------------------
/miner/ua.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "github.com/hunterhug/marmot/util"
5 | "math/rand"
6 | "strings"
7 | )
8 |
9 | // Ua Global User-Agent provide
10 | var Ua = map[int]string{}
11 |
12 | // UaInit User-Agent init
13 | func UaInit() {
14 | Ua = map[int]string{
15 | 0: "Mozilla/5.0 (SymbianOS/9.1; U; [en]; SymbianOS/91 Series60/3.0) AppleWebkit/413 (KHTML, like Gecko) Safari/413",
16 | 1: "Mozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
17 | 2: "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
18 | 3: "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1/110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
19 | 4: "Mozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 Nokia5800d-1/21.0.025; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
20 | 5: "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/12.0.024; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.12344",
21 | 6: "Mozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95)",
22 | 7: "Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)",
23 | 8: "Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; MSN 2.5; Windows 98)",
24 | 9: "Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 95)",
25 | 10: "Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 98)",
26 | 11: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)",
27 | 12: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)",
28 | 13: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98)",
29 | 14: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; BIGLOBE)",
30 | 15: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; canoncopyer)",
31 | 16: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; Compaq)",
32 | 17: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95)",
33 | 18: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)",
34 | 19: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; i-CABLE)",
35 | 20: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; ocnie5-1)",
36 | 21: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)",
37 | 22: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98)",
38 | 23: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; CNETHomeBuild051099)",
39 | 24: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)",
40 | 25: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt; ocnie5-1)",
41 | 26: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; wn_ie5_en_v1)",
42 | 27: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.; DigExt)",
43 | 28: "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT)",
44 | 29: "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)",
45 | 30: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)",
46 | 31: "Mozilla/4.0 (compatible; MSIE 5.01; MSN 2.5; Windows 98)",
47 | 32: "Mozilla/4.0 (compatible; MSIE 5.01; Windows 95)",
48 | 33: "Mozilla/4.0 (compatible; MSIE 5.01; Windows 98)",
49 | 34: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)",
50 | 35: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DigExt)",
51 | 36: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; TUCOWS)",
52 | 37: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)",
53 | 38: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; .NET CLR 1.1.4322)",
54 | 39: "Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC)",
55 | 40: "Mozilla/4.0 (compatible; MSIE 5.16; Mac_PowerPC)",
56 | 41: "Mozilla/4.0 (compatible; MSIE 5.17; Mac_PowerPC)",
57 | 42: "Mozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC)",
58 | 43: "Mozilla/4.0 (compatible; MSIE 5.23; Mac_PowerPC)",
59 | 44: "Mozilla/4.0 (compatible; MSIE 5.5; Windows 95)",
60 | 45: "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98)",
61 | 46: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)",
62 | 47: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)",
63 | 48: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.1.4322)",
64 | 49: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; by TSG)",
65 | 50: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)",
66 | 51: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
67 | 52: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
68 | 53: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)",
69 | 54: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607)",
70 | 55: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 1.0.3705)",
71 | 56: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; NOKTURNAL KICKS ASS)",
72 | 57: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FDM;",
73 | 58: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.41115)",
74 | 59: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)",
75 | 60: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1",
76 | 61: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
77 | 62: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)",
78 | 63: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1",
79 | 64: "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)",
80 | 65: "Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; iebar; .NET CLR 1.0.3705)",
81 | 66: "Mozilla/4.0 (compatible; MSIE 6.0; Win32);",
82 | 67: "Mozilla/4.0 (compatible; MSIE 6.0; Win32); .NET CLR 1.0.3705)",
83 | 68: "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Hotbar 4.4.6.0)",
84 | 69: "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90)",
85 | 70: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0)",
86 | 71: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; BVG",
87 | 72: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)",
88 | 73: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
89 | 74: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)",
90 | 75: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; FDM)",
91 | 76: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; {FF0C8E09-3C86-44CB-834A-B8CEEC80A1D7}; iOpus-I-M)",
92 | 77: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; i-Nav 3.0.1.0F; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
93 | 78: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MathPlayer 2.0; .NET CLR 1.1.4322)",
94 | 79: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Maxthon; .NET CLR 1.1.4322)",
95 | 80: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)",
96 | 81: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;)",
97 | 82: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
98 | 83: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1);",
99 | 84: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705)",
100 | 85: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
101 | 86: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)",
102 | 87: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)",
103 | 88: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 2.0.40607)",
104 | 89: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Alexa Toolbar)",
105 | 90: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; BrowserBob)",
106 | 91: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; DFO-MPO Internet Explorer 6.0)",
107 | 92: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ENGINE; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT))",
108 | 93: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ESB{F40811EE-DF17-4BC9-8785-B362ABF34098}; .NET CLR 1.1.4322)",
109 | 94: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FDM)",
110 | 95: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FTDv3 Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
111 | 96: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.40607)",
112 | 97: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; AtHome033)",
113 | 98: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; HCI0449; .NET CLR 1.0.3705)",
114 | 99: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; i-NavFourF; .NET CLR 1.1.4322)",
115 | 100: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon;",
116 | 101: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; .NET CLR 1.1.4322)",
117 | 102: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; .NET CLR 1.1.4322)",
118 | 103: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; Maxthon; .NET CLR 1.1.4322)",
119 | 104: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; FunWebProducts; .NET CLR 1.1.4322)",
120 | 105: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Woningstichting Den Helder; .NET CLR 1.0.3705)",
121 | 106: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)",
122 | 107: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.41115)",
123 | 108: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; .NET CLR 1.1.4322)",
124 | 109: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; Maxthon; .NET CLR 1.1.4322)",
125 | 110: "Mozilla/4.0 (compatible; MSIE 6.0; Windows XP)",
126 | 111: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)",
127 | 112: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322)",
128 | 113: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
129 | 114: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1)",
130 | 115: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
131 | 116: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)",
132 | 117: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
133 | 118: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.1)",
134 | 119: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
135 | 120: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)",
136 | 121: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; InfoPath.1",
137 | 122: "Mozilla/4.0 (compatible; GoogleToolbar 5.0.2124.2070; Windows 6.0; MSIE 8.0.6001.18241)",
138 | 123: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits Go v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
139 | 124: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
140 | 125: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Sleipnir/2.9.8)",
141 | 126: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)",
142 | 127: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
143 | 128: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
144 | 129: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
145 | 130: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)",
146 | 131: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
147 | 132: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/6.0)",
148 | 133: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13",
149 | 134: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19",
150 | 135: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.33 Safari/530.5",
151 | 136: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.38 Safari/532.0",
152 | 137: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.55 Safari/533.4",
153 | 138: "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3",
154 | 139: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.55 Safari/534.3",
155 | 140: "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.43 Safari/534.7",
156 | 141: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10 ChromePlus/1.5.2.0",
157 | 142: "Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14",
158 | 143: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16",
159 | 144: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24",
160 | 145: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Iron/11.0.700.2 Chrome/11.0.700.2 Safari/534.24",
161 | 146: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24",
162 | 147: "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30 ChromePlus/1.6.3.1",
163 | 148: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30",
164 | 149: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1",
165 | 150: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) RockMelt/0.9.64.361 Chrome/13.0.782.218 Safari/535.1",
166 | 151: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1",
167 | 152: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1",
168 | 153: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1",
169 | 154: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1",
170 | 155: "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
171 | 156: "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/10.04 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2",
172 | 157: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2",
173 | 158: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7",
174 | 159: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7",
175 | 160: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; Lunascape 4.7.3)",
176 | 161: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Lunascape 5.0 alpha2)",
177 | 162: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Lunascape 5.0 alpha2)",
178 | 163: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Tablet PC 2.0; Lunascape 5.0 alpha2)",
179 | 164: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MAGW; .NET4.0C; Lunascape 6.5.8.24780)",
180 | 165: "Mozilla/2.02 (Macintosh; I; PPC)",
181 | 166: "Mozilla/3.01 (Macintosh; I; PPC)",
182 | 167: "Mozilla/4.01 (Macintosh; I; PPC)",
183 | 168: "Mozilla/4.03 [en]C-IMS (Win95; I)",
184 | 169: "Mozilla/4.04 [en] (Win95; I ;Nav)",
185 | 170: "Mozilla/4.04 [en] (Win95; I)",
186 | 171: "Mozilla/4.04 [en] (WinNT; I ;Nav)",
187 | 172: "Mozilla/4.04 [en] (WinNT; I)",
188 | 173: "Mozilla/4.04 [en] (Macintosh; I; PPC Nav)",
189 | 174: "Mozilla/4.04 [en] (X11; I; SunOS 5.5 sun4u)",
190 | 175: "Mozilla/4.05 [en] (Win95; I)",
191 | 176: "Mozilla/4.05 (Macintosh; I; PPC)",
192 | 177: "Mozilla/4.06 [en] (Win98; I)",
193 | 178: "Mozilla/4.06 [en] (Macintosh; I; PPC)",
194 | 179: "Mozilla/4.08 (Macintosh; I; PPC)",
195 | 180: "Mozilla/4.5 [en] (Win95; I)",
196 | 181: "Mozilla/4.5 [en] (Win98; I)",
197 | 182: "Mozilla/4.5 [en] (WinNT; I)",
198 | 183: "Mozilla/4.5 (Macintosh; I; PPC)",
199 | 184: "Mozilla/4.51 [en] (Win95; I)",
200 | 185: "Mozilla/4.51 [en] (Win98; I)",
201 | 186: "Mozilla/4.51 [en] (WinNT; I)",
202 | 187: "Mozilla/4.51 [en] (X11; I; SunOS 5.8 sun4u)",
203 | 188: "Mozilla/4.6 [en] (Win95; I)",
204 | 189: "Mozilla/4.6 [en] (Win98; I)",
205 | 190: "Mozilla/4.6 [en] (WinNT; I)",
206 | 191: "Mozilla/4.6 [en] (WinNT; I)",
207 | 192: "Mozilla/4.7 [en] (WinNT; I)",
208 | 193: "Mozilla/4.7 [en] (Win95; I)",
209 | 194: "Mozilla/4.7 [en] (Win98; I)",
210 | 195: "Mozilla/4.7 [en] (WinNT; I)",
211 | 196: "Mozilla/4.7 [en] (WinNT; I)",
212 | 197: "Mozilla/4.7 [en] (WinNT; U)",
213 | 198: "Mozilla/4.7 [en] (Macintosh; I; PPC)",
214 | 199: "Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)",
215 | 200: "Mozilla/4.76 [en] (X11; U; SunOS 5.8 sun4u)",
216 | 201: "Mozilla/4.78 [en] (X11; U; SunOS 5.9 sun4u)",
217 | 202: "Mozilla/4.8 [en] (X11; U; SunOS 5.7 sun4u)",
218 | 203: "Mozilla/5.0 (Windows; U; Win98; en-JP; m18) Gecko/20001108 Netscape6/6.0",
219 | 204: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; m18) Gecko/20010131 Netscape6/6.01",
220 | 205: "Mozilla/5.0 (Windows; U; Win 9x 4.90; en-JP; rv:0.9.4) Gecko/20011128 Netscape6/6.2.1",
221 | 206: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:0.9.4.1) Gecko/20020508 Netscape6/6.2.3",
222 | 207: "Mozilla/5.0 (Macintosh; N; PPC; en-JP; macen-pub12) Gecko/20001108 Netscape6/6.0",
223 | 208: "Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
224 | 209: "Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.4) Gecko/20011022 Netscape6/6.2",
225 | 210: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:0.9.4.1) Gecko/20020315 Netscape6/6.2.2",
226 | 211: "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02",
227 | 212: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)",
228 | 213: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)",
229 | 214: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)",
230 | 215: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.2) Gecko/20040805 Netscape/7.2",
231 | 216: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-JP; rv:1.0.2) Gecko/20021120 Netscape/7.01",
232 | 217: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1",
233 | 218: "Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.0.1) Gecko/20020921 Netscape/7.0",
234 | 219: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; rv:1.5) Gecko/20031007",
235 | 220: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113",
236 | 221: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040616",
237 | 222: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040113",
238 | 223: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.2.1) Gecko/20030225",
239 | 224: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.4.1) Gecko/20031030",
240 | 225: "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.1) Gecko/20040805",
241 | 226: "Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.4) Gecko/20040414",
242 | 227: "Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20040913 Firefox/0.10",
243 | 228: "Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20040913 Firefox/0.10",
244 | 229: "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20040913 Firefox/0.10",
245 | 230: "Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20041001 Firefox/0.10.1",
246 | 231: "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1",
247 | 232: "Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20041001 Firefox/0.10.1",
248 | 233: "Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1",
249 | 234: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.6) Gecko/20040206 Firefox/0.8",
250 | 235: "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.6) Gecko/20040206 Firefox/0.8",
251 | 236: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.6) Gecko/20040207 Firefox/0.8",
252 | 237: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040206 Firefox/0.8",
253 | 238: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040614 Firefox/0.9",
254 | 239: "Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.7) Gecko/20040708 Firefox/0.9",
255 | 240: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040707 Firefox/0.9.2",
256 | 241: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/0.9.2 StumbleUpon/1.994",
257 | 242: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040803 Firefox/0.9.3",
258 | 243: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3",
259 | 244: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3",
260 | 245: "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3",
261 | 246: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3",
262 | 247: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20040803 Firefox/0.9.3",
263 | 248: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)",
264 | 249: "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)",
265 | 250: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
266 | 251: "Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0",
267 | 252: "Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0",
268 | 253: "Mozilla/5.0 (Windows; U; Win98; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0",
269 | 254: "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0",
270 | 255: "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0",
271 | 256: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041107 Firefox/1.0",
272 | 257: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0",
273 | 258: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
274 | 259: "Mozilla/5.0 (Windows; U; Windows NT 5.0; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0",
275 | 260: "Mozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.5) Gecko/20041110 Firefox/1.0",
276 | 261: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041107 Firefox/1.0",
277 | 262: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0",
278 | 263: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0",
279 | 264: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0",
280 | 265: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
281 | 266: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 StumbleUpon/1.999",
282 | 267: "Mozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.7.5) Gecko/20041210 Firefox/1.0",
283 | 268: "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0",
284 | 269: "Mozilla/5.0 (Windows; U; Windows NT 5.1; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0",
285 | 270: "Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.7.5) Gecko/20041108 Firefox/1.0",
286 | 271: "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
287 | 272: "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8b) Gecko/20050212 Firefox/1.0+ (MOOX M3)",
288 | 273: "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
289 | 274: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+",
290 | 275: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+",
291 | 276: "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.5) Gecko/20050103 Firefox/1.0",
292 | 277: "Mozilla/5.0 (X11; U; Linux i386; en-US; rv:1.7.5) Gecko/20041109 Firefox/1.0",
293 | 278: "Mozilla/5.0 (X11; U; Linux i686; de-DE; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)",
294 | 279: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
295 | 280: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0",
296 | 281: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0",
297 | 282: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 (Debian package 1.0-2)",
298 | 283: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041116 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0-2ubuntu3)",
299 | 284: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041119 Firefox/1.0",
300 | 285: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041123 Firefox/1.0",
301 | 286: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)",
302 | 287: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041130 Firefox/1.0",
303 | 288: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041214 Firefox/1.0 StumbleUpon/1.999",
304 | 289: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041219 Firefox/1.0 (Debian package 1.0+dfsg.1-1)",
305 | 290: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20050110 Firefox/1.0 (Debian package 1.0+dfsg.1-2)",
306 | 291: "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0",
307 | 292: "Mozilla/5.0 (X11; U; Linux i686; nl-NL; rv:1.7.5) Gecko/20050221 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0+dfsg.1-6ubuntu1)",
308 | 293: "Mozilla/5.0 (X11; U; Linux i686; rv:1.8b) Gecko/20050124 Firefox/1.0+",
309 | 294: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.6) Gecko/20050306 Firefox/1.0.1 (Debian package 1.0.1-2)",
310 | 295: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050223 Firefox/1.0.1",
311 | 296: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050225 Firefox/1.0.1",
312 | 297: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1",
313 | 298: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP-mac; rv:1.8) Gecko/20051111 Firefox/1.5",
314 | 299: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8) Gecko/20051111 Firefox/1.5",
315 | 300: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5",
316 | 301: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1",
317 | 302: "Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2",
318 | 303: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3",
319 | 304: "Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4",
320 | 305: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20",
321 | 306: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12",
322 | 307: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-JP-mac; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20",
323 | 308: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6",
324 | 309: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)",
325 | 310: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.17) Gecko/2009122116 Firefox/3.0.17 GTB6 (.NET CLR 3.5.30729)",
326 | 311: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6",
327 | 312: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 GTB5",
328 | 313: "Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20100101 Firefox/4.0",
329 | 314: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.0) Gecko/20100101 Firefox/4.0",
330 | 315: "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0",
331 | 316: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0.1) Gecko/20100101 Firefox/5.0.1",
332 | 317: "Mozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0",
333 | 318: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2",
334 | 319: "Mozilla/5.0 (Windows NT 6.0; rv:7.0.1) Gecko/20100101 Firefox/7.0.1",
335 | 320: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:7.0.1) Gecko/20100101 Firefox/7.0.1",
336 | 321: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:8.0.1) Gecko/20100101 Firefox/8.0.1",
337 | 322: "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1",
338 | 323: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0.1) Gecko/20100101 Firefox/9.0.1",
339 | 324: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6",
340 | 325: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.7",
341 | 326: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8",
342 | 327: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8.1",
343 | 328: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.5 (KHTML, like Gecko) Safari/85.8",
344 | 329: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1",
345 | 330: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1",
346 | 331: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8",
347 | 332: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.4 (KHTML, like Gecko) Safari/125.9",
348 | 333: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.11",
349 | 334: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12",
350 | 335: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12",
351 | 336: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; nl-nl) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12",
352 | 337: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.5.6 (KHTML, like Gecko) Safari/125.12",
353 | 338: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312",
354 | 339: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1.1 (KHTML, like Gecko) Safari/312",
355 | 340: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3",
356 | 341: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.5",
357 | 342: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6",
358 | 343: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412 (KHTML, like Gecko) Safari/412",
359 | 344: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/125.11",
360 | 345: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/412.2.2",
361 | 346: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.7 (KHTML, like Gecko) Safari/412.5",
362 | 347: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/416.11 (KHTML, like Gecko) Safari/416.12",
363 | 348: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.2",
364 | 349: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3",
365 | 350: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1",
366 | 351: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/523.10.3 (KHTML, like Gecko) Version/3.0.4 Safari/523.10",
367 | 352: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6",
368 | 353: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12 (KHTML, like Gecko) Version/3.0.4 Safari/523.12",
369 | 354: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12.2 (KHTML, like Gecko) Version/3.0.4 Safari/523.12.2",
370 | 355: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13",
371 | 356: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_3; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20",
372 | 357: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22",
373 | 358: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-jp) AppleWebKit/525.26.2 (KHTML, like Gecko) Version/3.2 Safari/525.26.12",
374 | 359: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1",
375 | 360: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16",
376 | 361: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-jp) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
377 | 362: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/531.21.11 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
378 | 363: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16",
379 | 364: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7",
380 | 365: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
381 | 366: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25",
382 | 367: "Mozilla/4.0 (Windows 95;US) Opera 3.62 [en]",
383 | 368: "Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 6.0 [en]",
384 | 369: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.01 [en]",
385 | 370: "Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 6.03 [en]",
386 | 371: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.05 [en]",
387 | 372: "Mozilla/4.0 (compatible; MSIE 5.0; Windows XP) Opera 6.06 [en]",
388 | 373: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.06 [en]",
389 | 374: "Opera 7.11",
390 | 375: "Opera/7.23 (Windows NT 5.0; U) [en]",
391 | 376: "Opera/7.52 (Windows NT 5.1; U) [en]",
392 | 377: "Opera/7.53 (Windows NT 5.0; U) [en]",
393 | 378: "Opera/7.54 (Windows NT 5.0; U) [en]",
394 | 379: "Opera/7.54 (Windows NT 5.1; U) [en]",
395 | 380: "Opera/7.54 (Windows NT 5.1; U)",
396 | 381: "Opera/7.54 (X11; Linux i686; U) [en]",
397 | 382: "Opera/7.54 (X11; Linux i686; U) [sv]",
398 | 383: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.11 [en]",
399 | 384: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.21 [pt-BR]",
400 | 385: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.22 [en]",
401 | 386: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]",
402 | 387: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]",
403 | 388: "Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.23 [en]",
404 | 389: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.23 [fr]",
405 | 390: "Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC) Opera 7.50 [en]",
406 | 391: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.53 [en]",
407 | 392: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.53 [en]",
408 | 393: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]",
409 | 394: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54 [en]",
410 | 395: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]",
411 | 396: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54u1 [en]",
412 | 397: "Opera/7.54 (Windows 98; U) [en]",
413 | 398: "Mozilla/4.78 (Windows NT 5.1; U) Opera 7.23 [en]",
414 | 399: "Mozilla/5.0 (Windows NT 5.0; U) Opera 7.54 [en]",
415 | 400: "Mozilla/4.0 (compatible; MSIE 6.0; X11; OpenBSD i386) Opera 7.54 [en]",
416 | 401: "Opera/8.0 (X11; Linux i686; U; en)",
417 | 402: "Opera/8.01 (Windows ME; U; en)",
418 | 403: "Opera/8.01 (Windows NT 5.1; U; en)",
419 | 404: "Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.01",
420 | 405: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.01",
421 | 406: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; tr) Opera 8.02",
422 | 407: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.02",
423 | 408: "Opera/8.5 (Windows NT 5.0; U; en)",
424 | 409: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.5",
425 | 410: "Opera/8.51 (Windows NT 5.1; U; en)",
426 | 411: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.51",
427 | 412: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; de) Opera 8.52",
428 | 413: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; pl) Opera 8.52",
429 | 414: "Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.52",
430 | 415: "Opera/8.53 (Windows NT 5.1; U; en)",
431 | 416: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.53",
432 | 417: "Opera/8.54 (Windows NT 5.1; U; en)",
433 | 418: "Opera/8.54 (Windows NT 5.0; U; en)",
434 | 419: "Mozilla/5.0 (X11; Linux i686; U; cs) Opera 8.54",
435 | 420: "Mozilla/4.0 (compatible; MSIE 6.0; KDDI-SA39) Opera 8.60 [en]",
436 | 421: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 9.00",
437 | 422: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.00",
438 | 423: "Opera/9.00 (Windows NT 5.1; U; en)",
439 | 424: "Opera/9.0 (Windows NT 5.1; U; en)",
440 | 425: "Opera/9.00 (Macintosh; PPC Mac OS X; U; en)",
441 | 426: "Opera/9.02 (Macintosh; PPC Mac OS X; U; en)",
442 | 427: "Opera/9.02 (Windows NT 5.1; U; zh-tw)",
443 | 428: "Opera/9.10 (Windows NT 6.0; U; en)",
444 | 429: "Opera/9.21 (Windows NT 6.0; U; en)",
445 | 430: "Opera/9.22 (Windows NT 5.1; U; en)",
446 | 431: "Opera/9.23 (Windows NT 5.1; U; en)",
447 | 432: "Opera/9.23 (Windows ME; U; en)",
448 | 433: "Opera/9.26 (Windows NT 5.1; U; en)",
449 | 434: "Opera/9.51 (Windows NT 5.1; U; en)",
450 | 435: "Opera/9.52 (Macintosh; Intel Mac OS X; U; en)",
451 | 436: "Opera/9.52 (Windows NT 5.1; U; en)",
452 | 437: "Opera/9.60 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1",
453 | 438: "Opera/9.60 (Windows NT 5.1; U; en) Presto/2.1.1",
454 | 439: "Opera/9.61 (Windows NT 5.1; U; en) Presto/2.1.1",
455 | 440: "Opera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1",
456 | 441: "Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; en) Opera 10.10",
457 | 442: "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.9.168 Version/11.50",
458 | 443: "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.10.229 Version/11.60",
459 | 444: "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.10.289 Version/12.00",
460 | }
461 |
462 | // this *.txt maybe not found if you exec binary, so we just fill several ua
463 | temp, err := util.ReadFromFile(util.CurDir() + "/config/ua.txt")
464 |
465 | if err == nil {
466 | uas := strings.Split(string(temp), "\n")
467 | for i, ua := range uas {
468 | Ua[i] = strings.TrimSpace(strings.Replace(ua, "\r", "", -1))
469 | }
470 | }
471 |
472 | }
473 |
474 | // RandomUa back random User-Agent
475 | func RandomUa() string {
476 | length := len(Ua)
477 | if length == 0 {
478 | return ""
479 | }
480 | return Ua[rand.Intn(length-1)]
481 | }
482 |
--------------------------------------------------------------------------------
/miner/util.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "net/http"
7 | "strings"
8 |
9 | "github.com/hunterhug/marmot/util"
10 | )
11 |
12 | // Wait some second
13 | func Wait(waitTime int) {
14 | if waitTime <= 0 {
15 | return
16 | } else {
17 | Logger.Debugf("[GoWorker] Wait %d Second.", waitTime)
18 | util.Sleep(waitTime)
19 | }
20 | }
21 |
22 | // CopyM Header map[string][]string ,can use to copy a http header, so that they are not effect each other
23 | func CopyM(h http.Header) http.Header {
24 | if h == nil || len(h) == 0 {
25 | return h
26 | }
27 | h2 := make(http.Header, len(h))
28 | for k, vv := range h {
29 | vv2 := make([]string, len(vv))
30 | copy(vv2, vv)
31 | h2[k] = vv2
32 | }
33 | return h2
34 | }
35 |
36 | // TooSortSizes if a file size smaller than sizes(KB), it will be thrown an error
37 | func TooSortSizes(data []byte, sizes float64) error {
38 | if float64(len(data))/1000 < sizes {
39 | return errors.New(fmt.Sprintf("FileSize:%d bytes,%d kb < %f kb dead too sort", len(data), len(data)/1000, sizes))
40 | }
41 | return nil
42 | }
43 |
44 | // OutputMaps Just debug a map
45 | func OutputMaps(uuid string, info string, args map[string][]string) {
46 | s := "\n"
47 | for k, v := range args {
48 | s = s + fmt.Sprintf("%-25s | %-6s\n", k, strings.Join(v, " |||| "))
49 | }
50 | Logger.Debugf("%s %s-%s", uuid, info, s)
51 | }
52 |
--------------------------------------------------------------------------------
/miner/worker.go:
--------------------------------------------------------------------------------
1 | package miner
2 |
3 | import (
4 | "bytes"
5 | "errors"
6 | "io"
7 | "net/http"
8 | "net/url"
9 | "strings"
10 |
11 | "github.com/hunterhug/marmot/util"
12 | "mime/multipart"
13 | )
14 |
15 | // NewWorkerByClient New Worker by Your Client
16 | func NewWorkerByClient(client *http.Client) *Worker {
17 | worker := new(Worker)
18 | worker.Request = newRequest()
19 | worker.Header = http.Header{}
20 | worker.Response = new(Response)
21 |
22 | worker.Client = client
23 | return worker
24 | }
25 |
26 | // NewAPI New API Worker, No Cookie Keep, share the same http client
27 | func NewAPI() *Worker {
28 | return NewWorkerByClient(NoCookieClient)
29 | }
30 |
31 | // NewWorker New a worker, if ipString is a proxy address, New a proxy client. evey time gen a new http client!
32 | // Proxy address such as:
33 | //
34 | // http://[user]:[password@]ip:port, [] stand it can choose or not. case: socks5://127.0.0.1:1080
35 | func NewWorker(proxyIpString interface{}) (*Worker, error) {
36 | if proxyIpString != nil {
37 | client, err := NewProxyClient(strings.ToLower(proxyIpString.(string)))
38 | if err != nil {
39 | return nil, err
40 | }
41 |
42 | worker := NewWorkerByClient(client)
43 | worker.Ip = proxyIpString.(string)
44 | return worker, err
45 | } else {
46 | client := NewClient()
47 | worker := NewWorkerByClient(client)
48 | worker.Ip = "localhost"
49 | return worker, nil
50 | }
51 | }
52 |
53 | // NewWorkerWithProxy Alias func
54 | func NewWorkerWithProxy(proxyIpString interface{}) (*Worker, error) {
55 | return NewWorker(proxyIpString)
56 | }
57 |
58 | // NewWorkerWithNoProxy Alias func
59 | func NewWorkerWithNoProxy() *Worker {
60 | w, _ := NewWorker(nil)
61 | return w
62 | }
63 |
64 | // New Alias Name for NewWorker
65 | func New(ipString interface{}) (*Worker, error) {
66 | return NewWorker(ipString)
67 | }
68 |
69 | // fast new request
70 | func newRequest() *Request {
71 | req := new(Request)
72 | req.Data = url.Values{}
73 | req.BData = []byte{}
74 | return req
75 | }
76 |
77 | // Clone Should Clone a new worker if you want to use repeat
78 | func (worker *Worker) Clone() *Worker {
79 | cloneWorker := NewWorkerByClient(worker.Client)
80 | cloneWorker.Ip = worker.Ip
81 | return cloneWorker
82 | }
83 |
84 | // Go Auto decide which method, Default Get.
85 | func (worker *Worker) Go() (body []byte, e error) {
86 | switch strings.ToUpper(worker.Method) {
87 | case POST:
88 | return worker.Post()
89 | case POSTJSON:
90 | return worker.PostJSON()
91 | case POSTXML:
92 | return worker.PostXML()
93 | case POSTFILE:
94 | return worker.PostFILE()
95 | case PUT:
96 | return worker.Put()
97 | case PUTJSON:
98 | return worker.PutJSON()
99 | case PUTXML:
100 | return worker.PutXML()
101 | case PUTFILE:
102 | return worker.PutFILE()
103 | case DELETE:
104 | return worker.Delete()
105 | case OTHER:
106 | return []byte(""), errors.New("please use method OtherGo(method, contentType string) or OtherGoBinary(method, contentType string)")
107 | default:
108 | return worker.Get()
109 | }
110 | }
111 |
112 | func (worker *Worker) GoByMethod(method string) (body []byte, e error) {
113 | return worker.SetMethod(method).Go()
114 | }
115 |
116 | // ToString This make effect only your worker exec serial! Attention!
117 | // Change Your Raw data To string
118 | func (worker *Worker) ToString() string {
119 | if worker.Raw == nil {
120 | return ""
121 | }
122 | return string(worker.Raw)
123 | }
124 |
125 | // JsonToString This make effect only your worker exec serial! Attention!
126 | // Change Your JSON' like Raw data to string
127 | func (worker *Worker) JsonToString() (string, error) {
128 | if worker.Raw == nil {
129 | return "", nil
130 | }
131 | temp, err := util.JsonBack(worker.Raw)
132 | if err != nil {
133 | return "", err
134 | }
135 | return string(temp), nil
136 | }
137 |
138 | // Main method I make!
139 | func (worker *Worker) sent(method, contentType string, binary bool) (body []byte, e error) {
140 | // Lock it for save
141 | worker.mux.Lock()
142 | defer worker.mux.Unlock()
143 |
144 | // Before Action, we can change or add something before Go()
145 | if worker.BeforeAction != nil {
146 | worker.BeforeAction(worker.Ctx, worker)
147 | }
148 |
149 | // Wait if must
150 | if worker.Wait > 0 {
151 | Wait(worker.Wait)
152 | }
153 |
154 | // For debug
155 | uuid := "[GoWorker]"
156 | Logger.Debugf("%s %s %s", uuid, method, worker.Url)
157 |
158 | // New a Request
159 | var request *http.Request
160 | var err error
161 |
162 | // If binary value is true and BData is not empty
163 | // suit for POSTJSON(), POSTFILE()
164 | if len(worker.BData) != 0 && binary {
165 | pr := bytes.NewReader(worker.BData)
166 | request, err = http.NewRequest(method, worker.Url, pr)
167 | } else if len(worker.Data) != 0 { // such POST() from table form
168 | pr := strings.NewReader(worker.Data.Encode())
169 | request, err = http.NewRequest(method, worker.Url, pr)
170 | } else {
171 | request, err = http.NewRequest(method, worker.Url, nil)
172 | }
173 |
174 | if err != nil {
175 | return nil, err
176 | }
177 |
178 | // Close avoid EOF
179 | // For client requests, setting this field prevents re-use of
180 | // TCP connections between requests to the same hosts, as if
181 | // Transport.DisableKeepAlives were set.
182 | // maybe you want long connection
183 | //request.Close = true
184 |
185 | // Clone Header, I add some HTTP header!
186 | request.Header = CloneHeader(worker.Header)
187 |
188 | // In fact content type must not empty
189 | if contentType != "" {
190 | request.Header.Set("Content-Type", contentType)
191 | }
192 | worker.Request.Request = request
193 |
194 | // Debug for RequestHeader
195 | OutputMaps(uuid, "Request Header", request.Header)
196 |
197 | // Tolerate abnormal way to create a Worker
198 | if worker.Client == nil {
199 | worker.Client = Client
200 | }
201 |
202 | // Do it
203 | response, err := worker.Client.Do(request)
204 | if err != nil {
205 | return nil, err
206 | }
207 |
208 | // Close it attention response may be nil
209 | if response != nil {
210 | //response.Close = true
211 | defer response.Body.Close()
212 | }
213 |
214 | // Debug
215 | OutputMaps(uuid, "Response Header", response.Header)
216 | Logger.Debugf("%s %s %s", uuid, response.Proto, response.Status)
217 |
218 | // Read output
219 | body, e = io.ReadAll(response.Body)
220 | worker.Raw = body
221 |
222 | worker.ResponseStatusCode = response.StatusCode
223 | worker.Response.Response = response
224 |
225 | // After action
226 | if worker.AfterAction != nil {
227 | worker.AfterAction(worker.Ctx, worker)
228 | }
229 |
230 | return
231 | }
232 |
233 | // Get method
234 | func (worker *Worker) Get() (body []byte, e error) {
235 | worker.Clear()
236 | return worker.sent(GET, "", false)
237 | }
238 |
239 | func (worker *Worker) Delete() (body []byte, e error) {
240 | worker.Clear()
241 | return worker.sent(DELETE, "", false)
242 | }
243 |
244 | // Post Almost include bellow:
245 | /*
246 | "application/x-www-form-urlencoded"
247 | "application/json"
248 | "text/xml"
249 | "multipart/form-data"
250 | */
251 | func (worker *Worker) Post() (body []byte, e error) {
252 | return worker.sent(POST, HTTPFORMContentType, false)
253 | }
254 |
255 | func (worker *Worker) PostJSON() (body []byte, e error) {
256 | return worker.sent(POST, HTTPJSONContentType, true)
257 | }
258 |
259 | func (worker *Worker) PostXML() (body []byte, e error) {
260 | return worker.sent(POST, HTTPXMLContentType, true)
261 | }
262 |
263 | func (worker *Worker) PostFILE() (body []byte, e error) {
264 | return worker.sentFile(POST)
265 |
266 | }
267 |
268 | var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
269 |
270 | func escapeQuotes(s string) string {
271 | return quoteEscaper.Replace(s)
272 | }
273 |
274 | func (worker *Worker) sentFile(method string) ([]byte, error) {
275 | if worker.FileName == "" || worker.FileFormName == "" {
276 | return nil, errors.New("fileName or fileFormName must not empty")
277 | }
278 |
279 | if len(worker.BData) == 0 {
280 | return nil, errors.New("BData must not empty")
281 | }
282 |
283 | bodyBuf := &bytes.Buffer{}
284 | bodyWriter := multipart.NewWriter(bodyBuf)
285 |
286 | fileWriter, err := bodyWriter.CreateFormFile(worker.FileFormName, worker.FileName)
287 | if err != nil {
288 | return nil, err
289 | }
290 |
291 | _, err = fileWriter.Write(worker.BData)
292 | if err != nil {
293 | return nil, err
294 | }
295 |
296 | for k, v := range worker.Data {
297 | for _, vv := range v {
298 | if vv != "" {
299 | bodyWriter.WriteField(k, vv)
300 | }
301 | }
302 | }
303 |
304 | contentType := bodyWriter.FormDataContentType()
305 |
306 | err = bodyWriter.Close()
307 | if err != nil {
308 | return nil, err
309 | }
310 |
311 | worker.SetBData(bodyBuf.Bytes())
312 |
313 | return worker.sent(method, contentType, true)
314 | }
315 |
316 | func (worker *Worker) Put() (body []byte, e error) {
317 | return worker.sent(PUT, HTTPFORMContentType, false)
318 | }
319 |
320 | func (worker *Worker) PutJSON() (body []byte, e error) {
321 | return worker.sent(PUT, HTTPJSONContentType, true)
322 | }
323 |
324 | func (worker *Worker) PutXML() (body []byte, e error) {
325 | return worker.sent(PUT, HTTPXMLContentType, true)
326 | }
327 |
328 | func (worker *Worker) PutFILE() (body []byte, e error) {
329 | return worker.sentFile(PUT)
330 | }
331 |
332 | /*
333 | OtherGo Method
334 |
335 | Method = "OPTIONS" ; Section 9.2
336 | | "GET" ; Section 9.3
337 | | "HEAD" ; Section 9.4
338 | | "POST" ; Section 9.5
339 | | "PUT" ; Section 9.6
340 | | "DELETE" ; Section 9.7
341 | | "TRACE" ; Section 9.8
342 | | "CONNECT" ; Section 9.9
343 | | extension-method
344 | extension-method = token
345 | token = 1*
346 |
347 | Content Type
348 |
349 | "application/x-www-form-urlencoded"
350 | "application/json"
351 | "text/xml"
352 | "multipart/form-data"
353 | */
354 | func (worker *Worker) OtherGo(method, contentType string) (body []byte, e error) {
355 | return worker.sent(method, contentType, false)
356 | }
357 |
358 | func (worker *Worker) OtherGoBinary(method, contentType string) (body []byte, e error) {
359 | return worker.sent(method, contentType, true)
360 | }
361 |
--------------------------------------------------------------------------------
/util/crypto.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "crypto/hmac"
5 | "crypto/md5"
6 | "crypto/sha256"
7 | "encoding/base64"
8 | "encoding/hex"
9 | "fmt"
10 | "io"
11 | "net/url"
12 | "strings"
13 | )
14 |
15 | // ComputeHmac256 HMAC with the SHA256
16 | func ComputeHmac256(message string, secret string) string {
17 | key := []byte(secret)
18 | h := hmac.New(sha256.New, key)
19 | h.Write([]byte(message))
20 | return base64.StdEncoding.EncodeToString(h.Sum(nil))
21 | }
22 |
23 | // StrToMd5 create md5 string
24 | func StrToMd5(s string) string {
25 | h := md5.New()
26 | h.Write([]byte(s))
27 | rs := hex.EncodeToString(h.Sum(nil))
28 | return rs
29 | }
30 |
31 | func Md5(str string) string {
32 | return StrToMd5(str)
33 | }
34 |
35 | func Base64E(body string) string {
36 | str := []byte(body)
37 | data := base64.StdEncoding.EncodeToString(str)
38 | return data
39 | }
40 |
41 | func Base64D(body string) string {
42 | data, err := base64.StdEncoding.DecodeString(body)
43 | if err != nil {
44 | return ""
45 | }
46 | s := fmt.Sprintf("%q", data)
47 | s = strings.Replace(s, "\"", "", -1)
48 | return s
49 | }
50 |
51 | func UrlE(s string) string {
52 | return url.QueryEscape(s)
53 | }
54 |
55 | func UrlD(s string) string {
56 | s, e := url.QueryUnescape(s)
57 | if e != nil {
58 | return e.Error()
59 | } else {
60 | return s
61 | }
62 | }
63 |
64 | func Md5FS(src io.Reader) string {
65 | h := md5.New()
66 | if err := CopyFF(src, h); err != nil {
67 | fmt.Println(err.Error())
68 | return ""
69 | }
70 | return fmt.Sprintf("%x", h.Sum([]byte("hunterhug")))
71 | }
72 |
--------------------------------------------------------------------------------
/util/file.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "errors"
5 | "io"
6 | "os"
7 | "os/exec"
8 | "path/filepath"
9 | "runtime"
10 | "strings"
11 | )
12 |
13 | func CurDir() string {
14 | _, filename, _, _ := runtime.Caller(1)
15 | return filepath.Dir(filename)
16 | }
17 |
18 | func GetBinaryCurrentPath() (string, error) {
19 | file, err := exec.LookPath(os.Args[0])
20 | if err != nil {
21 | return "", err
22 | }
23 |
24 | path, err := filepath.Abs(file)
25 | if err != nil {
26 | return "", err
27 | }
28 |
29 | if strings.Contains(path, "command-line-arguments") {
30 | return GetCurrentPath()
31 | }
32 |
33 | i := strings.LastIndex(path, "/")
34 | if i < 0 {
35 | i = strings.LastIndex(path, "\\")
36 | }
37 |
38 | if i < 0 {
39 | return "", errors.New(`error: Can't find "/" or "\"`)
40 | }
41 |
42 | return path[0 : i+1], nil
43 | }
44 |
45 | func GetCurrentPath() (string, error) {
46 | return os.Getwd()
47 | }
48 |
49 | func SaveToFile(filePath string, content []byte) error {
50 | err := os.WriteFile(filePath, content, 0777)
51 | return err
52 | }
53 |
54 | func ReadFromFile(filepath string) ([]byte, error) {
55 | return os.ReadFile(filepath)
56 | }
57 |
58 | func GetFilenameInfo(filepath string) (os.FileInfo, error) {
59 | info, err := os.Stat(filepath)
60 | return info, err
61 | }
62 |
63 | func Rename(oldFilename string, newFilename string) error {
64 | return os.Rename(oldFilename, newFilename)
65 | }
66 |
67 | func MakeDir(fileDir string) error {
68 | return os.MkdirAll(fileDir, 0777)
69 | }
70 |
71 | func MakeDirByFile(filepath string) error {
72 | temp := strings.Split(filepath, "/")
73 | if len(temp) <= 2 {
74 | return errors.New("please input complete file name like ./dir/filename or /home/dir/filename")
75 | }
76 | dirPath := strings.Join(temp[0:len(temp)-1], "/")
77 | return MakeDir(dirPath)
78 | }
79 |
80 | func FileExist(filename string) bool {
81 | fi, err := os.Stat(filename)
82 | if err != nil {
83 | return false
84 | }
85 | if fi.IsDir() {
86 | return false
87 | } else {
88 | return true
89 | }
90 | }
91 |
92 | func WalkDir(dirPth, suffix string) (files []string, err error) {
93 | files = make([]string, 0, 30)
94 | suffix = strings.ToUpper(suffix)
95 | err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error {
96 | if fi.IsDir() {
97 | return nil
98 | }
99 | if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
100 | files = append(files, filename)
101 | }
102 | return nil
103 | })
104 | return files, err
105 | }
106 |
107 | func ListDir(dirPth string, suffix string) (files []string, err error) {
108 | files = make([]string, 0, 10)
109 | dir, err := os.ReadDir(dirPth)
110 | if err != nil {
111 | return nil, err
112 | }
113 |
114 | suffix = strings.ToUpper(suffix)
115 |
116 | for _, fi := range dir {
117 | if fi.IsDir() {
118 | continue
119 | }
120 |
121 | if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
122 | files = append(files, dirPth+"/"+fi.Name())
123 | }
124 | }
125 |
126 | return files, nil
127 | }
128 |
129 | func ListDirOnlyName(dirPth string, suffix string) (files []string, err error) {
130 | files = make([]string, 0, 10)
131 |
132 | dir, err := os.ReadDir(dirPth)
133 | if err != nil {
134 | return nil, err
135 | }
136 |
137 | suffix = strings.ToUpper(suffix)
138 |
139 | for _, fi := range dir {
140 | if fi.IsDir() {
141 | continue
142 | }
143 | if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
144 | files = append(files, fi.Name())
145 | }
146 | }
147 |
148 | return files, nil
149 | }
150 |
151 | func HasFile(s string) bool {
152 | f, err := os.Open(s)
153 | if err != nil && os.IsNotExist(err) {
154 | return false
155 | }
156 | f.Close()
157 | return true
158 | }
159 |
160 | func CopyFF(src io.Reader, dst io.Writer) error {
161 | _, err := io.Copy(dst, src)
162 | return err
163 | }
164 |
165 | func CopyFS(src io.Reader, dst string) error {
166 | f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
167 | if err != nil {
168 | return err
169 | }
170 | defer f.Close()
171 | _, err = io.Copy(f, src)
172 | return err
173 | }
174 |
175 | func IsFile(filepath string) bool {
176 | info, err := os.Stat(filepath)
177 | if err != nil {
178 | return false
179 | } else {
180 | if info.IsDir() {
181 | return false
182 | } else {
183 | return true
184 | }
185 | }
186 | }
187 |
188 | func IsDir(filepath string) bool {
189 | info, err := os.Stat(filepath)
190 | if err != nil {
191 | return false
192 | } else {
193 | if info.IsDir() {
194 | return true
195 | } else {
196 | return false
197 | }
198 | }
199 | }
200 |
201 | func SizeofDir(dirPth string) int {
202 | if IsDir(dirPth) {
203 | files, _ := os.ReadDir(dirPth)
204 | return len(files)
205 | }
206 |
207 | return 0
208 | }
209 |
210 | func GetFileSuffix(f string) string {
211 | fa := strings.Split(f, ".")
212 | if len(fa) == 0 {
213 | return ""
214 | } else {
215 | return fa[len(fa)-1]
216 | }
217 | }
218 |
219 | func ValidFileName(filename string) string {
220 | patterns := []string{
221 | " ", "#01#",
222 | "\\", "#02#",
223 | "/", "#03#",
224 | ":", "#04#",
225 | "\"", "#05#",
226 | "?", "#06#",
227 | "<", "#07#",
228 | ">", "#08#",
229 | "|", "#09#",
230 | }
231 | r := strings.NewReplacer(patterns...)
232 | dudu := r.Replace(filename)
233 | return dudu
234 | }
235 |
236 | func ValidBackFileName(filename string) string {
237 | patterns := []string{
238 | "#01#", " ",
239 | "#02#", "\\",
240 | "#03#", "/",
241 | "#04#", ":",
242 | "#05#", "\"",
243 | "#06#", "?",
244 | "#07#", "<",
245 | "#08#", ">",
246 | "#09#", "|",
247 | }
248 | r := strings.NewReplacer(patterns...)
249 | dudu := r.Replace(filename)
250 | return dudu
251 | }
252 |
--------------------------------------------------------------------------------
/util/io.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "fmt"
5 | "strings"
6 | )
7 |
8 | func Input(say, defaults string) string {
9 | fmt.Println(say)
10 |
11 | var str string
12 | _, _ = fmt.Scanln(&str)
13 |
14 | if strings.TrimSpace(str) == "" {
15 | if strings.TrimSpace(defaults) != "" {
16 | return defaults
17 | } else {
18 | fmt.Println("can not empty")
19 | return Input(say, defaults)
20 | }
21 | }
22 |
23 | return str
24 | }
25 |
--------------------------------------------------------------------------------
/util/json.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "encoding/json"
5 | )
6 |
7 | func JsonBack(s []byte) ([]byte, error) {
8 | temp := new(interface{})
9 |
10 | err := json.Unmarshal(s, temp)
11 | if err != nil {
12 | return nil, err
13 | }
14 |
15 | result, err := json.Marshal(temp)
16 | return result, err
17 | }
18 |
--------------------------------------------------------------------------------
/util/time.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "time"
5 | )
6 |
7 | func Sleep(waitTime int) {
8 | time.Sleep(time.Duration(waitTime) * time.Second)
9 | }
10 |
11 | func Second(times int) time.Duration {
12 | return time.Duration(times) * time.Second
13 | }
14 |
15 | func TodayString(level int) string {
16 | formats := "20060102150405"
17 | switch level {
18 | case 1:
19 | formats = "2006"
20 | case 2:
21 | formats = "200601"
22 | case 3:
23 | formats = "20060102"
24 | case 4:
25 | formats = "2006010215"
26 | case 5:
27 | formats = "200601021504"
28 | default:
29 |
30 | }
31 | return time.Now().Format(formats)
32 | }
33 |
--------------------------------------------------------------------------------
/util/util.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "errors"
5 | "strconv"
6 | "strings"
7 | )
8 |
9 | func SI(s string) (i int, e error) {
10 | i, e = strconv.Atoi(s)
11 | return
12 | }
13 |
14 | func IS(i int) string {
15 | return strconv.Itoa(i)
16 | }
17 |
18 | func ToLower(s string) string {
19 | return strings.ToLower(s)
20 | }
21 |
22 | func DivideStringList(files []string, num int) (map[int][]string, error) {
23 | length := len(files)
24 | split := map[int][]string{}
25 | if num <= 0 {
26 | return split, errors.New("num must not negative")
27 | }
28 |
29 | if num > length {
30 | num = length
31 | }
32 |
33 | process := length / num
34 | for i := 0; i < num; i++ {
35 | split[i] = append(split[i], files[i*process:(i+1)*process]...)
36 | }
37 |
38 | remain := files[num*process:]
39 | for i := 0; i < len(remain); i++ {
40 | split[i%num] = append(split[i%num], remain[i])
41 | }
42 |
43 | return split, nil
44 | }
45 |
46 | func InArray(sa []string, a string) bool {
47 | for _, v := range sa {
48 | if a == v {
49 | return true
50 | }
51 | }
52 | return false
53 | }
54 |
55 | func Substr(str string, start, length int) string {
56 | rs := []rune(str)
57 | rl := len(rs)
58 | end := 0
59 |
60 | if start < 0 {
61 | start = rl - 1 + start
62 | }
63 |
64 | end = start + length
65 |
66 | if start > end {
67 | start, end = end, start
68 | }
69 |
70 | if start < 0 {
71 | start = 0
72 | }
73 |
74 | if start > rl {
75 | start = rl
76 | }
77 |
78 | if end < 0 {
79 | end = 0
80 | }
81 |
82 | if end > rl {
83 | end = rl
84 | }
85 |
86 | return string(rs[start:end])
87 | }
88 |
--------------------------------------------------------------------------------