├── _test ├── conf.yaml ├── todo_queue_test.go ├── proxy_test.go ├── stress_testing_test.go ├── conf_helper_test.go ├── slice_helper_test.go ├── scan_test.go ├── time_helper_test.go ├── req_test.go ├── base_test.go ├── map_helper_test.go ├── string_helper_test.go └── reg_test.go ├── _examples ├── cnlinfo │ ├── conf.yaml │ └── main.go ├── intercept │ ├── logo_16.ico │ └── main.go ├── tjbz │ ├── tjypflml.go │ └── tjyqhdmhcxhfdm.go ├── socket5 │ └── main.go ├── all_school │ └── main.go ├── search │ └── main.go ├── gkcx_eol_cn │ └── main.go ├── db_case │ └── main.go ├── mq │ └── main.go ├── qgyyzs │ └── main.go ├── upload_file │ └── main.go ├── baidu_tk │ └── main.go ├── ip_bczs_cn │ ├── go.mod │ └── main.go ├── get │ └── get.go ├── qihuo │ └── main.go ├── json │ └── main.go ├── weibo │ └── main.go ├── douyin │ └── main.go ├── websocket_client │ └── main.go └── baojia │ └── main.go ├── .gitignore ├── LICENSE ├── mail_helper.go ├── conf_helper.go ├── go.mod ├── id_helper.go ├── extract.go ├── stress_testing.go ├── logger.go ├── time_helper.go ├── scan.go ├── gathertool.go ├── job.go ├── out_helper.go ├── go.sum ├── data_structure.go ├── README.md ├── crypto_helper.go ├── string_helper.go └── conversion.go /_test/conf.yaml: -------------------------------------------------------------------------------- 1 | test: 2 | case: 123 -------------------------------------------------------------------------------- /_examples/cnlinfo/conf.yaml: -------------------------------------------------------------------------------- 1 | # 抓取结果输出文件 2 | out_file: "fenlei.json" 3 | # 请求延时 4 | req_delay_min : 1 5 | req_delay_max : 3 -------------------------------------------------------------------------------- /_examples/intercept/logo_16.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mangenotwork/gathertool/HEAD/_examples/intercept/logo_16.ico -------------------------------------------------------------------------------- /_examples/tjbz/tjypflml.go: -------------------------------------------------------------------------------- 1 | /* 2 | 国家统计局-统计用产品分类目录 ： http://www.stats.gov.cn/tjsj/tjbz/tjypflml/ 3 | */ 4 | 5 | package main 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .idea/ 3 | .vscode/ 4 | dist/ 5 | build/ 6 | vendor/ 7 | logs/ 8 | *.exe 9 | *.exe~ 10 | *.log 11 | *.db 12 | -------------------------------------------------------------------------------- /_examples/tjbz/tjyqhdmhcxhfdm.go: -------------------------------------------------------------------------------- 1 | /* 2 | 国家统计局-统计用区划和城乡划分代码 ： http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/ 3 | */ 4 | 5 | package main 6 | 7 | -------------------------------------------------------------------------------- /_test/todo_queue_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestTodoQueue(t *testing.T) { 8 | // TODO... 9 | } 10 | -------------------------------------------------------------------------------- /_examples/socket5/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | ) 6 | 7 | func main(){ 8 | gt.SockerProxy(":8111") 9 | } -------------------------------------------------------------------------------- /_test/proxy_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestSocketProxy(t *testing.T) { 9 | gt.SocketProxy("0.0.0.0:16666") 10 | } 11 | -------------------------------------------------------------------------------- /_examples/all_school/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | 抓取学校大全 5 | http://anshan.xuexiaodaquan.com/youeryuan/pn2.html 6 | 7 | 中学学校大全 8 | http://xuexiao.51sxue.com/slist/?o=&t=3&areaCodeS=&level=&sp=&score=&order=&areaS=%CB%C4%B4%A8%CA%A1&searchKey= 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /_test/stress_testing_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestStressTesting(t *testing.T) { 9 | test := gt.NewTestUrl("https://baidu.com", "Get", 20, 10) 10 | test.Run() 11 | } 12 | 13 | // TODO 14 | func TestScanNewHostScanUrl(t *testing.T) { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /_test/conf_helper_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | // go test -v -run=TestConf 9 | func TestConf(t *testing.T) { 10 | _ = gt.NewConf("./conf.yaml") 11 | t.Log(gt.Config.Get("test")) 12 | t.Log(gt.Config.GetString("test::case")) 13 | t.Log(gt.Config.GetAll()) 14 | } 15 | -------------------------------------------------------------------------------- /_test/slice_helper_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestSliceHelper(t *testing.T) { 9 | t1 := []string{"1", "2", "3", "4", "4", "3"} 10 | t.Log(gt.SliceContains[string](t1, "1")) 11 | 12 | t.Log(gt.SliceDeduplicate[string](t1)) 13 | 14 | t2 := []int{1, 2, 3, 4} 15 | t.Log(gt.SliceDel[int](t2, 1)) 16 | 17 | t.Log(gt.SliceMax[int](t2)) 18 | 19 | t.Log(gt.SliceMin[int](t2)) 20 | 21 | t.Log(gt.SlicePop[int](t2)) 22 | 23 | t.Log(t1) 24 | t.Log(gt.SliceReverse[string](t1)) 25 | 26 | t.Log(gt.SliceShuffle[string](t1)) 27 | 28 | t.Log(gt.SliceCopy[string](t1)) 29 | } 30 | -------------------------------------------------------------------------------- /_examples/intercept/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "log" 6 | ) 7 | 8 | func main(){ 9 | ipt := >.Intercept{ 10 | Ip : "0.0.0.0:8111", 11 | HttpPackageFunc: func(pack *gt.HttpPackage){ 12 | // 查看 ContentType 13 | log.Println("ContentType = ", pack.ContentType) 14 | // 获取数据包数据为 http,json等 TXT格式的数据 15 | log.Println("Txt = ", pack.Html()) 16 | // 获取数据包为图片，将图片转为 base64 17 | log.Println("img base64 = ", pack.Img2Base64()) 18 | // 获取数据包为图片，存储图片 19 | log.Println(pack.SaveImage("")) 20 | }, 21 | } 22 | // 启动服务 23 | ipt.RunServer() 24 | } 25 | 26 | /* 27 | golang实现http&https代理服务器 28 | golang实现http代理服务器并解析包 29 | */ -------------------------------------------------------------------------------- /_examples/search/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | 1. ip 扫描 3 | 2. DOS 攻击 4 | 3. DDOS 攻击 5 | 4. arp 6 | 5. ip的端口扫描 7 | 8 | */ 9 | package main 10 | 11 | import ( 12 | gt "github.com/mangenotwork/gathertool" 13 | "time" 14 | ) 15 | 16 | func main(){ 17 | //gt.SearchDomain("110.242.68.3") 18 | //gt.SearchPort("120.79.88.59", 5*time.Second) 19 | //gt.Ping("192.168.0.23") 20 | // 21 | //for i:=0; i<256; i++{ 22 | // gt.Ping(fmt.Sprintf("192.168.0.%d", i)) 23 | //} 24 | 25 | for { 26 | gt.Ping("120.79.88.59") 27 | time.Sleep(1*time.Second) 28 | } 29 | 30 | } 31 | 32 | 33 | 34 | /* 35 | 搜索引擎采集策略 36 | - 种子url 扩增扫描 37 | - ip 扫描到站点 38 | - 各大搜索引擎抓取 39 | 40 | 内容识别字段: title, description,

...

, keywords, site_name, property="og:..., 41 | */ 42 | -------------------------------------------------------------------------------- /_test/scan_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestApplicationTerminalOutClose(t *testing.T) { 9 | gt.ApplicationTerminalOutClose() 10 | } 11 | 12 | func TestNewHostScanUrl(t *testing.T) { 13 | scan := gt.NewHostScanUrl("http://127.0.0.1:8881/home", 3) 14 | gt.Info(scan.Run()) 15 | } 16 | 17 | func TestNewHostScanExtLinks(t *testing.T) { 18 | scan := gt.NewHostScanExtLinks("http://127.0.0.1:8881/home") 19 | gt.Info(scan.Run()) 20 | } 21 | 22 | func TestNewHostScanBadLink(t *testing.T) { 23 | scan := gt.NewHostScanBadLink("http://127.0.0.1:8881/home", 3) 24 | gt.Info(scan.Run()) 25 | } 26 | 27 | func TestNewHostPageSpeedCheck(t *testing.T) { 28 | scan := gt.NewHostPageSpeedCheck("http://127.0.0.1:8881/home", 3) 29 | gt.Info(scan.Run()) 30 | } 31 | -------------------------------------------------------------------------------- /_examples/gkcx_eol_cn/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "log" 6 | ) 7 | 8 | func main() { 9 | getSchoolList() 10 | 11 | } 12 | 13 | func getSchoolList() { 14 | caseUrl := "https://api.eol.cn/gkcx/api/?access_token=&admissions=¢ral=&department=&dual_class=&f211=&f985=&is_doublehigh=&is_dual_class=&keyword=&nature=&page=1&province_id=&ranktype=&request_type=1&school_type=&signsafe=&size=20&sort=view_total&top_school_id=[]&type=&uri=apidata/api/gk/school/lists" 15 | //ctx := gt.PostJson(caseUrl, "")//.SetSucceedFunc(getSchoolListSuueed) 16 | //log.Println(ctx.String()) 17 | 18 | gt.PostJson(caseUrl, "", gt.SucceedFunc(getSchoolListSuueed)) 19 | 20 | } 21 | 22 | func getSchoolListSuueed(ctx *gt.Context) { 23 | //log.Println("ctx = ", ctx) 24 | log.Println(ctx.RespBodyString()) 25 | log.Println(ctx.CheckReqMd5()) 26 | log.Println(ctx.CheckMd5()) 27 | } 28 | -------------------------------------------------------------------------------- /_examples/db_case/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | ) 6 | 7 | var ( 8 | host192 = "192.168.0.192" 9 | port = 3306 10 | user = "root" 11 | password = "root123" 12 | spiderBase = "spider-base" 13 | SpiderBaseDB, _ = gt.NewMysql(host192, port, user, password, spiderBase) 14 | 15 | ) 16 | 17 | func main(){ 18 | //case1() 19 | 20 | case2() 21 | } 22 | 23 | func case1(){ 24 | data := map[string]interface{}{ 25 | "num":123, 26 | "txt":"aaaaa'aaaaa", 27 | "txt2":`bbbb"bbb'bbbbbb'bbb"bbbbb`, 28 | } 29 | err := SpiderBaseDB.InsertAt("test", data) 30 | if err != nil { 31 | panic(err) 32 | } 33 | } 34 | 35 | func case2(){ 36 | gd := gt.NewGDMap().Add("num", 123).Add("txt", "aaaaa'aaaaa") 37 | gd.Add("txt2", `bbbb"bbb'bbbbbb'bbb"bbbbb`) 38 | err := SpiderBaseDB.InsertAtGd("test2", gd) 39 | if err != nil { 40 | panic(err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /_test/time_helper_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestTimeHelperTest(t *testing.T) { 10 | t.Log(gt.Timestamp2Date(gt.Timestamp())) 11 | t.Log(gt.TimestampStr()) 12 | t.Log(gt.Timestamp2Date(gt.BeginDayUnix())) 13 | t.Log(gt.Timestamp2Date(gt.EndDayUnix())) 14 | t.Log(gt.Timestamp2Date(gt.MinuteAgo(10))) 15 | t.Log(gt.Timestamp2Date(gt.HourAgo(1))) 16 | t.Log(gt.Timestamp2Date(gt.DayAgo(2))) 17 | t.Log(gt.DayDiff("2023-11-23 00:00:00", "2023-12-23 00:00:00")) 18 | t.Log(gt.DayDiffAtUnix(gt.DayAgo(10), gt.Timestamp())) 19 | t.Log(gt.NowToEnd()) 20 | t.Log(gt.IsToday(gt.DayAgo(10))) 21 | t.Log(gt.IsTodayList(gt.DayAgo(10))) 22 | t.Log(gt.Timestamp2Week(gt.Timestamp())) 23 | t.Log(gt.Timestamp2WeekXinQi(gt.Timestamp())) 24 | t.Log(gt.LatestDate(10)) 25 | t.Log(gt.GetCurrentMonthRange()) 26 | t.Log(gt.GetCurrentWeekRange()) 27 | t.Log(gt.GetTodayRange()) 28 | } 29 | 30 | func TestTimeHelperTickerRunTest(t *testing.T) { 31 | gt.TickerRun(1*time.Second, true, func() { 32 | gt.Info("test...") 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ManGe(漫鸽) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /_test/req_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestReq(t *testing.T) { 9 | ctx, err := gt.Request("https://www.doubao.com/", "get", []byte(""), "") 10 | if err != nil { 11 | t.Fatal(err) 12 | } 13 | t.Log(ctx.Html) 14 | 15 | ctx = gt.NewRequest("https://www.doubao.com/", "get", []byte(""), "") 16 | if ctx.Err != nil { 17 | t.Fatal(err) 18 | } 19 | t.Log(ctx.Html) 20 | 21 | t.Log(gt.IsUrl("https://www.doubao.com/")) 22 | t.Log(gt.UrlStr("www.doubao.com/")) 23 | 24 | gt.SetSleepMs(1, 3) 25 | 26 | ck := gt.NewCookie(map[string]string{ 27 | "case1": "1", 28 | "case2": "2", 29 | }) 30 | t.Log(ck.Delete("case1")) 31 | t.Log(ck.Get("case1")) 32 | 33 | gt.SearchDomain("180.97.248.210") 34 | 35 | gt.SearchPort("10.0.40.3") 36 | 37 | gt.ClosePingTerminalPrint() 38 | 39 | gt.GetCertificateInfo("https://www.doubao.com/") 40 | 41 | gt.SetStatusCodeSuccessEvent(203) 42 | gt.SetStatusCodeRetryEvent(401) 43 | gt.SetStatusCodeFailEvent(403) 44 | gt.SetAgent(1, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36") 45 | } 46 | -------------------------------------------------------------------------------- /_examples/mq/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | ) 6 | 7 | func main() { 8 | 9 | NsqProducer() 10 | 11 | //NsqConsumer() 12 | 13 | } 14 | 15 | func NsqProducer() { 16 | mq := gt.NewNsq("127.0.0.1") 17 | topic := "test" 18 | data := []byte("data") 19 | mq.Producer(topic, data) 20 | } 21 | 22 | func NsqConsumer() { 23 | topic := "test" 24 | mq := gt.NewNsq("127.0.0.1") 25 | for { 26 | data := mq.Consumer(topic) 27 | gt.Info(string(data)) 28 | } 29 | 30 | } 31 | 32 | func RabbitProducer() { 33 | mq := gt.NewRabbit("amqp://admin:123456@127.0.0.1:5672") 34 | topic := "test" 35 | data := []byte("data") 36 | mq.Producer(topic, data) 37 | } 38 | 39 | func RabbitConsumer() { 40 | topic := "test" 41 | mq := gt.NewRabbit("amqp://admin:123456@127.0.0.1:5672") 42 | data := mq.Consumer(topic) 43 | gt.Info(string(data)) 44 | } 45 | 46 | func KafkaProducer() { 47 | mq := gt.NewKafka([]string{"192.168.4.12:9092"}) 48 | topic := "test" 49 | data := []byte("data") 50 | mq.Producer(topic, data) 51 | } 52 | 53 | func KafkaConsumer() { 54 | topic := "test" 55 | mq := gt.NewKafka([]string{"192.168.4.12:9092"}) 56 | data := mq.Consumer(topic) 57 | gt.Info(string(data)) 58 | } 59 | -------------------------------------------------------------------------------- /_examples/qgyyzs/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | 环球医药网 爬虫 3 | http://data.qgyyzs.net 4 | 5 | 药智数据 6 | https://db.yaozh.com/instruct?p=1&pageSize=20 7 | */ 8 | package main 9 | 10 | import ( 11 | "github.com/PuerkitoBio/goquery" 12 | gt "github.com/mangenotwork/gathertool" 13 | "log" 14 | "net/http" 15 | "strings" 16 | ) 17 | 18 | func main() { 19 | //GcypList() 20 | 21 | //HospitalList() 22 | 23 | Instruct() 24 | } 25 | 26 | // GcypList 抓取国产药品列表 27 | func GcypList() { 28 | caseUrl := "http://data.qgyyzs.net/gcyp_1.aspx" 29 | ctx, _ := gt.Get(caseUrl) 30 | log.Println(ctx.RespBodyString()) 31 | } 32 | 33 | // HospitalList 抓取医院列表 34 | func HospitalList() { 35 | caseUrl := "http://data.qgyyzs.net/hospital_1.aspx" 36 | head := http.Header{} 37 | ctx, _ := gt.Get(caseUrl, head) 38 | htmlStr := gt.ConvertByte2String(ctx.RespBody, gt.GBK) 39 | //log.Println(htmlStr) 40 | dom, err := goquery.NewDocumentFromReader(strings.NewReader(htmlStr)) 41 | if err != nil { 42 | log.Println(err) 43 | return 44 | } 45 | result := dom.Find("div[class=content]") 46 | resultStr, _ := result.Html() 47 | for _, v := range gt.RegHtmlLi(resultStr) { 48 | log.Println(v) 49 | } 50 | } 51 | 52 | // Instruct 抓取药品说明书列表 53 | func Instruct() { 54 | caseUrl := "https://db.yaozh.com/instruct?p=1&pageSize=20" 55 | ctx, _ := gt.Get(caseUrl) 56 | log.Println(ctx.RespBodyString()) 57 | } 58 | -------------------------------------------------------------------------------- /mail_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 邮件的方法，应用场景有抓取完成通知. TODO 改为通知，新增webhook 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "gopkg.in/gomail.v2" 11 | ) 12 | 13 | type Mail struct { 14 | From string 15 | 16 | // 邮件授权码 17 | AuthCode string 18 | 19 | // QQ 邮箱： SMTP 服务器地址：smtp.qq.com（SSL协议端口：465/587, 非SSL协议端口：25） 20 | // 163 邮箱：SMTP 服务器地址：smtp.163.com（SSL协议端口：465/994，非SSL协议端口：25） 21 | Host string 22 | Port int 23 | C *gomail.Dialer 24 | Msg *gomail.Message 25 | } 26 | 27 | func NewMail(host, from, auth string, port int) *Mail { 28 | m := &Mail{ 29 | From: from, 30 | AuthCode: auth, 31 | Host: host, 32 | Port: port, 33 | } 34 | m.C = gomail.NewDialer( 35 | m.Host, 36 | m.Port, 37 | m.From, 38 | m.AuthCode, 39 | ) 40 | m.Msg = gomail.NewMessage() 41 | m.Msg.SetHeader("From", from) 42 | return m 43 | } 44 | 45 | func (m *Mail) Title(title string) *Mail { 46 | m.Msg.SetHeader("Subject", title) 47 | return m 48 | } 49 | 50 | func (m *Mail) HtmlBody(body string) *Mail { 51 | m.Msg.SetBody("text/html", body) 52 | return m 53 | } 54 | 55 | func (m *Mail) Send(to string) error { 56 | m.Msg.SetHeader("To", to) 57 | return m.C.DialAndSend(m.Msg) 58 | } 59 | 60 | func (m *Mail) SendMore(to []string) error { 61 | mgs := make([]*gomail.Message, 0) 62 | for _, v := range to { 63 | newMsg := m.Msg 64 | newMsg.SetHeader("To", v) 65 | mgs = append(mgs, newMsg) 66 | } 67 | return m.C.DialAndSend(mgs...) 68 | } 69 | -------------------------------------------------------------------------------- /_examples/upload_file/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "log" 6 | ) 7 | 8 | var fileLinks = map[string]string{ 9 | "https://jyzd.bfsu.edu.cn/uploadfile/bfsu/front/default/upload_file_35256.pdf": "/home/mange/Desktop/upload_file_1.pdf", 10 | "http://www.cuc.edu.cn/_upload/article/files/c8/44/741ea34245acbc7b8aa9b7660c0f/579130b8-3974-4c9d-99a3-19f4e4e095d1.pdf": "/home/mange/Desktop/upload_file_2.pdf", 11 | "https://job.buct.edu.cn/_upload/article/files/a0/b9/84fd06284b078e8051163a221546/2e302909-e81e-4aea-aa94-c5f37179cab0.pdf": "/home/mange/Desktop/upload_file_3.pdf", 12 | "http://kr.shanghai-jiuxin.com/file/2020/1031/774218be86d832f359637ab120eba52d.jpg": "/home/mange/Desktop/upload_file_1.jpg", 13 | "http://job.szu.edu.cn/UpLoadFile/InfoCenter/14247/File/20210119100014406.pdf": "/home/mange/Desktop/upload_file_4.pdf", 14 | } 15 | 16 | func main() { 17 | // 普通下载 18 | ctx, err := gt.Upload("https://jyzd.bfsu.edu.cn/uploadfile/bfsu/front/default/upload_file_35256.pdf", "/home/mange/Desktop/upload_file_3.pdf") 19 | log.Println(ctx.Resp.StatusCode, err) 20 | //for k,v := range fileLinks{ 21 | // gt.Upload(k,v) 22 | //} 23 | 24 | //// 并发下载 25 | //queue := gt.NewUploadQueue() 26 | //for k,v := range fileLinks{ 27 | // queue.Add(>.Task{ 28 | // Url: k, 29 | // SavePath: v, 30 | // Type: "upload", 31 | // }) 32 | //} 33 | //gt.StartJobGet(5, queue) 34 | 35 | } 36 | -------------------------------------------------------------------------------- /_examples/baidu_tk/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | 百度题库抓取 3 | */ 4 | 5 | package main 6 | 7 | func main() { 8 | //getZJ() 9 | } 10 | 11 | //// １．获取每一章节对应的目录 12 | //func getZJ(){ 13 | // caseUrl := "https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-16-knowpoint" 14 | // c, err := gt.Get(caseUrl) 15 | // if err != nil { 16 | // panic(err) 17 | // } 18 | // 19 | // dom,err := goquery.NewDocumentFromReader(strings.NewReader(c.RespBodyString())) 20 | // if err != nil{ 21 | // log.Println(err) 22 | // return 23 | // } 24 | // 25 | // dom.Find("div[class=detail]").Each(func(i int, div *goquery.Selection){ 26 | // chapter := div.Find("div[class=detail-chapter]") 27 | // chapterHtml, _ := chapter.Html() 28 | // 29 | // // 章节名称 30 | // zjTitle := gt.RegHtmlH(chapterHtml, "3") 31 | // log.Println("章节名称 ==> ", zjTitle) 32 | // 33 | // chapter.Each(func(i int, div2 *goquery.Selection){ 34 | // kpoint := div2.Find("div[class=detail-kpoint-1]") 35 | // kpointHtml, _ := kpoint.Html() 36 | // 37 | // // 小节名称 38 | // xjTitle := gt.RegHtmlH(kpointHtml, "4") 39 | // log.Println("小节名称 ==> ", xjTitle) 40 | // 41 | // // 获取课程 42 | // kpoint.Each(func(i int, div3 *goquery.Selection){ 43 | // kpoint2 := div3.Find("div[class=detail-kpoint-2]") 44 | // kpoint2Html,_ := kpoint2.Html() 45 | // 46 | // //课程名称 47 | // kcTitle := gt.RegHtmlHTxt(kpoint2Html, "5") 48 | // log.Println("课程名称 ==> ", kcTitle) 49 | // //课程链接 50 | // kcLink := gt.RegHtmlHrefTxt(kpoint2Html) 51 | // if len(kcLink) > 0 { 52 | // link := "https://tiku.baidu.com"+kcLink[0] 53 | // log.Println("课程链接 ==> ", link) 54 | // } 55 | // log.Println("目录 ==> ", zjTitle, xjTitle, kcTitle) 56 | // log.Println("-------------------------------") 57 | // }) 58 | // }) 59 | // 60 | // }) 61 | //} 62 | -------------------------------------------------------------------------------- /_examples/ip_bczs_cn/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mangenotwork/gathertool/_examples/ip_bczs_cn 2 | 3 | go 1.17 4 | 5 | replace github.com/mangenotwork/gathertool => ../../ 6 | 7 | require ( 8 | github.com/PuerkitoBio/goquery v1.8.0 9 | github.com/mangenotwork/gathertool v0.0.0-00010101000000-000000000000 10 | ) 11 | 12 | require ( 13 | github.com/andybalholm/cascadia v1.3.1 // indirect 14 | github.com/aws/aws-sdk-go v1.34.28 // indirect 15 | github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect 16 | github.com/garyburd/redigo v1.6.4 // indirect 17 | github.com/go-sql-driver/mysql v1.7.1 // indirect 18 | github.com/go-stack/stack v1.8.0 // indirect 19 | github.com/golang/snappy v0.0.4 // indirect 20 | github.com/jmespath/go-jmespath v0.4.0 // indirect 21 | github.com/json-iterator/go v1.1.12 // indirect 22 | github.com/klauspost/compress v1.15.14 // indirect 23 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 24 | github.com/modern-go/reflect2 v1.0.2 // indirect 25 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 26 | github.com/pkg/errors v0.9.1 // indirect 27 | github.com/richardlehane/mscfb v1.0.4 // indirect 28 | github.com/richardlehane/msoleps v1.0.3 // indirect 29 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 30 | github.com/xdg-go/scram v1.1.2 // indirect 31 | github.com/xdg-go/stringprep v1.0.4 // indirect 32 | github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect 33 | github.com/xuri/excelize/v2 v2.7.0 // indirect 34 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 35 | go.mongodb.org/mongo-driver v1.5.2 // indirect 36 | golang.org/x/crypto v0.14.0 // indirect 37 | golang.org/x/net v0.17.0 // indirect 38 | golang.org/x/sync v0.1.0 // indirect 39 | golang.org/x/sys v0.13.0 // indirect 40 | golang.org/x/text v0.13.0 // indirect 41 | ) 42 | -------------------------------------------------------------------------------- /_test/base_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestPost(t *testing.T) { 9 | ctx, err := gt.Post("http://127.0.0.1/post/test", []byte("xx"), "application/json") 10 | if err != nil { 11 | t.Error(err) 12 | } 13 | gt.Info(ctx.Json) 14 | } 15 | 16 | func TestPostForm(t *testing.T) { 17 | ctx, err := gt.PostForm("http://127.0.0.1/postFrom/test", map[string]string{"xx": "xx"}) 18 | if err != nil { 19 | t.Error(err) 20 | } 21 | gt.Info(ctx.RespBody) 22 | } 23 | 24 | func TestPostFile(t *testing.T) { 25 | ctx, err := gt.PostFile("http://127.0.0.1/postFile/test", "file", "./test.txt") 26 | if err != nil { 27 | t.Error(err) 28 | } 29 | gt.Info(ctx.RespBody) 30 | } 31 | 32 | func TestPut(t *testing.T) { 33 | ctx, err := gt.Put("http://127.0.0.1/put/test", []byte("xx"), "application/json") 34 | if err != nil { 35 | t.Error(err) 36 | } 37 | gt.Info(ctx.RespBody) 38 | } 39 | 40 | func TestDelete(t *testing.T) { 41 | ctx, err := gt.Delete("http://127.0.0.1/delete/test") 42 | if err != nil { 43 | t.Error(err) 44 | } 45 | gt.Info(ctx.RespBody) 46 | } 47 | 48 | func TestOptions(t *testing.T) { 49 | ctx, err := gt.Options("http://127.0.0.1/options/test") 50 | if err != nil { 51 | t.Error(err) 52 | } 53 | gt.Info(ctx.RespBody) 54 | } 55 | 56 | func TestHead(t *testing.T) { 57 | ctx, err := gt.Head("http://127.0.0.1/head/test") 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | gt.Info(ctx.RespBody) 62 | } 63 | 64 | func TestPatch(t *testing.T) { 65 | ctx, err := gt.Patch("http://127.0.0.1/patch/test") 66 | if err != nil { 67 | t.Error(err) 68 | } 69 | gt.Info(ctx.RespBody) 70 | } 71 | 72 | func TestNewProxyIP(t *testing.T) { 73 | gt.NewProxyIP("127.0.0.1", 8888, "", "", false) 74 | } 75 | 76 | func TestNewProxyPool(t *testing.T) { 77 | gt.NewProxyPool() 78 | } 79 | 80 | func TestNewCookiePool(t *testing.T) { 81 | gt.NewCookiePool() 82 | } 83 | -------------------------------------------------------------------------------- /conf_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 配置文件 yaml TODO 扩展到所有能支持的配置文件 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "fmt" 11 | "os" 12 | "strings" 13 | 14 | "gopkg.in/yaml.v3" 15 | ) 16 | 17 | var Config *conf 18 | 19 | type conf struct { 20 | Path string 21 | Data map[string]any 22 | } 23 | 24 | // NewConf 读取配置，只支持yaml 25 | func NewConf(appConfigPath string) error { 26 | Config = &conf{ 27 | Path: appConfigPath, 28 | Data: make(map[string]any), 29 | } 30 | err := Config.Init() 31 | return err 32 | } 33 | 34 | func (c *conf) Init() error { 35 | if !FileExists(c.Path) { 36 | return fmt.Errorf("未找到配置文件: %v", c.Path) 37 | } 38 | Info("读取配置文件:", c.Path) 39 | //读取yaml文件到缓存中 40 | config, err := os.ReadFile(c.Path) 41 | if err != nil { 42 | ErrorF("读取配置文件 %v 失败", c.Path) 43 | return err 44 | } 45 | return yaml.Unmarshal(config, c.Data) 46 | } 47 | 48 | func (c *conf) GetInt(key string) int { 49 | if c.Data == nil { 50 | _ = c.Init() 51 | } 52 | return Any2Int(c.Data[key]) 53 | } 54 | 55 | func (c *conf) get(key string) (interface{}, bool) { 56 | var ( 57 | d interface{} 58 | ok bool 59 | ) 60 | keyList := strings.Split(key, "::") 61 | temp := make(map[string]interface{}) 62 | temp = c.Data 63 | for _, v := range keyList { 64 | d, ok = temp[v] 65 | if !ok { 66 | break 67 | } 68 | temp = Any2Map(d) 69 | } 70 | return d, ok 71 | } 72 | 73 | // Get 获取配置,多级使用::，例如：user::name 74 | func (c *conf) Get(key string) (any, bool) { 75 | if c.Data == nil { 76 | _ = c.Init() 77 | } 78 | return c.get(key) 79 | } 80 | 81 | // GetString 获取配置,配置不存在返回 "", false 82 | // ex: conf.GetString("user::name") 83 | func (c *conf) GetString(key string) (string, bool) { 84 | if c.Data == nil { 85 | _ = c.Init() 86 | } 87 | val, has := c.get(key) 88 | if !has { 89 | return "", has 90 | } 91 | return Any2String(val), has 92 | } 93 | 94 | func (c *conf) GetAll() map[string]any { 95 | return c.Data 96 | } 97 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mangenotwork/gathertool 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.38.1 7 | github.com/dgrijalva/jwt-go v3.2.0+incompatible 8 | github.com/go-sql-driver/mysql v1.7.1 9 | github.com/nsqio/go-nsq v1.1.0 10 | github.com/streadway/amqp v1.0.0 11 | github.com/xuri/excelize/v2 v2.7.0 12 | golang.org/x/crypto v0.14.0 13 | golang.org/x/net v0.17.0 14 | golang.org/x/text v0.13.0 15 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df 16 | gopkg.in/yaml.v3 v3.0.1 17 | ) 18 | 19 | require ( 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/eapache/go-resiliency v1.3.0 // indirect 22 | github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect 23 | github.com/eapache/queue v1.1.0 // indirect 24 | github.com/golang/snappy v0.0.4 // indirect 25 | github.com/hashicorp/errwrap v1.0.0 // indirect 26 | github.com/hashicorp/go-multierror v1.1.1 // indirect 27 | github.com/hashicorp/go-uuid v1.0.3 // indirect 28 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 29 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 30 | github.com/jcmturner/gofork v1.7.6 // indirect 31 | github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect 32 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 33 | github.com/klauspost/compress v1.15.14 // indirect 34 | github.com/kr/text v0.1.0 // indirect 35 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 36 | github.com/pierrec/lz4/v4 v4.1.17 // indirect 37 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 38 | github.com/richardlehane/mscfb v1.0.4 // indirect 39 | github.com/richardlehane/msoleps v1.0.3 // indirect 40 | github.com/rogpeppe/go-internal v1.11.0 // indirect 41 | github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect 42 | github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect 43 | golang.org/x/sys v0.13.0 // indirect 44 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect 45 | ) 46 | -------------------------------------------------------------------------------- /_test/map_helper_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | // go test -v -run=TestMapHelperOderMap 9 | func TestMapHelperOderMap(t *testing.T) { 10 | intMap := gt.NewOrderMap[int, int]() 11 | intMap.Add(1, 1) 12 | t.Log(intMap.Get(1)) 13 | intMap.Del(1) 14 | t.Log(intMap.Get(1)) 15 | intMap.Add(1, 1).Add(2, 2).Add(3, 3) 16 | intMap.RangeAt(func(id, k, v int) { 17 | t.Log(id, k, v) 18 | }) 19 | stringMap := gt.NewOrderMap[string, int]() 20 | stringMap.Add("a", 1) 21 | t.Log(stringMap.Get("a")) 22 | stringMap.Del("a") 23 | t.Log(stringMap.Get("a")) 24 | stringMap.Add("a", 1) 25 | stringMap.AddMap(map[string]int{"b": 2, "c": 3}) 26 | stringMap.RangeAt(func(id int, k string, v int) { 27 | t.Log(id, k, v) 28 | }) 29 | stringMap.Reverse() 30 | stringMap.RangeAt(func(id int, k string, v int) { 31 | t.Log(id, k, v) 32 | }) 33 | t.Log(stringMap.KeyList()) 34 | 35 | t.Log(stringMap.Json()) 36 | 37 | t.Log(stringMap.Move("a", 2)) 38 | stringMap.DebugPrint() 39 | t.Log(stringMap.Move("a", 3)) 40 | t.Log(stringMap.Insert("d", 4, 1)) 41 | stringMap.DebugPrint() 42 | 43 | t.Log("洗牌 ------> ") 44 | stringMap.Shuffle() 45 | stringMap.DebugPrint() 46 | 47 | t.Log(stringMap.GetAtPosition(2)) 48 | t.Log(stringMap.Pop()) 49 | stringMap.DebugPrint() 50 | t.Log(stringMap.BackPop()) 51 | stringMap.DebugPrint() 52 | 53 | } 54 | 55 | // go test -v -run=TestMapHelperSet 56 | func TestMapHelperSet(t *testing.T) { 57 | s := gt.NewSet() 58 | s.Add(1) 59 | s.Add(1) 60 | s.DebugPrint() 61 | s.Add(2) 62 | s.DebugPrint() 63 | t.Log(s.Has(3)) 64 | t.Log(s.Has(1)) 65 | s.Delete(1) 66 | s.DebugPrint() 67 | } 68 | 69 | // go test -v -run=TestMapHelperStack 70 | func TestMapHelperStack(t *testing.T) { 71 | s := gt.NewStack[string]() 72 | s.Push("a") 73 | s.Push("b") 74 | s.DebugPrint() 75 | t.Log(s.Pop()) 76 | s.DebugPrint() 77 | } 78 | 79 | // go test -v -run=TestMapHelper 80 | func TestMapHelper(t *testing.T) { 81 | map1 := map[int]string{1: "a", 2: "b"} 82 | map2 := gt.MapCopy(map1) 83 | t.Log(map2) 84 | t.Log(gt.MapMergeCopy(map1, map[int]string{3: "c", 2: "b"})) 85 | 86 | t.Log(gt.Map2Slice(map1)) 87 | 88 | t.Log(gt.Slice2Map([]any{"a", 1, "b", 2})) 89 | } 90 | -------------------------------------------------------------------------------- /_examples/get/get.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "time" 7 | 8 | gt "github.com/mangenotwork/gathertool" 9 | ) 10 | 11 | func main() { 12 | SimpleGet1() 13 | SimpleGet2() 14 | SimpleGet3() 15 | SimpleGet4() 16 | 17 | //Case() 18 | } 19 | 20 | // SimpleGet1 简单的get请求实例, 写法一： 方法做为请求函数的参数； 21 | func SimpleGet1() { 22 | // 创建请求 23 | ctx := gt.NewGet("http://192.168.3.1", 24 | //设置请求成功后的方法： 请求的数据 25 | gt.SucceedFunc(func(ctx *gt.Context) { 26 | log.Println(string(ctx.RespBody)) 27 | }), 28 | 29 | //设置请求失败后的方法： 打印失败信息 30 | gt.FailedFunc(func(ctx *gt.Context) { 31 | log.Println(ctx.Err) 32 | }), 33 | 34 | //设置重试前的方法（遇到403,502 等状态码会重试）： 睡眠1s再重试 35 | gt.RetryFunc(func(ctx *gt.Context) { 36 | time.Sleep(1 * time.Second) 37 | }), 38 | ) 39 | // 执行创建的请求 40 | ctx.Do() 41 | 42 | } 43 | 44 | // SimpleGet2 最简单的get请求实例， 写法二： 上下文处理； 45 | func SimpleGet2() { 46 | // 创建请求 47 | ctx, err := gt.Get("http://192.168.3.1") 48 | // 打印请求结果与请求错误 49 | log.Println(ctx.RespBodyString(), err) 50 | } 51 | 52 | // SimpleGet3 简单的get请求实例, 写法三： 给请求设置方法； 53 | func SimpleGet3() { 54 | // 创建请求 55 | gt.NewGet("http://192.168.3.1").SetSucceedFunc(func(ctx *gt.Context) { 56 | //设置请求成功后的方法 57 | log.Println(string(ctx.RespBody)) 58 | }).SetFailedFunc(func(ctx *gt.Context) { 59 | //设置请求失败后的方法 60 | log.Println(ctx.Err) 61 | }).SetRetryFunc(func(*gt.Context) { 62 | //设置重试前的方法 63 | time.Sleep(1 * time.Second) 64 | }).Do() 65 | } 66 | 67 | // SimpleGet4 简单的get请求实例, 写法四： 外部函数为请求方法； 68 | func SimpleGet4() { 69 | _, _ = gt.Get("http://192.168.3.1", 70 | gt.SucceedFunc(succeed), 71 | gt.FailedFunc(fail), 72 | gt.RetryFunc(retry), 73 | ) 74 | } 75 | 76 | // 成功后的方法 77 | func succeed(ctx *gt.Context) { 78 | log.Println(string(ctx.RespBody)) 79 | //处理数据 80 | } 81 | 82 | // 设置需要重试状态码， 重试前的方法 83 | func retry(ctx *gt.Context) { 84 | ctx.Client = &http.Client{ 85 | Timeout: 1 * time.Second, 86 | } 87 | log.Println("休息1s") 88 | time.Sleep(1 * time.Second) 89 | } 90 | 91 | // 失败后的方法 92 | func fail(ctx *gt.Context) { 93 | log.Println("请求失败: ", ctx.Err) 94 | } 95 | 96 | func Case() { 97 | a := gt.Any2String("aasddas") 98 | log.Println(a) 99 | } 100 | -------------------------------------------------------------------------------- /_examples/cnlinfo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | ) 6 | 7 | // 行业信息网 : http://www.cnlinfo.net/ 8 | 9 | func init() { 10 | // 读取配置 11 | err := gt.NewConf("conf.yaml") 12 | if err != nil { 13 | panic(err) 14 | } 15 | } 16 | 17 | // Rse 抓取结果保存的数据结构 18 | type Rse struct { 19 | Broad string `json:"broad"` 20 | Fenlei []string `json:"fenlei"` 21 | } 22 | 23 | var ( 24 | rse = make([]*Rse, 0) 25 | ) 26 | 27 | // 获取行业分类 28 | func main() { 29 | caseUrl := "http://www.cnlinfo.net/allgongsifenlei/yiqiyibiao.htm" 30 | ctx, _ := gt.Get(caseUrl) 31 | hangyeList, err := gt.GetPointClassHTML(ctx.Html, "ul", "hangye-list") 32 | if err != nil { 33 | gt.Error("没有获取到 ul class = hangye-list") 34 | return 35 | } 36 | hangyeHtml := "" 37 | if len(hangyeList) > 0 { 38 | hangyeHtml = hangyeList[0] 39 | } 40 | // 获取并便利分类大类 41 | for _, li := range gt.RegHtmlA(hangyeHtml) { 42 | broad := gt.RegHtmlATxt(li)[0] 43 | href := gt.RegHtmlHrefTxt(li)[0] 44 | // 抓取小类 45 | getClassify2(broad, href) 46 | } 47 | // 保存并输出到json文件 48 | err = gt.OutJsonFile(rse, gt.Config.GetStr("out_file")) 49 | if err != nil { 50 | gt.Error(err) 51 | } 52 | } 53 | 54 | // 获取小类 55 | func getClassify2(broad, caseUrl string) { 56 | var ( 57 | notData = 0 58 | reqDelayMax = gt.Config.GetInt("req_delay_max") 59 | reqDelayMin = gt.Config.GetInt("req_delay_min") 60 | ) 61 | d := &Rse{ 62 | Broad: broad, 63 | Fenlei: make([]string, 0), 64 | } 65 | gt.Infof("抓取大类 %v : %v", broad, caseUrl) 66 | ctx, _ := gt.Get(caseUrl, gt.SetSleep(reqDelayMin, reqDelayMax)) 67 | hangyeContent, err := gt.GetPointClassHTML(ctx.Html, "ul", "hangye-fenlei-content") 68 | if err != nil { 69 | gt.Error("没有获取到 ul class = hangye-fenlei-content") 70 | notData++ 71 | } 72 | for _, ul := range hangyeContent { 73 | for _, a := range gt.RegHtmlATxt(ul) { 74 | d.Fenlei = append(d.Fenlei, a) 75 | } 76 | } 77 | // 第二种样式 78 | hangyeContent1, err := gt.GetPointClassHTML(ctx.Html, "ul", "fenlei_list1") 79 | if err != nil { 80 | gt.Error("没有获取到 ul class = fenlei_list1") 81 | notData++ 82 | } 83 | for _, ul := range hangyeContent1 { 84 | for _, a := range gt.RegHtmlATxt(ul) { 85 | d.Fenlei = append(d.Fenlei, a) 86 | } 87 | } 88 | rse = append(rse, d) 89 | if notData == 2 { 90 | panic("该页面没有数据") 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /_examples/qihuo/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | url: http://futures.100ppi.com/qhb/day-2022-07-05.html 3 | 抓取期货数据 4 | */ 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | gt "github.com/mangenotwork/gathertool" 10 | "log" 11 | "strings" 12 | ) 13 | 14 | // 数据库对象 15 | var ( 16 | host192 = "192.168.0.192" 17 | port = 3306 18 | user = "root" 19 | password = "root123" 20 | dbName = "test" 21 | DB, _ = gt.NewMysql(host192, port, user, password, dbName) 22 | OutTable = "qihuo" 23 | ) 24 | 25 | func main() { 26 | date := "2022-07-05" 27 | caseUrl := "http://futures.100ppi.com/qhb/day-%s.html" 28 | ctx, _ := gt.Get(fmt.Sprintf(caseUrl, date)) 29 | //log.Println(ctx.Html) 30 | // 数据提取 31 | datas, _ := gt.GetPointHTML(ctx.Html, "div", "id", "domestic") 32 | Data(datas, date, "内期表", "备注：内期表=国内期货主力合约表") 33 | datas, _ = gt.GetPointHTML(ctx.Html, "div", "id", "overseas") 34 | Data(datas, date, "外期表", "备注：外期表=国外期货主力合约表") 35 | } 36 | 37 | func Data(datas []string, date, typeName, note string) { 38 | for _, data := range datas { 39 | table, _ := gt.GetPointHTML(data, "table", "id", "fdata") 40 | if len(table) > 0 { 41 | trList := gt.RegHtmlTr(table[0]) 42 | jys := "" 43 | for _, tr := range trList { 44 | td := gt.RegHtmlTd(tr) 45 | log.Println("td = ", td, len(td)) 46 | if len(td) == 1 { 47 | jys = gt.RegHtmlTdTxt(td[0])[0] 48 | continue 49 | } 50 | name := gt.RegHtmlTdTxt(td[0])[0] 51 | if strings.Index(name, "商品名称") != -1 { 52 | continue 53 | } 54 | zlhy := gt.RegHtmlTdTxt(td[1])[0] 55 | jsj := gt.RegHtmlTdTxt(td[2])[0] 56 | zd := gt.RegDelHtml(gt.RegHtmlTdTxt(td[3])[0]) 57 | cjj := gt.RegHtmlTdTxt(td[4])[0] 58 | ccl := gt.RegHtmlTdTxt(td[5])[0] 59 | dw := gt.RegHtmlTdTxt(td[6])[0] 60 | log.Println("日期 = ", date) 61 | log.Println("机构 = ", jys) 62 | log.Println("商品名称 = ", name) 63 | log.Println("主力合约 = ", zlhy) 64 | log.Println("结算价 = ", jsj) 65 | log.Println("涨跌 = ", zd) 66 | log.Println("成交量 = ", cjj) 67 | log.Println("持仓量 = ", ccl) 68 | log.Println("单位 = ", dw) 69 | gd := gt.NewGDMap().Add("date", date).Add("type_name", typeName).Add("note", note) 70 | gd.Add("jys", jys).Add("name", name).Add("zlhy", zlhy).Add("jsj", jsj) 71 | gd.Add("zd", zd).Add("cjj", cjj).Add("ccl", ccl).Add("dw", dw) 72 | err := DB.InsertAtGd(OutTable, gd) 73 | if err != nil { 74 | panic(err) 75 | } 76 | } 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /_examples/json/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "log" 6 | ) 7 | 8 | func main(){ 9 | txt := `{ 10 | "reason":"查询成功!", 11 | "result":{ 12 | "city":"苏州", 13 | "realtime":{ 14 | "temperature":"17", 15 | "humidity":"69", 16 | "info":"阴", 17 | "wid":"02", 18 | "direct":"东风", 19 | "power":"2级", 20 | "aqi":"30" 21 | }, 22 | "future":[ 23 | { 24 | "date":"2021-10-25", 25 | "temperature":"12\/21℃", 26 | "weather":"多云", 27 | "wid":{ 28 | "day":"01", 29 | "night":"01" 30 | }, 31 | "direct":"东风" 32 | }, 33 | { 34 | "date":"2021-10-26", 35 | "temperature":"13\/21℃", 36 | "weather":"多云", 37 | "wid":{ 38 | "day":"01", 39 | "night":"01" 40 | }, 41 | "direct":"东风转东北风" 42 | }, 43 | { 44 | "date":"2021-10-27", 45 | "temperature":"13\/22℃", 46 | "weather":"多云", 47 | "wid":{ 48 | "day":"01", 49 | "night":"01" 50 | }, 51 | "direct":"东北风" 52 | }, 53 | { 54 | "date":"2021-10-28", 55 | "temperature":"13\/21℃", 56 | "weather":"多云转晴", 57 | "wid":{ 58 | "day":"01", 59 | "night":"00" 60 | }, 61 | "direct":"东北风" 62 | }, 63 | { 64 | "date":"2021-10-29", 65 | "temperature":"14\/21℃", 66 | "weather":"多云转小雨", 67 | "wid":{ 68 | "day":"01", 69 | "night":"07" 70 | }, 71 | "direct":"东北风" 72 | } 73 | ] 74 | }, 75 | "error_code":0 76 | }` 77 | 78 | jx1 := "/result/future/[0]/date" 79 | jx2 := "/result/future/[0]" 80 | jx3 := "/result/future" 81 | 82 | log.Println(gt.JsonFind(txt, jx1)) 83 | log.Println(gt.JsonFind2Json(txt, jx2)) 84 | log.Println(gt.JsonFind2Json(txt, jx3)) 85 | log.Println(gt.JsonFind2Map(txt, jx2)) 86 | log.Println(gt.JsonFind2Arr(txt, jx3)) 87 | 88 | } 89 | 90 | -------------------------------------------------------------------------------- /_test/string_helper_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestStringHelperTypeConversion(t *testing.T) { 9 | t.Log(gt.Any2String(1)) 10 | t.Log(gt.Any2String(2.22)) 11 | t.Log(gt.Any2String(true)) 12 | t.Log(gt.Any2String([]int{1, 2})) 13 | 14 | t.Log(gt.Json2Map("{\"a\":2,\"3\":4}")) 15 | t.Log(gt.Any2Map("{\"a\":2,\"3\":4}")) 16 | 17 | t.Log(gt.Any2Int("123")) 18 | t.Log(gt.Any2Int("123a")) 19 | t.Log(gt.Any2Int64("123")) 20 | t.Log(gt.Any2Arr("123,123")) 21 | t.Log(gt.Any2Float64("123.123")) 22 | 23 | t.Log(gt.Int2Hex(123)) 24 | t.Log(gt.Int642Hex(123)) 25 | t.Log(gt.Hex2Int("7b")) 26 | t.Log(gt.Hex2Int64("7b")) 27 | t.Log(gt.Str2Int("123")) 28 | t.Log(gt.Str2Int32("123")) 29 | t.Log(gt.Str2Float64("123.123")) 30 | t.Log(gt.Str2Float32("123.123")) 31 | t.Log(gt.Uint82Str([]uint8{1, 2, 3})) 32 | t.Log(gt.Str2Byte("abc")) 33 | t.Log(gt.Byte2Str([]byte("abc"))) 34 | t.Log(gt.Bool2Byte(true)) 35 | t.Log(gt.Byte2Bool([]byte("0"))) 36 | t.Log(gt.Int2Byte(123)) 37 | t.Log(gt.Byte2Int([]byte("1111"))) 38 | t.Log(gt.Int642Byte(123)) 39 | t.Log(gt.Byte2Int64(gt.Int642Byte(123))) 40 | t.Log(gt.Float322Byte(123.123)) 41 | t.Log(gt.Float322Uint32(123.123)) 42 | t.Log(gt.Byte2Float32(gt.Float322Byte(123.123))) 43 | t.Log(gt.Float642Byte(123.123)) 44 | t.Log(gt.Float642Uint64(123.123)) 45 | t.Log(gt.Byte2Float64(gt.Float642Byte(123.123))) 46 | t.Log(gt.Float642Uint64(123.123)) 47 | t.Log(gt.Byte2Float64(gt.Float642Byte(123.123))) 48 | type A struct { 49 | C string 50 | } 51 | a1 := &A{C: "1"} 52 | t.Log(gt.Struct2Map(a1, true)) 53 | 54 | t.Log(gt.CleaningStr(" sada\n\t\r ")) 55 | t.Log(gt.StrDeleteSpace(" sada ")) 56 | t1 := gt.EncodeByte("test1") 57 | t.Log(t1) 58 | t.Log(gt.DecodeByte(t1)) 59 | t.Log(gt.Byte2Bit([]byte("aaa"))) 60 | t.Log(gt.Bit2Byte(gt.Byte2Bit([]byte("aaa")))) 61 | 62 | a2 := &A{} 63 | t.Log(gt.DeepCopy[*A](a2, a1)) 64 | t.Log(a2) 65 | 66 | t.Log(gt.PanicToError(func() { 67 | panic(1) 68 | })) 69 | 70 | t.Log(gt.ByteToBinaryString(byte('a'))) 71 | t.Log(gt.UnicodeDec("aaa")) 72 | t.Log(gt.UnicodeDecByte([]byte("aaa"))) 73 | t.Log(gt.UnescapeUnicode(gt.UnicodeDecByte([]byte("aaa")))) 74 | t.Log(gt.Base64Encode("一二三")) 75 | t.Log(gt.Base64Decode(gt.Base64Encode("一二三"))) 76 | t.Log(gt.Base64UrlEncode("一二三")) 77 | t.Log(gt.Base64UrlDecode(gt.Base64UrlEncode("一二三"))) 78 | 79 | t2, _ := gt.ToUTF8("HZGB2312", "一二三") 80 | t.Log(t2) 81 | t.Log(gt.UTF8To("HZGB2312", t2)) 82 | 83 | t3, _ := gt.ToUTF16("HZGB2312", "一二三") 84 | t.Log(t3) 85 | t.Log(gt.UTF16To("HZGB2312", t3)) 86 | 87 | t4, _ := gt.ToBIG5("HZGB2312", "一二三") 88 | t.Log(t4) 89 | t.Log(gt.BIG5To("HZGB2312", t4)) 90 | 91 | t5, _ := gt.ToGDK("HZGB2312", "一二三") 92 | t.Log(t5) 93 | t.Log(gt.GDKTo("HZGB2312", t5)) 94 | 95 | t6, _ := gt.ToGB18030("HZGB2312", "一二三") 96 | t.Log(t6) 97 | t.Log(gt.GB18030To("HZGB2312", t6)) 98 | 99 | t7, _ := gt.ToGB2312("HZGB2312", "一二三") 100 | t.Log(t7) 101 | t.Log(gt.GB2312To("HZGB2312", t7)) 102 | 103 | t8, _ := gt.ToHZGB2312("HZGB2312", "一二三") 104 | t.Log(t8) 105 | t.Log(gt.HZGB2312To("HZGB2312", t8)) 106 | 107 | t.Log(gt.IF[string](true, "a", "b")) 108 | 109 | t.Log(gt.ReplaceAllToOne("aaacvccc", []string{"a", "c"}, "")) 110 | 111 | t.Log(gt.HumanFriendlyTraffic(12354646)) 112 | t.Log(gt.StrToSize("123415kb")) 113 | 114 | t.Log(gt.IP2Binary("127.0.0.1")) 115 | t.Log(gt.UInt32ToIP(12700001)) 116 | t.Log(gt.IP2Int64("127.0.0.1")) 117 | 118 | t.Log(gt.GzipCompress([]byte("abc"))) 119 | t.Log(gt.GzipDecompress(gt.GzipCompress([]byte("abc")))) 120 | t.Log(gt.ConvertStr2GBK("一二三")) 121 | t.Log(gt.ConvertGBK2Str("一二三")) 122 | t.Log(gt.ByteToGBK([]byte("一二三"))) 123 | t.Log(gt.ByteToUTF8([]byte("一二三"))) 124 | t.Log(gt.ID()) 125 | t.Log(gt.IDMd5()) 126 | 127 | } 128 | 129 | func TestStringHelper(t *testing.T) { 130 | t.Log(gt.MD5("asdsad")) 131 | } 132 | -------------------------------------------------------------------------------- /_examples/weibo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "log" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | func main() { 11 | 12 | //case1() 13 | 14 | //case2() 15 | 16 | //case3() 17 | 18 | case4() 19 | } 20 | 21 | func case1() { 22 | 23 | /* 24 | category 25 | 0 热门 26 | 1760 头条 27 | 99991 榜单 28 | 10011 高笑 29 | 7 社会 30 | 12 时尚 31 | 10018 电影 32 | 10007 美女 33 | 3 体育 34 | 10005 动漫 35 | */ 36 | 37 | url := "https://weibo.com/a/aj/transform/loadingmoreunlogin?ajwvr=6&category=0&page=2&lefnav=0&cursor=&__rnd=" + gt.TimestampStr() 38 | ctx, _ := gt.Get(url, gt.SucceedFunc(succed)) 39 | ctx.Req.AddCookie(&http.Cookie{Name: "SUBP", Value: "0033WrSXqPxfM72-Ws9jqgMF55529P9D9WWENAjmKyIZz1AWjDi68mRw", HttpOnly: true}) 40 | ctx.Req.AddCookie(&http.Cookie{Name: "SUB", Value: "_2AkMXxWiSf8NxqwFRmPoWz2nlbop1zwvEieKhmZlJJRMxHRl-yT9jqlAItRB6PEVGfTP09XmsX_7CR2H1OUv6b-f-1bJl", HttpOnly: true}) 41 | ctx.Do() 42 | } 43 | 44 | func succed(ctx *gt.Context) { 45 | 46 | //html := gt.ConvertByte2String(ctx.RespBody, gt.GB2312) 47 | htmlBody, _ := gt.UnescapeUnicode(ctx.RespBody) 48 | html := string(htmlBody) 49 | html = strings.Replace(html, "\\r", "", -1) 50 | html = strings.Replace(html, "\\n", "", -1) 51 | html = strings.Replace(html, "\\", "", -1) 52 | 53 | } 54 | 55 | // cookie 持久化 56 | func case2() { 57 | // ========== 获取 tid 58 | //由 https://passport.weibo.com/js/visitor/mini_original.js?v=20161116 //main 执行入口 js代码 找到tid的方式 59 | // 也找到了 fpPostInterface: "visitor/genvisitor" 60 | // fpCollectInterface: "visitor/record" 61 | 62 | // cb 固定值 gen_callback 63 | //getTidUrl := "https://passport.weibo.com/visitor/genvisitor?cb=gen_callback&fp={\"os\":\"1\",\"browser\":\"Chrome70,0,3538,25\",\"fonts\":\"undefined\",\"screenInfo\":\"1920*1080*24\",\"plugins\":\"\"}" 64 | //ctx, _ := gt.Get(getTidUrl) 65 | //log.Println(string(ctx.RespBody)) 66 | 67 | // ============ 获取sub subp 68 | // F12观察到 https://passport.weibo.com/visitor/visitor?a=restore&cb=restore_back&from=weibo&_rand=0.8977533486179248 69 | // 获取的cookie 70 | // a 固定 incarnate 71 | // t 上一步得到的tid 72 | // w 上一步 new_tid = true 就是3 否则为2 73 | // cb 固定 cross_domain 74 | // from 固定 weibo 75 | 76 | getSubUrl := "https://passport.weibo.com/visitor/visitor?a=incarnate&t=h2b7xQtQwqk2cEjMgH/0AaWYvpijlgCCAs3qDzj2W58=&w=3&c&cb=restore_back&from=weibo" 77 | ctx, _ := gt.Get(getSubUrl) 78 | log.Println(ctx.Resp) 79 | log.Println(ctx.Resp.Cookies()) 80 | log.Println(string(ctx.RespBody)) 81 | 82 | //getSubUrl := "https://passport.weibo.com/visitor/visitor?a=restore&cb=restore_back&from=weibo&_rand=0.8977533486179248" 83 | //ctx,_ := gt.Get(getSubUrl) 84 | //ctx.Req.AddCookie(&http.Cookie{Name: "tid",Value: `HVyCvxI2gJHD6a6/hDOaWo3MNgwy9X8LwZFUG7o3vNA=__095`, HttpOnly: true}) 85 | ////ctx.Req.AddCookie(&http.Cookie{Name: "SRT",Value: "D.QqHBTrsR5QPiUcRtOeYoWr9NUPB3R39Qi-bYNdo35QWwMdbbN-YjTmntNbHi5mYNUCsuTZbgVdYC43MNAZSAMQHK549Q4qiaK4S1VFM6R4YbVP9GUqYYT3AqW-kmdA77%2AB.vAflW-P9Rc0lR-ykKDvnJqiQVbiRVPBtS%21r3J8sQVqbgVdWiMZ4siOzu4DbmKPWQU4PYU%21SiM4b9M-yMi%21VkR3mpIbPw", HttpOnly: true}) 86 | ////ctx.Req.AddCookie(&http.Cookie{Name: "SRF",Value: "1620470800", HttpOnly: true}) 87 | //ctx.Do() 88 | //log.Println(ctx.Resp) 89 | //log.Println(ctx.Resp.Cookies()) 90 | //log.Println(ctx.Resp.TransferEncoding) 91 | //log.Println(string(ctx.RespBody)) 92 | } 93 | 94 | func case3() { 95 | gt.MongoConn() 96 | } 97 | 98 | // 判断微博, 手机号是否注册 99 | func case4() { 100 | caseUrl := "https://weibo.com/signup/v5/formcheck?type=mobilesea&zone=0086&value=18483663083&from=&__rnd=" 101 | log.Println(caseUrl) 102 | ctx := gt.NewGet(caseUrl) 103 | ctx.AddHeader("referer", "https://weibo.com/signup/signup.php") 104 | ctx.AddHeader("accept-language", "zh-CN,zh;q=0.9") 105 | ctx.Do() 106 | log.Println(ctx.Resp.Status, gt.UnicodeDec(ctx.RespBodyString())) 107 | } 108 | -------------------------------------------------------------------------------- /id_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : id，有雪花id, uuid(todo) 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "crypto/md5" 11 | "encoding/hex" 12 | "fmt" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | // =================================================== 雪花Id 18 | 19 | type IdWorker struct { 20 | startTime int64 21 | workerIdBits uint 22 | datacenterIdBits uint 23 | maxWorkerId int64 24 | maxDatacenterId int64 25 | sequenceBits uint 26 | workerIdLeftShift uint 27 | datacenterIdLeftShift uint 28 | timestampLeftShift uint 29 | sequenceMask int64 30 | workerId int64 31 | datacenterId int64 32 | sequence int64 33 | lastTimestamp int64 34 | signMask int64 35 | idLock *sync.Mutex 36 | } 37 | 38 | func (idw *IdWorker) InitIdWorker(workerId, datacenterId int64) error { 39 | var baseValue int64 = -1 40 | idw.startTime = 1463834116272 41 | idw.workerIdBits = 5 42 | idw.datacenterIdBits = 5 43 | idw.maxWorkerId = baseValue ^ (baseValue << idw.workerIdBits) 44 | idw.maxDatacenterId = baseValue ^ (baseValue << idw.datacenterIdBits) 45 | idw.sequenceBits = 12 46 | idw.workerIdLeftShift = idw.sequenceBits 47 | idw.datacenterIdLeftShift = idw.workerIdBits + idw.workerIdLeftShift 48 | idw.timestampLeftShift = idw.datacenterIdBits + idw.datacenterIdLeftShift 49 | idw.sequenceMask = baseValue ^ (baseValue << idw.sequenceBits) 50 | idw.sequence = 0 51 | idw.lastTimestamp = -1 52 | idw.signMask = ^baseValue + 1 53 | idw.idLock = &sync.Mutex{} 54 | if idw.workerId < 0 || idw.workerId > idw.maxWorkerId { 55 | return fmt.Errorf("workerId[%v] is less than 0 or greater than maxWorkerId[%v]", 56 | workerId, datacenterId) 57 | } 58 | if idw.datacenterId < 0 || idw.datacenterId > idw.maxDatacenterId { 59 | return fmt.Errorf("datacenterId[%d] is less than 0 or greater than maxDatacenterId[%d]", 60 | workerId, datacenterId) 61 | } 62 | idw.workerId = workerId 63 | idw.datacenterId = datacenterId 64 | return nil 65 | } 66 | 67 | // NextId 返回一个唯一的 INT64 ID 68 | func (idw *IdWorker) NextId() (int64, error) { 69 | idw.idLock.Lock() 70 | timestamp := time.Now().UnixNano() 71 | if timestamp < idw.lastTimestamp { 72 | return -1, fmt.Errorf(fmt.Sprintf("Clock moved backwards. Refusing to generate id for %d milliseconds", 73 | idw.lastTimestamp-timestamp)) 74 | } 75 | if timestamp == idw.lastTimestamp { 76 | idw.sequence = (idw.sequence + 1) & idw.sequenceMask 77 | if idw.sequence == 0 { 78 | timestamp = idw.tilNextMillis() 79 | idw.sequence = 0 80 | } 81 | } else { 82 | idw.sequence = 0 83 | } 84 | idw.lastTimestamp = timestamp 85 | idw.idLock.Unlock() 86 | id := ((timestamp - idw.startTime) << idw.timestampLeftShift) | 87 | (idw.datacenterId << idw.datacenterIdLeftShift) | 88 | (idw.workerId << idw.workerIdLeftShift) | 89 | idw.sequence 90 | if id < 0 { 91 | id = -id 92 | } 93 | return id, nil 94 | } 95 | 96 | // tilNextMillis 97 | func (idw *IdWorker) tilNextMillis() int64 { 98 | timestamp := time.Now().UnixNano() 99 | if timestamp <= idw.lastTimestamp { 100 | timestamp = time.Now().UnixNano() / int64(time.Millisecond) 101 | } 102 | return timestamp 103 | } 104 | 105 | func ID64() (int64, error) { 106 | currWorker := &IdWorker{} 107 | err := currWorker.InitIdWorker(1000, 2) 108 | if err != nil { 109 | return 0, err 110 | } 111 | return currWorker.NextId() 112 | } 113 | 114 | func ID() int64 { 115 | id, _ := ID64() 116 | return id 117 | } 118 | 119 | func IDStr() string { 120 | currWorker := &IdWorker{} 121 | err := currWorker.InitIdWorker(1000, 2) 122 | if err != nil { 123 | return "" 124 | } 125 | id, err := currWorker.NextId() 126 | if err != nil { 127 | return "" 128 | } 129 | return Any2String(id) 130 | } 131 | 132 | func IDMd5() string { 133 | return Get16MD5Encode(IDStr()) 134 | } 135 | 136 | // MD5 MD5 137 | func MD5(str string) string { 138 | h := md5.New() 139 | h.Write([]byte(str)) 140 | return hex.EncodeToString(h.Sum(nil)) 141 | } 142 | -------------------------------------------------------------------------------- /_test/reg_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "testing" 6 | ) 7 | 8 | func TestReg(t *testing.T) { 9 | // 优先检查正则表达式是否报错，程序是否中断 10 | ctx, _ := gt.Get("https://golang.google.cn/") 11 | gt.RegFindAllTxt(`(?is:)`, ctx.Html) 12 | gt.RegHtmlA(ctx.Html) 13 | gt.RegHtmlTitle(ctx.Html) 14 | gt.RegHtmlKeyword(ctx.Html) 15 | gt.RegHtmlDescription(ctx.Html) 16 | gt.RegHtmlTr(ctx.Html) 17 | gt.RegHtmlInput(ctx.Html) 18 | gt.RegHtmlTd(ctx.Html) 19 | gt.RegHtmlP(ctx.Html) 20 | gt.RegHtmlSpan(ctx.Html) 21 | gt.RegHtmlSrc(ctx.Html) 22 | gt.RegHtmlHref(ctx.Html) 23 | gt.RegHtmlH1(ctx.Html) 24 | gt.RegHtmlH2(ctx.Html) 25 | gt.RegHtmlH3(ctx.Html) 26 | gt.RegHtmlH4(ctx.Html) 27 | gt.RegHtmlH5(ctx.Html) 28 | gt.RegHtmlH6(ctx.Html) 29 | gt.RegHtmlTbody(ctx.Html) 30 | gt.RegHtmlVideo(ctx.Html) 31 | gt.RegHtmlCanvas(ctx.Html) 32 | gt.RegHtmlCode(ctx.Html) 33 | gt.RegHtmlImg(ctx.Html) 34 | gt.RegHtmlUl(ctx.Html) 35 | gt.RegHtmlLi(ctx.Html) 36 | gt.RegHtmlMeta(ctx.Html) 37 | gt.RegHtmlSelect(ctx.Html) 38 | gt.RegHtmlTable(ctx.Html) 39 | gt.RegHtmlButton(ctx.Html) 40 | gt.RegHtmlTableOnly(ctx.Html) 41 | gt.RegHtmlDiv(ctx.Html) 42 | gt.RegHtmlOption(ctx.Html) 43 | 44 | // 取标签内容 45 | gt.RegHtmlATxt(ctx.Html) 46 | gt.RegHtmlTitleTxt(ctx.Html) 47 | gt.RegHtmlKeywordTxt(ctx.Html) 48 | gt.RegHtmlDescriptionTxt(ctx.Html) 49 | gt.RegHtmlTrTxt(ctx.Html) 50 | gt.RegHtmlTdTxt(ctx.Html) 51 | gt.RegHtmlPTxt(ctx.Html) 52 | gt.RegHtmlSpanTxt(ctx.Html) 53 | gt.RegHtmlSrcTxt(ctx.Html) 54 | gt.RegHtmlHrefTxt(ctx.Html) 55 | gt.RegHtmlH1Txt(ctx.Html) 56 | gt.RegHtmlH2Txt(ctx.Html) 57 | gt.RegHtmlH3Txt(ctx.Html) 58 | gt.RegHtmlH4Txt(ctx.Html) 59 | gt.RegHtmlH5Txt(ctx.Html) 60 | gt.RegHtmlH6Txt(ctx.Html) 61 | gt.RegHtmlCodeTxt(ctx.Html) 62 | gt.RegHtmlUlTxt(ctx.Html) 63 | gt.RegHtmlLiTxt(ctx.Html) 64 | gt.RegHtmlSelectTxt(ctx.Html) 65 | gt.RegHtmlTableTxt(ctx.Html) 66 | gt.RegHtmlButtonTxt(ctx.Html) 67 | gt.RegHtmlDivTxt(ctx.Html) 68 | gt.RegHtmlOptionTxt(ctx.Html) 69 | gt.RegValue(ctx.Html) 70 | 71 | // 删除 72 | gt.RegDelHtml(ctx.Html) 73 | gt.RegDelNumber(ctx.Html) 74 | 75 | // 是否含有 76 | gt.IsNumber(ctx.Html) 77 | gt.IsNumber2Len(ctx.Html, 2) 78 | gt.IsNumber2Heard(ctx.Html, 2) 79 | gt.IsFloat(ctx.Html) 80 | gt.IsFloat2Len(ctx.Html, 2) 81 | gt.IsEngAll(ctx.Html) 82 | gt.IsEngLen(ctx.Html, 2) 83 | gt.IsEngNumber(ctx.Html) 84 | gt.IsLeastNumber(ctx.Html, 2) 85 | gt.IsLeastCapital(ctx.Html, 2) 86 | gt.IsLeastLower(ctx.Html, 2) 87 | gt.IsLeastSpecial(ctx.Html, 2) 88 | gt.HaveNumber(ctx.Html) 89 | gt.HaveSpecial(ctx.Html) 90 | gt.IsEmail(ctx.Html) 91 | gt.IsDomain(ctx.Html) 92 | gt.IsURL(ctx.Html) 93 | gt.IsPhone(ctx.Html) 94 | gt.IsLandline(ctx.Html) 95 | gt.AccountRational(ctx.Html) 96 | gt.IsXMLFile(ctx.Html) 97 | gt.IsUUID3(ctx.Html) 98 | gt.IsUUID4(ctx.Html) 99 | gt.IsUUID5(ctx.Html) 100 | gt.IsRGB(ctx.Html) 101 | gt.IsFullWidth(ctx.Html) 102 | gt.IsHalfWidth(ctx.Html) 103 | gt.IsBase64(ctx.Html) 104 | gt.IsLatitude(ctx.Html) 105 | gt.IsLongitude(ctx.Html) 106 | gt.IsDNSName(ctx.Html) 107 | gt.IsIPv4(ctx.Html) 108 | gt.IsWindowsPath(ctx.Html) 109 | gt.IsUnixPath(ctx.Html) 110 | 111 | //常用正则 112 | gt.RegTime(ctx.Html) 113 | gt.RegLink(ctx.Html) 114 | gt.RegEmail(ctx.Html) 115 | gt.RegIPv4(ctx.Html) 116 | gt.RegIPv6(ctx.Html) 117 | gt.RegIP(ctx.Html) 118 | gt.RegMD5Hex(ctx.Html) 119 | gt.RegSHA1Hex(ctx.Html) 120 | gt.RegSHA256Hex(ctx.Html) 121 | gt.RegGUID(ctx.Html) 122 | gt.RegMACAddress(ctx.Html) 123 | gt.RegEmail2(ctx.Html) 124 | gt.RegUUID3(ctx.Html) 125 | gt.RegUUID4(ctx.Html) 126 | gt.RegUUID5(ctx.Html) 127 | gt.RegUUID(ctx.Html) 128 | gt.RegInt(ctx.Html) 129 | gt.RegFloat(ctx.Html) 130 | gt.RegRGBColor(ctx.Html) 131 | gt.RegFullWidth(ctx.Html) 132 | gt.RegHalfWidth(ctx.Html) 133 | gt.RegBase64(ctx.Html) 134 | gt.RegLatitude(ctx.Html) 135 | gt.RegLongitude(ctx.Html) 136 | gt.RegDNSName(ctx.Html) 137 | gt.RegFullURL(ctx.Html) 138 | gt.RegURLSchema(ctx.Html) 139 | gt.RegURLUsername(ctx.Html) 140 | gt.RegURLPath(ctx.Html) 141 | gt.RegURLPort(ctx.Html) 142 | gt.RegURLIP(ctx.Html) 143 | gt.RegURLSubdomain(ctx.Html) 144 | gt.RegWinPath(ctx.Html) 145 | gt.RegUnixPath(ctx.Html) 146 | } 147 | -------------------------------------------------------------------------------- /_examples/douyin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/PuerkitoBio/goquery" 6 | gt "github.com/mangenotwork/gathertool" 7 | "log" 8 | "net/url" 9 | "strings" 10 | ) 11 | 12 | func main(){ 13 | caseUrl := "https://www.douyin.com/search/美女" 14 | ctx, _ := gt.Get(caseUrl) 15 | html := ctx.RespBodyString() 16 | log.Println("state = ", ctx.Resp.Status) 17 | log.Println("html = ", html) 18 | 19 | //https://www.douyin.com/aweme/v1/web/search/item/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_video_web&sort_type=0&publish_time=0&keyword=%E7%BE%8E%E5%A5%B3&search_source=normal_search&query_correct_type=1&is_filter_search=0&offset=0&count=24&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Linux+x86_64&browser_name=Mozilla&browser_version=5.0+(X11%3B+Linux+x86_64)+AppleWebKit%2F537.36+(KHTML,+like+Gecko)+Chrome%2F88.0.4324.182+Safari%2F537.36&browser_online=true&msToken=hYnIG63GJJkmsYDWKUjWsvEDcDPmZEuy8MYgsjXxSlU1ZN6MC8hm8GhE_9a3ZyzPYfjdiNH1NAyJ3f9kVKYoWZKhLA97yJEFpZE0iuAmTv-d_ANDzS_XIdM=&X-Bogus=DFSzswSO6iXANti9Sm4Fs7kkNL-I&_signature=_02B4Z6wo000012Jp3cwAAIDAlRxGJKBU6ztibdlAALnad5Pv.LnK808o1d1sNqHrn0TuRCRBKdT7SBouK177aJxXYha8PtRCONuke.0SGNVqrwEC0oaLm3b1csTU2JaAVmPgWXxliTxTbadT4d 20 | //https://www.douyin.com/aweme/v1/web/search/item/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_video_web&sort_type=0&publish_time=0&keyword=%E7%BE%8E%E5%A5%B3&search_source=normal_search&query_correct_type=1&is_filter_search=0&offset=0&count=24&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Linux+x86_64&browser_name=Mozilla&browser_version=5.0+(X11%3B+Linux+x86_64)+AppleWebKit%2F537.36+(KHTML,+like+Gecko)+Chrome%2F88.0.4324.182+Safari%2F537.36&browser_online=true&msToken=V87HYI3JZjqkHIzHUXeCX-ydzD-tg0a-3XnKe4ve8CKjU3aTgNooos1-YkuX4vqKdeh8M-AxIGQ7Se9ZYuaY9hMQ_DMOusY6fvd8-8m-s_PB18E3jWDonM4x&X-Bogus=DFSzswSO6EGANti9Sm4FddkkNL-N&_signature=_02B4Z6wo00001reRrigAAIDBQOQ1w1S1fya3laqAAMy9pcfZE.rxv6pEdxi2yA2oLpfeiTpV7qKmXzx8P8z6GWU76dRxCzblqXzukwCUP3ENdbAGBoU090l4J8MOqXBZgVDV19sE4LsODg5b07 21 | 22 | /* 23 | H = 6383 24 | , q = "douyin_website" 25 | , X = "" 26 | , W = "cbae2a42b075f1dfc39b9e7df764637c821e7bf2"; 27 | 28 | https://lf1-cdn-tos.bytegoofy.com/goofy/ies/douyin/search/index.50db626a.js:formatted --> 11452 29 | 30 | 31 | */ 32 | 33 | 34 | } 35 | 36 | func main1(){ 37 | //ctx, err := gt.Upload("https://v26-web.douyinvod.com/1671afda348c88fb13af3d73bc727002/6155656b/video/tos/cn/tos-cn-ve-15/1b4fef278b124e659a69c648793dd278/?a=6383&br=2249&bt=2249&cd=0%7C0%7C0&ch=26&cr=0&cs=0&cv=1&dr=0&ds=4&er=&ft=~MeSY~88-o10DN7nh7TQPqeUfTusSYjVWy&l=2021093014201201021205308106205B1D&lr=all&mime_type=video_mp4&net=0&pl=0&qs=0&rc=amlybzk6ZnF2ODMzNGkzM0ApZjQ2M2g2OTw0Nzo7OzxmZmdrbS1scjRnaC9gLS1kLTBzczYyMmMzYGJiYGAwYzQ1YDU6Yw%3D%3D&vl=&vr=", 38 | // "/home/mange/Desktop/dy1.mp4") 39 | //log.Println(ctx.Resp.StatusCode, err) 40 | 41 | caseUrl := "https://www.douyin.com/video/7000282511469268264" 42 | ctx, _ := gt.Get(caseUrl) 43 | html := ctx.RespBodyString() 44 | log.Println("state = ", ctx.Resp.Status) 45 | log.Println("html = ", html) 46 | dom,err := goquery.NewDocumentFromReader(strings.NewReader(ctx.RespBodyString())) 47 | if err != nil{ 48 | log.Println(err) 49 | return 50 | } 51 | result := dom.Find("script[id=RENDER_DATA]") 52 | log.Println(result.Html()) 53 | res,_ := result.Html() 54 | unescape, _ := url.QueryUnescape(res) 55 | log.Println(unescape) 56 | 57 | m := make(map[string]interface{}) 58 | err = json.Unmarshal([]byte(unescape), &m) 59 | log.Println("json err = ", err) 60 | log.Println("m = ", m, "\n\n\n\n") 61 | 62 | c19 := m["C_19"].(map[string]interface{}) 63 | //log.Println("c19 = ", c19) 64 | 65 | aweme := c19["aweme"].(map[string]interface{}) 66 | //log.Println("aweme = ", aweme) 67 | 68 | detail := aweme["detail"].(map[string]interface{}) 69 | log.Println("detail = ", detail) 70 | 71 | awemeId := detail["awemeId"].(string) 72 | log.Println("awemeId = ", awemeId) 73 | 74 | awemeType := detail["awemeType"] 75 | log.Println("awemeType = ", awemeType) 76 | 77 | groupId := detail["groupId"] 78 | log.Println("groupId = ", groupId) 79 | 80 | authorInfo := detail["authorInfo"].(map[string]interface{}) 81 | log.Println("authorInfo = ", authorInfo) 82 | 83 | desc := detail["desc"] 84 | log.Println("desc = ", desc) 85 | 86 | authorUserId := detail["authorUserId"] 87 | log.Println("authorUserId = ", authorUserId) 88 | 89 | createTime := detail["createTime"] 90 | log.Println("createTime = ", createTime) 91 | 92 | video := detail["video"] 93 | log.Println("video = ", video) 94 | 95 | download := detail["download"].(map[string]interface{}) 96 | log.Println("download = ", download) 97 | 98 | url := download["url"].(string) 99 | ctx1, err := gt.Upload(url,"/home/mange/Desktop/"+awemeId+".mp4") 100 | log.Println(ctx1.Resp.StatusCode, err) 101 | 102 | } 103 | 104 | /* 105 | */ -------------------------------------------------------------------------------- /extract.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : HTML内容提取； json内容提取 TODO 测试 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "encoding/json" 12 | "fmt" 13 | "reflect" 14 | "strings" 15 | 16 | "golang.org/x/net/html" 17 | ) 18 | 19 | // GetPointHTML 获取指定位置的HTML， 用标签， 标签属性， 属性值来定位 20 | func GetPointHTML(htmlStr, label, attr, val string) ([]string, error) { 21 | rse := make([]string, 0) 22 | doc, err := html.Parse(strings.NewReader(htmlStr)) 23 | if err != nil { 24 | return rse, err 25 | } 26 | var f func(*html.Node) 27 | f = func(n *html.Node) { 28 | if n.Type == html.ElementNode { 29 | //log.Println("attr = ", n.Attr, n.Namespace, n.Data) 30 | if n.Data == label { 31 | if attr == "" && val == "" { 32 | rse = add(rse, n) 33 | } else { 34 | for _, a := range n.Attr { 35 | if a.Key == attr && a.Val == val { 36 | rse = add(rse, n) 37 | } 38 | } 39 | } 40 | } 41 | } 42 | for c := n.FirstChild; c != nil; c = c.NextSibling { 43 | f(c) 44 | } 45 | } 46 | f(doc) 47 | return rse, nil 48 | } 49 | 50 | func add(rse []string, n *html.Node) []string { 51 | var buf bytes.Buffer 52 | err := html.Render(&buf, n) 53 | if err == nil { 54 | rse = append(rse, buf.String()) 55 | } 56 | return rse 57 | } 58 | 59 | // GetPointIDHTML 获取指定标签id属性的html 60 | func GetPointIDHTML(htmlStr, label, val string) ([]string, error) { 61 | return GetPointHTML(htmlStr, label, "id", val) 62 | } 63 | 64 | // GetPointClassHTML 获取指定标签class属性的html 65 | func GetPointClassHTML(htmlStr, label, val string) ([]string, error) { 66 | return GetPointHTML(htmlStr, label, "class", val) 67 | } 68 | 69 | // ==================================== json find 70 | 71 | // JsonFind 按路径寻找指定json值 72 | // 用法参考 ./_examples/json/main.go 73 | // @find : 寻找路径，与目录的url类似， 下面是一个例子： 74 | // json: {a:[{b:1},{b:2}]} 75 | // find=/a/[0] => {b:1} 76 | // find=a/[0]/b => 1 77 | func JsonFind(jsonStr, find string) (any, error) { 78 | if !IsJson(jsonStr) { 79 | return nil, fmt.Errorf("不是标准的Json格式") 80 | } 81 | jxList := strings.Split(find, "/") 82 | jxLen := len(jxList) 83 | var ( 84 | data = Any2Map(jsonStr) 85 | value any 86 | err error 87 | ) 88 | for i := 0; i < jxLen; i++ { 89 | l := len(jxList[i]) 90 | if l > 2 && string(jxList[i][0]) == "[" && string(jxList[i][l-1]) == "]" { 91 | numStr := jxList[i][1 : l-1] 92 | dataList := Any2Arr(value) 93 | value = dataList[Any2Int(numStr)] 94 | data, err = interface2Map(value) 95 | if err != nil { 96 | continue 97 | } 98 | } else { 99 | if IsHaveKey(data, jxList[i]) { 100 | value = data[jxList[i]] 101 | data, err = interface2Map(value) 102 | if err != nil { 103 | continue 104 | } 105 | } else { 106 | value = nil 107 | } 108 | } 109 | } 110 | return value, nil 111 | } 112 | 113 | // JsonFind2Json 寻找json,输出 json格式字符串 114 | func JsonFind2Json(jsonStr, find string) (string, error) { 115 | value, err := JsonFind(jsonStr, find) 116 | if err != nil { 117 | return "", err 118 | } 119 | return Any2Json(value) 120 | } 121 | 122 | // JsonFind2Map 寻找json,输出 map[string]any 123 | func JsonFind2Map(jsonStr, find string) (map[string]any, error) { 124 | value, err := JsonFind(jsonStr, find) 125 | if err != nil { 126 | return nil, err 127 | } 128 | return Any2Map(value), nil 129 | } 130 | 131 | // JsonFind2Arr 寻找json,输出 []any 132 | func JsonFind2Arr(jsonStr, find string) ([]any, error) { 133 | value, err := JsonFind(jsonStr, find) 134 | if err != nil { 135 | return nil, err 136 | } 137 | return Any2Arr(value), nil 138 | } 139 | 140 | // JsonFind2Str 寻找json,输出字符串 141 | func JsonFind2Str(jsonStr, find string) (string, error) { 142 | value, err := JsonFind(jsonStr, find) 143 | if err != nil { 144 | return "", err 145 | } 146 | return Any2String(value), nil 147 | } 148 | 149 | // JsonFind2Int 寻找json,输出int 150 | func JsonFind2Int(jsonStr, find string) (int, error) { 151 | value, err := JsonFind(jsonStr, find) 152 | if err != nil { 153 | return 0, err 154 | } 155 | return Any2Int(value), nil 156 | } 157 | 158 | // JsonFind2Int64 寻找json,输出int64 159 | func JsonFind2Int64(jsonStr, find string) (int64, error) { 160 | value, err := JsonFind(jsonStr, find) 161 | if err != nil { 162 | return 0, err 163 | } 164 | return Any2Int64(value), nil 165 | } 166 | 167 | // IsJson 是否是json格式 168 | func IsJson(str string) bool { 169 | var tempMap map[string]any 170 | err := json.Unmarshal([]byte(str), &tempMap) 171 | if err != nil { 172 | return false 173 | } 174 | return true 175 | } 176 | 177 | // IsHaveKey map[string]any 是否存在 输入的key 178 | func IsHaveKey[T comparable](data map[T]any, key T) bool { 179 | _, ok := data[key] 180 | return ok 181 | } 182 | 183 | // Any2Map any -> map[string]any 184 | func interface2Map(data any) (map[string]any, error) { 185 | if v, ok := data.(map[string]any); ok { 186 | return v, nil 187 | } 188 | if reflect.ValueOf(data).Kind() == reflect.String { 189 | return Json2Map(data.(string)) 190 | } 191 | return nil, fmt.Errorf("not map type") 192 | } 193 | -------------------------------------------------------------------------------- /stress_testing.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 接口压力测试， 并输出结果 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "log" 11 | "net" 12 | "net/http" 13 | "sync" 14 | "sync/atomic" 15 | "time" 16 | ) 17 | 18 | type stateCodeData struct { 19 | Code int 20 | ReqTime int64 21 | } 22 | 23 | // StressUrl 压力测试一个url 24 | type StressUrl struct { 25 | Url string 26 | Method string 27 | Sum int64 28 | Total int 29 | TQueue TodoQueue 30 | 31 | // 请求时间累加 32 | sumReqTime int64 33 | 34 | // 测试结果 35 | avgReqTime time.Duration 36 | 37 | // 接口传入的json 38 | JsonData string 39 | 40 | // 接口传入类型 41 | ContentType string 42 | 43 | // 是否重试 44 | isRetry bool 45 | 46 | stateCodeList []*stateCodeData 47 | stateCodeListMux *sync.Mutex 48 | } 49 | 50 | // NewTestUrl 实例化一个新的url压测 51 | func NewTestUrl(url, method string, sum int64, total int) *StressUrl { 52 | return &StressUrl{ 53 | Url: url, 54 | Method: method, 55 | Sum: sum, 56 | Total: total, 57 | TQueue: NewQueue(), 58 | sumReqTime: int64(0), 59 | isRetry: false, 60 | stateCodeList: make([]*stateCodeData, 0), 61 | stateCodeListMux: &sync.Mutex{}, 62 | } 63 | } 64 | 65 | // SetJson 设置json 66 | func (s *StressUrl) SetJson(str string) { 67 | s.JsonData = str 68 | } 69 | 70 | // OpenRetry 开启重试请求 71 | func (s *StressUrl) OpenRetry() { 72 | s.isRetry = true 73 | } 74 | 75 | // Run 运行压测 76 | func (s *StressUrl) Run(vs ...any) { 77 | //解析可变参 78 | var ( 79 | succeedFunc SucceedFunc 80 | n int64 81 | wg sync.WaitGroup 82 | reqTimeout ReqTimeOut 83 | reqTimeoutMs ReqTimeOutMs 84 | header http.Header 85 | ) 86 | for _, v := range vs { 87 | switch vv := v.(type) { 88 | // 使用方传入了 header 89 | case SucceedFunc: 90 | succeedFunc = vv 91 | case ReqTimeOut: 92 | reqTimeout = vv 93 | case ReqTimeOutMs: 94 | reqTimeoutMs = vv 95 | case http.Header: 96 | header = vv 97 | case *http.Header: 98 | header = *vv 99 | } 100 | } 101 | 102 | //初始化队列 103 | for n = 0; n < s.Sum; n++ { 104 | _ = s.TQueue.Add(&Task{Url: s.Url}) 105 | } 106 | log.Println("总执行次数： ", s.TQueue.Size()) 107 | var count int64 = 0 108 | for job := 0; job < s.Total; job++ { 109 | wg.Add(1) 110 | go func(i int) { 111 | defer wg.Done() 112 | for { 113 | var ( 114 | ctx = &Context{} 115 | ) 116 | if s.TQueue.IsEmpty() { 117 | break 118 | } 119 | task := s.TQueue.Poll() 120 | if task == nil { 121 | continue 122 | } 123 | // 定义适用于压力测试的client 124 | t := http.DefaultTransport.(*http.Transport).Clone() 125 | t.MaxIdleConns = s.Total * 2 126 | t.MaxIdleConnsPerHost = s.Total * 2 127 | t.DisableKeepAlives = true 128 | t.DialContext = (&net.Dialer{ 129 | Timeout: 3 * time.Second, 130 | KeepAlive: 3 * time.Second, 131 | }).DialContext 132 | t.IdleConnTimeout = 3 * time.Second 133 | t.ExpectContinueTimeout = 1 * time.Second 134 | client := http.Client{ 135 | Transport: t, 136 | Timeout: 1 * time.Second, 137 | } 138 | switch s.Method { 139 | case "get", "Get", "GET": 140 | ctx = NewGet(task.Url, client, succeedFunc, reqTimeout, reqTimeoutMs, header) 141 | case "post", "Post", "POST": 142 | ctx = NewPost(task.Url, []byte(s.JsonData), s.ContentType, client, succeedFunc, reqTimeout, reqTimeoutMs, header) 143 | default: 144 | log.Println("暂时不支持的 Method.") 145 | } 146 | if ctx == nil { 147 | continue 148 | } 149 | ctx.JobNumber = i 150 | if !s.isRetry { 151 | ctx.CloseRetry() 152 | } 153 | ctx.Do() 154 | 155 | atomic.AddInt64(&s.sumReqTime, int64(ctx.Ms)) 156 | atomic.AddInt64(&count, int64(1)) 157 | s.stateCodeListMux.Lock() 158 | if ctx.Resp != nil { 159 | s.stateCodeList = append(s.stateCodeList, &stateCodeData{ 160 | Code: ctx.Resp.StatusCode, 161 | ReqTime: int64(ctx.Ms), 162 | }) 163 | } else { 164 | //请求错误 165 | s.stateCodeList = append(s.stateCodeList, &stateCodeData{ 166 | Code: -1, 167 | ReqTime: int64(ctx.Ms), 168 | }) 169 | } 170 | s.stateCodeListMux.Unlock() 171 | } 172 | 173 | }(job) 174 | } 175 | wg.Wait() 176 | Info("执行次数 : ", count) 177 | 178 | var ( 179 | maxTime int64 = 0 180 | minTime int64 = 9999999999 181 | ) 182 | 183 | fb := make(map[int]int) 184 | for _, v := range s.stateCodeList { 185 | if v.ReqTime >= maxTime { 186 | maxTime = v.ReqTime 187 | } 188 | if v.ReqTime <= minTime { 189 | minTime = v.ReqTime 190 | } 191 | 192 | if _, ok := fb[v.Code]; ok { 193 | fb[v.Code]++ 194 | } else { 195 | fb[v.Code] = 1 196 | } 197 | //s.sumReqTime = s.sumReqTime + v.ReqTime 198 | } 199 | 200 | Info("状态码分布: ", fb) 201 | avg := float64(s.sumReqTime) / float64(s.Sum) 202 | avg = avg / (1000 * 1000) 203 | Info("平均用时： ", avg, "ms") 204 | Info("最高用时: ", float64(maxTime)/(1000*1000), "ms") 205 | Info("最低用时: ", float64(minTime)/(1000*1000), "ms") 206 | Info("执行完成！！！") 207 | } 208 | -------------------------------------------------------------------------------- /_examples/websocket_client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | gt "github.com/mangenotwork/gathertool" 5 | "log" 6 | "math/rand" 7 | "time" 8 | ) 9 | 10 | type Token struct { 11 | Code int64 `json:"code"` 12 | Msg string `json:"msg"` 13 | Time int64 `json:"time"` 14 | B Body `json:"body"` 15 | } 16 | 17 | type Body struct { 18 | P Pager `json:"pager"` 19 | D Data `json:"data"` 20 | } 21 | 22 | type Pager struct { 23 | Page int64 `json:"page"` 24 | Count int64 `json:"count"` 25 | PageCount int64 `json:"page_count"` 26 | } 27 | 28 | type Data struct { 29 | Userid int64 `json:"userid"` 30 | Appid int64 `json:"appid"` 31 | TokenString string `json:"token_string"` 32 | Password string `json:"password"` 33 | } 34 | 35 | func main() { 36 | GetAllUser() 37 | } 38 | 39 | func main1() { 40 | idStr := "104" 41 | //tokenUrl := "https://test.api.ymzy.cn/kimi/logic/allow/init/user" 42 | //par := `{ 43 | // "nick_name":"test-` + idStr + `", 44 | // "user_id":104, 45 | // "password":"1", 46 | // "app_id":1 47 | // }` 48 | //ctx, _ := gt.PostJson(tokenUrl, par) 49 | // 50 | //m := Token{} 51 | // 52 | //err := json.Unmarshal(ctx.RespBody, &m) 53 | //if err != nil { 54 | // log.Println("Umarshal failed:", err) 55 | // return 56 | //} 57 | //log.Println("Token = ", m.B.D.TokenString) 58 | 59 | /* 60 | /v1/ws? 61 | user_id=304930&app_id=1&name=%E6%88%91%E8%BF%98%E5%93%88%E5%93%88%E5%93%88%E5%93%88%E5%93%88%E5%93%88%E5%93%88%E5%93%88%E5%93%88%E5%93%88 62 | &avatar=https://p2.ymzy.cn/ave/20220215/7ebc2087f6aba85c.jpg 63 | &token=bmJmIjoxNjI2MzMxNzYzLCJuaWNrbmFtZSI6IjEyMzAwMDAwMDIzIiwicHJvdl9pZCI6NTAsInByb3ZfbW9kZWwiOjIsIn 64 | &device_id=304930-e9f44268-b747-4c2e-971c-1c15a855a15a 65 | &source=1 66 | 67 | */ 68 | 69 | token := "bmJmIjoxNjI2MzMxNzYzLCJuaWNrbmFtZSI6IjEyMzAwMDAwMDIzIiwicHJvdl9pZCI6NTAsInByb3ZfbW9kZWwiOjIsIn" 70 | 71 | // 连接 72 | host := "wss://test.api.ymzy.cn" 73 | path := "/kimi/v1/ws?user_id=" + idStr + "&device_id=" + idStr + "&token=" + token + "&app_id=1&name=104&avatar=&source=1" 74 | wsc, err := gt.WsClient(host, path, false) 75 | log.Println(wsc, err) 76 | rand.Seed(time.Now().UnixNano()) 77 | 78 | //// 发消息 79 | //gt.PostForm("http://192.168.0.9:8001/auth/send", map[string]string{ 80 | // "conversation_type":"0", 81 | // "sender_type":"2", 82 | // "app_id":"1", 83 | // "from":idStr, 84 | // "from_device_id":"test_"+idStr, 85 | // "from_avatar": userAvatar, 86 | // "from_nickname": userNickName, 87 | // "to": "-1", 88 | // "content_type":"1", 89 | // "content":"你好我找一下客服", 90 | // 91 | //}) 92 | 93 | for { 94 | // CHANGE 95 | // QUESTION 96 | // ARTIFICIAL 97 | // commands := []string{"CHANGE", "QUESTION", "ARTIFICIAL"} 98 | // 99 | // command := `{ 100 | // "type":12, 101 | // "request_id":"", 102 | // "command": "`+commands[rand.Intn(len(commands)-1)]+`", 103 | // "device_id":"123123213123213213", 104 | // "app_id":1, 105 | // "user_id":104, 106 | // "receiver_id":1, 107 | // "seq":12321321321323 108 | //}` 109 | // 110 | // err = wsc.Send([]byte(command)) 111 | // log.Println(err) 112 | data := make([]byte, 100) 113 | err = wsc.Read(data) 114 | log.Println(err) 115 | log.Println("data = ", string(data)) 116 | time.Sleep(5 * time.Second) 117 | //time.Sleep(10*time.Microsecond) 118 | } 119 | 120 | wsc.Close() 121 | } 122 | 123 | var ( 124 | host197 = "192.168.0.197" 125 | port = 3306 126 | user = "root" 127 | password = "root123" 128 | gkzyUser = "gkzy-user" 129 | GkzyUserDB, _ = gt.NewMysql(host197, port, user, password, gkzyUser) 130 | ) 131 | 132 | func GetAllUser() { 133 | data, _ := GkzyUserDB.Select("select * from user limit 300, 100") 134 | for _, v := range data { 135 | log.Println(v["user_id"]) 136 | go Work(v["user_id"], v["avatar"], v["nick_name"]) 137 | } 138 | select {} 139 | } 140 | 141 | func Work(idStr, userAvatar, userNickName string) { 142 | token := "bmJmIjoxNjI2MzMxNzYzLCJuaWNrbmFtZSI6IjEyMzAwMDAwMDIzIiwicHJvdl9pZCI6NTAsInByb3ZfbW9kZWwiOjIsIn" 143 | 144 | // 连接 145 | host := "wss://test.api.ymzy.cn" 146 | path := "/kimi/v1/ws?user_id=" + idStr + "&device_id=" + idStr + "&token=" + token + "&app_id=1&name=104&avatar=&source=1" 147 | wsc, err := gt.WsClient(host, path, false) 148 | log.Println(wsc, err) 149 | rand.Seed(time.Now().UnixNano()) 150 | 151 | for { 152 | // 发消息 153 | header := gt.NewHeader(map[string]string{ 154 | "kimi-token": token, 155 | }) 156 | ctx, _ := gt.PostJson("https://test.api.ymzy.cn/kimi/logic/auth/send", `{ 157 | "conversation_type":0, 158 | "sender_type":2, 159 | "app_id":1, 160 | "from":`+idStr+`, 161 | "from_device_id":"test_`+idStr+`", 162 | "from_avatar": "`+userAvatar+`", 163 | "from_nickname": "`+userNickName+`", 164 | "to": -1, 165 | "content_type":1, 166 | "content":"[自动化压力测试] 你好我找一下客服" 167 | }`, header) 168 | 169 | log.Println(ctx.Json) 170 | 171 | data := make([]byte, 100) 172 | if wsc != nil { 173 | //err = wsc.Read(data) 174 | //log.Println(err) 175 | log.Println("data = ", string(data)) 176 | } 177 | 178 | time.Sleep(30 * time.Second) 179 | //time.Sleep(10*time.Microsecond) 180 | } 181 | 182 | wsc.Close() 183 | } 184 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : logger TODO 测试 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "os" 13 | "runtime" 14 | "strconv" 15 | "strings" 16 | "time" 17 | ) 18 | 19 | // LogClose 是否关闭日志 20 | var LogClose = true 21 | var std = newStd() 22 | 23 | // CloseLog 关闭日志 24 | func CloseLog() { 25 | LogClose = false 26 | } 27 | 28 | type logger struct { 29 | outFile bool 30 | outFileWriter *os.File 31 | } 32 | 33 | func newStd() *logger { 34 | return &logger{} 35 | } 36 | 37 | // SetLogFile 设置日志输出到的指定文件 38 | func SetLogFile(name string) { 39 | std.outFile = true 40 | std.outFileWriter, _ = os.OpenFile(name+time.Now().Format("-20060102")+".log", 41 | os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 42 | } 43 | 44 | // Level 日志等级 45 | type Level int 46 | 47 | var LevelMap = map[Level]string{ 48 | 1: "[Info] ", 49 | 2: "[Debug] ", 50 | 3: "[Warn] ", 51 | 4: "[Error] ", 52 | 5: "[HTTP/s]", 53 | } 54 | 55 | // Log 日志 56 | func (l *logger) Log(level Level, args string, times int) { 57 | var buffer bytes.Buffer 58 | buffer.WriteString(time.Now().Format(TimeMilliTemplate)) 59 | buffer.WriteString(LevelMap[level]) 60 | _, file, line, _ := runtime.Caller(times) 61 | fileList := strings.Split(file, "/") 62 | // 最多显示两级路径 63 | if len(fileList) > 3 { 64 | fileList = fileList[len(fileList)-3:] 65 | } 66 | buffer.WriteString(strings.Join(fileList, "/")) 67 | buffer.WriteString(":") 68 | buffer.WriteString(strconv.Itoa(line)) 69 | buffer.WriteString(" \t| ") 70 | buffer.WriteString(args) 71 | buffer.WriteString("\n") 72 | out := buffer.Bytes() 73 | if LogClose { 74 | _, _ = buffer.WriteTo(os.Stdout) 75 | } 76 | 77 | // 输出到文件或远程日志服务 78 | if l.outFile { 79 | _, _ = l.outFileWriter.Write(out) 80 | } 81 | } 82 | 83 | // Info 日志-信息 84 | func Info(args ...any) { 85 | std.Log(1, fmt.Sprint(args...), 2) 86 | } 87 | 88 | // InfoF 日志-信息 89 | func InfoF(format string, args ...any) { 90 | std.Log(1, fmt.Sprintf(format, args...), 2) 91 | } 92 | 93 | // InfoTimes 日志-信息, 指定日志代码位置的定位调用层级 94 | func InfoTimes(times int, args ...any) { 95 | std.Log(1, fmt.Sprint(args...), times) 96 | } 97 | 98 | // InfoFTimes 日志-信息, 指定日志代码位置的定位调用层级 99 | func InfoFTimes(times int, format string, args ...any) { 100 | std.Log(1, fmt.Sprintf(format, args...), times) 101 | } 102 | 103 | // Debug 日志-调试 104 | func Debug(args ...any) { 105 | std.Log(2, fmt.Sprint(args...), 2) 106 | } 107 | 108 | // DebugF 日志-调试 109 | func DebugF(format string, args ...any) { 110 | std.Log(2, fmt.Sprintf(format, args...), 2) 111 | } 112 | 113 | // DebugTimes 日志-调试, 指定日志代码位置的定位调用层级 114 | func DebugTimes(times int, args ...any) { 115 | std.Log(1, fmt.Sprint(args...), times) 116 | } 117 | 118 | // DebugFTimes 日志-调试, 指定日志代码位置的定位调用层级 119 | func DebugFTimes(format string, times int, args ...any) { 120 | std.Log(1, fmt.Sprintf(format, args...), times) 121 | } 122 | 123 | // Warn 日志-警告 124 | func Warn(args ...any) { 125 | std.Log(3, fmt.Sprint(args...), 2) 126 | } 127 | 128 | // WarnF 日志-警告 129 | func WarnF(format string, args ...any) { 130 | std.Log(3, fmt.Sprintf(format, args...), 2) 131 | } 132 | 133 | // WarnTimes 日志-警告, 指定日志代码位置的定位调用层级 134 | func WarnTimes(times int, args ...any) { 135 | std.Log(1, fmt.Sprint(args...), times) 136 | } 137 | 138 | // WarnFTimes 日志-警告, 指定日志代码位置的定位调用层级 139 | func WarnFTimes(format string, times int, args ...any) { 140 | std.Log(1, fmt.Sprintf(format, args...), times) 141 | } 142 | 143 | // Error 日志-错误 144 | func Error(args ...any) { 145 | std.Log(4, fmt.Sprint(args...), 2) 146 | } 147 | 148 | // ErrorF 日志-错误 149 | func ErrorF(format string, args ...any) { 150 | std.Log(4, fmt.Sprintf(format, args...), 2) 151 | } 152 | 153 | // ErrorTimes 日志-错误, 指定日志代码位置的定位调用层级 154 | func ErrorTimes(times int, args ...any) { 155 | std.Log(4, fmt.Sprint(args...), times) 156 | } 157 | 158 | // ErrorFTimes 日志-错误, 指定日志代码位置的定位调用层级 159 | func ErrorFTimes(format string, times int, args ...any) { 160 | std.Log(4, fmt.Sprintf(format, args...), times) 161 | } 162 | 163 | func Panic(args ...any) { 164 | panic(args) 165 | } 166 | 167 | // HTTPTimes 日志-信息, 指定日志代码位置的定位调用层级 168 | func HTTPTimes(times int, args ...any) { 169 | std.Log(5, fmt.Sprint(args...), times) 170 | } 171 | 172 | // Bar 终端显示的进度条 173 | type Bar struct { 174 | percent int64 //百分比 175 | cur int64 //当前进度位置 176 | total int64 //总进度 177 | rate string //进度条 178 | graph string //显示符号 179 | } 180 | 181 | func (bar *Bar) NewOption(start, total int64) { 182 | bar.cur = start 183 | bar.total = total 184 | if bar.graph == "" { 185 | bar.graph = "█" 186 | } 187 | bar.percent = bar.getPercent() 188 | for i := 0; i < int(bar.percent); i += 2 { 189 | bar.rate += bar.graph //初始化进度条位置 190 | } 191 | } 192 | 193 | func (bar *Bar) getPercent() int64 { 194 | return int64(float32(bar.cur) / float32(bar.total) * 100) 195 | } 196 | 197 | func (bar *Bar) NewOptionWithGraph(start, total int64, graph string) { 198 | bar.graph = graph 199 | bar.NewOption(start, total) 200 | } 201 | 202 | func (bar *Bar) Play(cur int64) { 203 | bar.cur = cur 204 | last := bar.percent 205 | bar.percent = bar.getPercent() 206 | if bar.percent != last && bar.percent%2 == 0 { 207 | bar.rate += bar.graph 208 | } 209 | fmt.Printf("\r[%-50s]%3d%% %8d/%d", bar.rate, bar.percent, bar.cur, bar.total) 210 | } 211 | 212 | func (bar *Bar) Finish() { 213 | fmt.Println() 214 | } 215 | -------------------------------------------------------------------------------- /_examples/baojia/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | url : http://www.100ppi.com/mprice/mlist-1--1.html 3 | 抓取商品报价 4 | */ 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | gt "github.com/mangenotwork/gathertool" 10 | "log" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | // 数据库对象 16 | var ( 17 | host192 = "127.0.0.1" 18 | port = 3306 19 | user = "root" 20 | password = "123456" 21 | dbName = "sp_data" 22 | DB, _ = gt.NewMysql(host192, port, user, password, dbName) 23 | Host = "http://www.100ppi.com/mprice/" 24 | ) 25 | 26 | func main() { 27 | //GetMPrice() 28 | 29 | //RunGetMList() 30 | 31 | RunGetDetail() 32 | } 33 | 34 | // GetMPrice 抓取商品分类数据处理后存入数据库 35 | func GetMPrice() { 36 | OutTable := "table_fenlei" 37 | caseUrl := "https://www.100ppi.com/mprice/plist-1-1-1.html" 38 | ctx, _ := gt.Get(caseUrl) 39 | data, _ := gt.GetPointClassHTML(ctx.Html, "div", "p_list2") 40 | for _, v := range data { 41 | aDataList := gt.RegHtmlA(v) 42 | for _, aData := range aDataList { 43 | gt.Info(aData) 44 | href := "https://www.100ppi.com/mprice/" + gt.RegHtmlHrefTxt(aData)[0] 45 | fenlei := gt.RegHtmlATxt(aData)[0] 46 | fenlei = gt.RegDelHtml(fenlei) 47 | if strings.Index(href, "plist") == -1 { 48 | continue 49 | } 50 | gt.Info("href = ", href) 51 | gt.Info("fenlei = ", fenlei) 52 | gt.Info("数据提取完成，准备写入数据库") 53 | gd := gt.NewGDMap().Add("fenlei", fenlei).Add("href", href) 54 | err := DB.InsertAtGd(OutTable, gd) 55 | if err != nil { 56 | panic(err) 57 | } 58 | } 59 | } 60 | } 61 | 62 | // RunGetMList 从数据库中获取分类链接进行抓取商品列表 63 | func RunGetMList() { 64 | hrefList, err := DB.Select("select * from table_fenlei") 65 | if err != nil { 66 | log.Panic(err) 67 | } 68 | for _, hrefData := range hrefList { 69 | href := hrefData["href"] 70 | href = href[0:len(href)-6] + "%d.html" 71 | gt.Info(href) 72 | GetMList(href) 73 | } 74 | } 75 | 76 | // GetMList 抓取商品列表数据并处理后存储到数据库 77 | func GetMList(caseUrl string) { 78 | OutTable := "m_list" 79 | // 50页 80 | for i := 1; i < 11; i++ { 81 | time.Sleep(200 * time.Millisecond) 82 | // 请求 83 | ctx, _ := gt.Get(fmt.Sprintf(caseUrl, i)) 84 | // 数据提取 85 | table := gt.RegHtmlTable(ctx.Html) 86 | if len(table) > 0 { 87 | trList := gt.RegHtmlTr(table[0]) 88 | for _, tr := range trList { 89 | log.Println("tr = ", tr) 90 | tdList := gt.RegHtmlTd(tr) 91 | if len(tdList) == 0 { 92 | continue 93 | } 94 | spNameHtml := tdList[0] 95 | spName := gt.RegHtmlATxt(spNameHtml)[0] 96 | spUrl := Host + gt.RegHtmlHrefTxt(spNameHtml)[0] 97 | spNoteHtml := tdList[1] 98 | spNote := gt.CleaningStr(gt.RegHtmlDivTxt(spNoteHtml)[0]) 99 | pingpaiHtml := tdList[2] 100 | pingpai := gt.RegHtmlTdTxt(pingpaiHtml)[0] 101 | baojiaHtml := tdList[3] 102 | baojia := gt.CleaningStr(gt.RegHtmlTdTxt(baojiaHtml)[0]) 103 | baojiaTypeHtml := tdList[4] 104 | baojiaType := gt.CleaningStr(gt.RegHtmlTdTxt(baojiaTypeHtml)[0]) 105 | diziHtml := tdList[5] 106 | dizi := gt.CleaningStr(gt.RegHtmlTdTxt(diziHtml)[0]) 107 | shangjiaHtml := tdList[6] 108 | shangjia := "" 109 | shangjiaA := gt.RegHtmlATxt(shangjiaHtml) 110 | if len(shangjiaA) > 0 { 111 | shangjia = shangjiaA[0] 112 | } else { 113 | shangjia = gt.RegHtmlDivTxt(shangjiaHtml)[0] 114 | } 115 | shangjiaUrl := "" 116 | shangjiaUrlList := gt.RegHtmlHrefTxt(spNameHtml) 117 | if len(shangjiaUrlList) > 0 { 118 | shangjiaUrl = Host + shangjiaUrlList[0] 119 | } 120 | dateHtml := tdList[7] 121 | date := gt.RegHtmlTdTxt(dateHtml)[0] 122 | log.Println("商品名称 = ", spName) 123 | log.Println("商品详情地址 = ", spUrl) 124 | log.Println("规格 = ", spNote) 125 | log.Println("品牌/产地 = ", pingpai) 126 | log.Println("报价 = ", baojia) 127 | log.Println("报价类型 = ", baojiaType) 128 | log.Println("交货地 = ", dizi) 129 | log.Println("交易商 = ", shangjia) 130 | log.Println("交易商详情地址 = ", shangjiaUrl) 131 | log.Println("发布时间 = ", date) 132 | // 固定字段顺序Map, 写入mysql数据库 133 | gd := gt.NewGDMap().Add("sp_name", spName).Add("sp_url", spUrl).Add("sp_note", spNote) 134 | gd.Add("pingpai", pingpai).Add("baojia", baojia).Add("baojia_type", baojiaType) 135 | gd.Add("dizi", dizi).Add("shangjia", shangjia).Add("shangjia_url", shangjiaUrl) 136 | gd.Add("date", date).Add("req_md5", ctx.CheckMd5()) 137 | err := DB.InsertAtGd(OutTable, gd) 138 | if err != nil { 139 | panic(err) 140 | } 141 | } 142 | } 143 | } 144 | } 145 | 146 | // RunGetDetail 从数据库中获取商品详细链接进行抓取商品详情和商家信息 147 | func RunGetDetail() { 148 | caseList, err := DB.Select("select * from m_list") 149 | if err != nil { 150 | log.Panic(err) 151 | } 152 | for _, caseData := range caseList { 153 | GetDetail(caseData["sp_url"]) 154 | } 155 | } 156 | 157 | // GetDetail 抓取商品详细数据并处理后存储到数据库 158 | func GetDetail(caseUrl string) { 159 | //caseUrl := "http://www.100ppi.com/mprice/detail-4264843.html" 160 | time.Sleep(200 * time.Millisecond) 161 | ctx, _ := gt.Get(caseUrl) 162 | //gt.Info(ctx.Html) 163 | 164 | // 提取商品详情信息 165 | infoData, _ := gt.GetPointClassHTML(ctx.Html, "table", "st2-table tac") 166 | //gt.Info("infoData = ", infoData) 167 | 168 | // 提取商品描述 169 | noteData, _ := gt.GetPointClassHTML(ctx.Html, "table", "mb20 st2-table tac") 170 | //gt.Info("noteData = ", noteData[0]) 171 | 172 | // 提取报价走势图 173 | imgData, _ := gt.GetPointClassHTML(ctx.Html, "div", "vain") 174 | //gt.Info("imgData = ", imgData) 175 | 176 | if len(noteData) < 2 { 177 | return 178 | } 179 | 180 | // 商家信息 181 | shangJiaData := noteData[1] 182 | //gt.Info("shangJiaData = ", shangJiaData) 183 | 184 | // 存储商品详细信息 185 | gd1 := gt.NewGDMap().Add("info", infoData[0]).Add("note", noteData[0]).Add("img", imgData[0]) 186 | gd1.Add("shangjia", shangJiaData) 187 | err := DB.InsertAtGd("sp_detail", gd1) 188 | if err != nil { 189 | panic(err) 190 | } 191 | 192 | // 将商家信息单独提取出来存储数据 193 | trList := gt.RegHtmlTrTxt(shangJiaData) 194 | 195 | gs := GetSJData(trList[1]) 196 | gt.Info(gs) 197 | fzr := GetSJData(trList[2]) 198 | gt.Info(fzr) 199 | lx1 := GetSJData(trList[3]) 200 | gt.Info(lx1) 201 | lx2 := GetSJData(trList[4]) 202 | gt.Info(lx2) 203 | gd2 := gt.NewGDMap().Add("gs", gs).Add("fzr", fzr).Add("lx1", lx1).Add("lx2", lx2) 204 | err = DB.InsertAtGd("sj_info", gd2) 205 | if err != nil { 206 | panic(err) 207 | } 208 | 209 | } 210 | 211 | func GetSJData(data string) string { 212 | data = gt.RegDelHtml(data) 213 | data = strings.Replace(data, "\n", "", -1) 214 | data = strings.Replace(data, "\r", "", -1) 215 | data = strings.Replace(data, "\t", "", -1) 216 | data = strings.Replace(data, " ", "", -1) 217 | data = strings.Replace(data, "进入报价通", "", -1) 218 | data = strings.Replace(data, "联系报价人", "", -1) 219 | data = strings.Replace(data, "扫码分享", "", -1) 220 | return data 221 | } 222 | -------------------------------------------------------------------------------- /_examples/ip_bczs_cn/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "strings" 7 | "time" 8 | 9 | "github.com/PuerkitoBio/goquery" 10 | gt "github.com/mangenotwork/gathertool" 11 | ) 12 | 13 | func main() { 14 | //main2() 15 | main1() 16 | } 17 | 18 | var ( 19 | // 全局声明抓取任务队列 20 | queue = gt.NewQueue() 21 | // 全局声明数据库客户端对象 22 | //host = "192.168.0.192" 23 | //port = 3306 24 | //user = "root" 25 | //password = "root123" 26 | //dbname = "test" 27 | //db,_ = gt.NewMysql(host, port, user, password, dbname) 28 | c, _ = gt.NewCSV("data.xls") 29 | ) 30 | 31 | func main1() { 32 | // 1.在页面 http://ip.bczs.net/country/CN 获取所有ip 33 | _, _ = gt.Get("http://ip.bczs.net/country/CN", gt.SucceedFunc(IPListSucceed)) 34 | 35 | // 2. 并发抓取详情数据, 20个并发 36 | gt.StartJobGet(10, queue, 37 | gt.SucceedFunc(GetIPSucceed), //请求成功后执行的方法 38 | gt.RetryFunc(GetIPRetry), //遇到 502,403 等状态码重试前执行的方法，一般为添加休眠时间或更换代理 39 | gt.FailedFunc(GetIPFailed), //请求失败后执行的方法 40 | ) 41 | //c.Close() 42 | 43 | select {} 44 | 45 | } 46 | 47 | // IPListSucceed 请求成功执行 48 | func IPListSucceed(ctx *gt.Context) { 49 | i := 1 50 | for _, tbody := range gt.RegHtmlTbody(ctx.RespBodyString()) { 51 | for _, tr := range gt.RegHtmlTr(tbody) { 52 | if i > 500 { 53 | goto End 54 | } 55 | td := gt.RegHtmlTdTxt(tr) 56 | log.Println(td) 57 | if len(td) < 3 { 58 | gt.Error("异常数据 ： ", td) 59 | continue 60 | } 61 | startIp := gt.Any2String(gt.RegHtmlATxt(td[0])[0]) // IP起始 62 | endIP := td[1] // 结束 63 | number := td[2] // 数量 64 | // 创建队列 抓取详情信息 65 | queue.Add(>.Task{ 66 | Url: "http://ip.bczs.net/" + startIp, 67 | Data: map[string]interface{}{ 68 | "start_ip": startIp, 69 | "end_ip": endIP, 70 | "number": number, 71 | }, 72 | }) 73 | i++ 74 | } 75 | } 76 | End: 77 | } 78 | 79 | // GetIPSucceed 获取详情信息成功的处理 80 | func GetIPSucceed(cxt *gt.Context) { 81 | // 使用goquery库提取数据 82 | dom, err := goquery.NewDocumentFromReader(strings.NewReader(cxt.RespBodyString())) 83 | if err != nil { 84 | log.Println(err) 85 | return 86 | } 87 | result, err := dom.Find("div[id=result] .well").Html() 88 | if err != nil { 89 | log.Println(err) 90 | } 91 | // 固定顺序map 92 | gd := gt.NewGDMap().Add("start_ip", cxt.Task.GetDataStr("start_ip")) 93 | gd.Add("end_ip", cxt.Task.GetDataStr("end_ip")) 94 | gd.Add("number", cxt.Task.GetDataStr("number")).Add("result", result) 95 | log.Println("采集到的数据: ", []string{cxt.Task.GetDataStr("start_ip"), cxt.Task.GetDataStr("end_ip"), 96 | cxt.Task.GetDataStr("number"), result}) 97 | // 保存抓取数据 98 | //err = db.InsertAtGd("ip_result", gd) 99 | 100 | if err != nil { 101 | panic(err) 102 | } 103 | err = c.Add([]string{cxt.Task.GetDataStr("start_ip"), cxt.Task.GetDataStr("end_ip"), 104 | cxt.Task.GetDataStr("number"), result}) 105 | log.Println("写入csv失败: ", err) 106 | if err != nil { 107 | panic(err) 108 | } 109 | 110 | } 111 | 112 | // GetIPRetry 获取详情信息重试的处理 113 | func GetIPRetry(c *gt.Context) { 114 | //更换代理 115 | //c.SetProxy(uri) 116 | 117 | // or 118 | c.Client = &http.Client{ 119 | // 设置代理 120 | //Transport: &http.Transport{ 121 | // Proxy: http.ProxyURL(uri), 122 | //}, 123 | Timeout: 5 * time.Second, 124 | } 125 | 126 | log.Println("休息1s") 127 | time.Sleep(1 * time.Second) 128 | } 129 | 130 | // GetIPFailed 获取详情信息失败执行返还给队列 131 | func GetIPFailed(c *gt.Context) { 132 | queue.Add(c.Task) //请求失败归还到队列 133 | } 134 | 135 | // 第二种实现 136 | func main2() { 137 | // 连接数据库 138 | var ( 139 | host = "192.168.0.192" 140 | port = 3306 141 | user = "root" 142 | password = "root123" 143 | dbname = "test" 144 | ) 145 | db, err := gt.NewMysql(host, port, user, password, dbname) 146 | if err != nil { 147 | panic(err) 148 | } 149 | // 请求数据 150 | ctx, err := gt.Get("http://ip.bczs.net/country/CN") 151 | if err != nil { 152 | log.Println(err) 153 | return 154 | } 155 | // 提取数据并保存 156 | for _, tbody := range gt.RegHtmlTbody(ctx.Html) { 157 | for _, tr := range gt.RegHtmlTr(tbody) { 158 | td := gt.RegHtmlTdTxt(tr) 159 | if len(td) < 3 { 160 | gt.Error("异常数据 ： ", td) 161 | continue 162 | } 163 | // 保存抓取数据 164 | err := db.InsertAt("ip", map[string]interface{}{ 165 | "start": td[0], 166 | "end": td[1], 167 | "num": td[2], 168 | }) 169 | if err != nil { 170 | panic(err) 171 | } 172 | } 173 | } 174 | } 175 | 176 | // --------------------------------------------------- ip属地实例 177 | // 思路, 将所有ip转10进制 ip >= 属地IP段启始ip && ip <= 属地IP段结束ip 或得地址 178 | 179 | // 数据库对象 180 | var ( 181 | host192 = "192.168.3.2" 182 | port = 3306 183 | user = "root" 184 | password = "admin" 185 | dbName = "niu_pp" 186 | DB, _ = gt.NewMysql(host192, port, user, password, dbName) 187 | Host = "http://www.100ppi.com/mprice/" 188 | ) 189 | 190 | func Run() { 191 | // 1.在页面 http://ip.bczs.net/country/CN 获取所有ip 192 | ctx, _ := gt.Get("http://ip.bczs.net/country/MO") 193 | for _, tbody := range gt.RegHtmlTbody(ctx.RespBodyString()) { 194 | for _, tr := range gt.RegHtmlTr(tbody) { 195 | td := gt.RegHtmlTdTxt(tr) 196 | log.Println(td) 197 | if len(td) < 3 { 198 | gt.Error("异常数据 ： ", td) 199 | continue 200 | } 201 | startIp := gt.Any2String(gt.RegHtmlATxt(td[0])[0]) // IP起始 202 | GetIPInfo(startIp) 203 | time.Sleep(600 * time.Millisecond) 204 | } 205 | } 206 | } 207 | 208 | func NewTable() { 209 | fields := map[string]string{ 210 | "rse": "varchar(200)", 211 | "start": "varchar(50)", 212 | "start_int": "bigint", 213 | "end": "varchar(50)", 214 | "end_int": "bigint", 215 | "sd1": "varchar(50)", 216 | "sd2": "varchar(50)", 217 | "created": "int(11)", 218 | } 219 | err := DB.NewTable("ip_set", fields) 220 | log.Println(err) 221 | } 222 | 223 | func GetIPInfo(startIp string) { 224 | caseUrl := "http://ip.bczs.net/" + startIp 225 | log.Println("caseUrl = ", caseUrl) 226 | ctx, _ := gt.Get(caseUrl, gt.RetryFunc(GetIPRetry)) 227 | htmlData := ctx.Html 228 | //log.Println(htmlData) 229 | well, err := gt.GetPointClassHTML(htmlData, "div", "well") 230 | log.Println(well, err) 231 | 232 | data1 := gt.RegFindAll(`(?is:IP数据：.*?
)`, well[0]) 233 | log.Println(data1) 234 | rse := data1[0][0] 235 | rse = strings.Replace(rse, "IP数据：", "", -1) 236 | rse = strings.Replace(rse, "
", "", -1) 237 | log.Println("rse = ", rse) 238 | rseList := strings.Split(rse, " ") 239 | log.Println("rseList = ", rseList) 240 | ipArang := rseList[0] 241 | ipList := strings.Split(ipArang, "-") 242 | ipStart := ipList[0] 243 | ipStartInt := gt.IP2Int64(ipStart) 244 | ipEnd := ipList[1] 245 | ipEndInt := gt.IP2Int64(ipEnd) 246 | log.Println("ipStart = ", ipStart, ipStartInt) 247 | log.Println("ipEnd = ", ipEnd, ipEndInt) 248 | ipSD := rseList[1] 249 | ipSDList := strings.Split(ipSD, "\t") 250 | ipSD1, ipSD2 := "中国", "中国" 251 | if len(ipSDList) > 2 { 252 | ipSD1 = ipSDList[0] 253 | ipSD2 = ipSDList[1] 254 | } 255 | 256 | data := map[string]interface{}{ 257 | "rse": rse, 258 | "start": ipStart, 259 | "start_int": ipStartInt, 260 | "end": ipEnd, 261 | "end_int": ipEndInt, 262 | "sd1": ipSD1, 263 | "sd2": ipSD2, 264 | "created": time.Now().Unix(), 265 | } 266 | err = DB.InsertAt("ip_set", data) 267 | log.Println(err) 268 | } 269 | -------------------------------------------------------------------------------- /time_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 时间相关的操作 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "fmt" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | const TimeTemplate = "2006-01-02 15:04:05" 16 | const TimeMilliTemplate = "2006-01-02 15:04:05.000" 17 | const TimeTemplateYMD = "2006-01-02" 18 | const OneDay = 24 19 | const OneHours = 60 20 | const OneMinutes = 60 21 | const ZeroHMS = " 00:00:00" 22 | const TimeFormatList = "01-02 15:04" 23 | 24 | // Timestamp 获取时间戳 25 | func Timestamp() int64 { 26 | return time.Now().Unix() 27 | } 28 | 29 | func TimestampStr() string { 30 | return strconv.FormatInt(time.Now().Unix(), 10) 31 | } 32 | 33 | // BeginDayUnix 获取当天 0点 34 | func BeginDayUnix() int64 { 35 | timeStr := time.Now().Format(TimeTemplateYMD) 36 | t, _ := time.ParseInLocation(TimeTemplateYMD, timeStr, time.Local) 37 | return t.Unix() 38 | } 39 | 40 | // EndDayUnix 获取当天 24点 41 | func EndDayUnix() int64 { 42 | timeStr := time.Now().Format(TimeTemplateYMD) 43 | t, _ := time.ParseInLocation(TimeTemplateYMD, timeStr, time.Local) 44 | return t.Unix() + OneDay*OneHours*OneMinutes 45 | } 46 | 47 | // MinuteAgo 获取多少分钟前的时间戳 48 | func MinuteAgo(i int) int64 { 49 | return time.Now().Unix() - int64(i*OneMinutes) 50 | } 51 | 52 | // HourAgo 获取多少小时前的时间戳 53 | func HourAgo(i int) int64 { 54 | return time.Now().Unix() - int64(i*OneHours*OneMinutes) 55 | } 56 | 57 | // DayAgo 获取多少天前的时间戳 58 | func DayAgo(i int) int64 { 59 | return time.Now().Unix() - int64(i*OneHours*OneMinutes*OneDay) 60 | } 61 | 62 | // DayDiff 两个时间字符串的日期差, 返回的是天 63 | func DayDiff(beginDay string, endDay string) int { 64 | begin, _ := time.Parse(TimeTemplate, beginDay+ZeroHMS) 65 | end, _ := time.Parse(TimeTemplate, endDay+ZeroHMS) 66 | 67 | diff := end.Unix() - begin.Unix() 68 | return int(diff / (OneDay * OneHours * OneMinutes)) 69 | } 70 | 71 | func DayDiffAtUnix(s, e int64) int { 72 | diff := e - s 73 | return int(diff / (OneDay * OneHours * OneMinutes)) 74 | } 75 | 76 | // TickerRun 间隔运行 77 | // t: 间隔时间， runFirst: 间隔前或者后执行 f: 运行的方法 78 | func TickerRun(t time.Duration, runFirst bool, f func()) { 79 | if runFirst { 80 | f() 81 | } 82 | tick := time.NewTicker(t) 83 | for range tick.C { 84 | f() 85 | } 86 | } 87 | 88 | // Timestamp2Date timestamp -> "2006-01-02 15:04:05" 89 | func Timestamp2Date(timestamp int64) string { 90 | tm := time.Unix(timestamp, 0) 91 | return tm.Format(TimeTemplate) 92 | } 93 | 94 | // NowToEnd 计算当前时间到这天结束还有多久 95 | func NowToEnd() (int64, error) { 96 | now := time.Now() 97 | nextDay := now.Add(24 * time.Hour) 98 | loc, err := time.LoadLocation("Local") 99 | if err != nil { 100 | return -1, err 101 | } 102 | endTime := fmt.Sprintf("%d-%d-%d 00:00:00", nextDay.Year(), nextDay.Month(), nextDay.Day()) 103 | times, err := time.ParseInLocation(TimeTemplate, endTime, loc) 104 | if err != nil { 105 | return -1, err 106 | } 107 | distanceSec := times.Unix() - now.Unix() 108 | return distanceSec, nil 109 | } 110 | 111 | // IsToday 判断是否是今天 "2006-01-02 15:04:05" 112 | // timestamp 需要判断的时间 113 | func IsToday(timestamp int64) string { 114 | nowTime := time.Now() 115 | t := time.Unix(timestamp, 0) 116 | if t.Day() == nowTime.Day() && t.Month() == nowTime.Month() && t.Year() == nowTime.Year() { 117 | return "今天 " + t.Format("15:04:05") 118 | } 119 | return t.Format(TimeTemplate) 120 | } 121 | 122 | // IsTodayList 列表页的时间显示 "01-02 15:04" 123 | func IsTodayList(timestamp int64) string { 124 | nowTime := time.Now() 125 | t := time.Unix(timestamp, 0) 126 | if t.Day() == nowTime.Day() && t.Month() == nowTime.Month() && t.Year() == nowTime.Year() { 127 | return "今天 " + t.Format("15:04") 128 | } 129 | return t.Format(TimeFormatList) 130 | } 131 | 132 | // Timestamp2Week 获取timestamp是周几 133 | func Timestamp2Week(timestamp int64) string { 134 | tm := time.Unix(timestamp, 0) 135 | tm.Weekday() 136 | switch tm.Weekday() { 137 | case time.Sunday: 138 | return "周天" 139 | case time.Monday: 140 | return "周一" 141 | case time.Tuesday: 142 | return "周二" 143 | case time.Wednesday: 144 | return "周三" 145 | case time.Thursday: 146 | return "周四" 147 | case time.Friday: 148 | return "周五" 149 | case time.Saturday: 150 | return "周六" 151 | } 152 | return "周一" 153 | } 154 | 155 | // Timestamp2WeekXinQi 获取timestamp是星期几 156 | func Timestamp2WeekXinQi(timestamp int64) string { 157 | tm := time.Unix(timestamp, 0) 158 | tm.Weekday() 159 | switch tm.Weekday() { 160 | case time.Sunday: 161 | return "星期天" 162 | case time.Monday: 163 | return "星期一" 164 | case time.Tuesday: 165 | return "星期二" 166 | case time.Wednesday: 167 | return "星期三" 168 | case time.Thursday: 169 | return "星期四" 170 | case time.Friday: 171 | return "星期五" 172 | case time.Saturday: 173 | return "星期六" 174 | } 175 | return "星期一" 176 | } 177 | 178 | // LatestDate 返回最近好多天的日期 179 | func LatestDate(date int) []string { 180 | outList := make([]string, 0) 181 | now := time.Now() 182 | outList = append(outList, now.Format("2006-01-02")) 183 | nowInt := now.Unix() 184 | for i := 0; i < date; i++ { 185 | nowInt -= 86400 186 | outList = append(outList, time.Unix(nowInt, 0).Format("2006-01-02")) 187 | } 188 | return outList 189 | } 190 | 191 | // GetCurrentMonthRange 获取当前月的时间范围（月初1号0点 ~ 月末最后一刻） 192 | // 返回： 193 | // 194 | // firstDay：当月1号 00:00:00 195 | // lastDay：当月最后一天 23:59:59.999999999 196 | func GetCurrentMonthRange() (firstDay, lastDay time.Time) { 197 | now := time.Now() // 当前时间（当地时区） 198 | 199 | // 年、月不变，日设为1，时/分/秒/纳秒设为0 200 | firstDay = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) 201 | 202 | // 先获取下个月1号0点，再减1天得到当月最后一天0点，最后减1纳秒得到23:59:59.999999999 203 | nextMonthFirstDay := firstDay.AddDate(0, 1, 0) // 下个月1号0点 204 | lastDayStart := nextMonthFirstDay.AddDate(0, 0, -1) // 当月最后一天0点 205 | lastDay = lastDayStart.Add(24*time.Hour - time.Nanosecond) // 最后一天23:59:59.999999999 206 | 207 | return firstDay, lastDay 208 | } 209 | 210 | // GetCurrentWeekRange 获取当前周的时间范围（周一凌晨 ~ 周日23:59:59） 211 | // 返回： 212 | // 213 | // monday ：本周一 00:00:00 214 | // sundayEnd：本周日 23:59:59.999999999 215 | func GetCurrentWeekRange() (monday, sundayEnd time.Time) { 216 | now := time.Now() // 当前时间（当地时区） 217 | 218 | // 计算当前是星期几（转换为：周一=1，周二=2，...，周日=7） 219 | currentWeekday := int(now.Weekday()) 220 | if currentWeekday == 0 { // 若为周日（0），转为7 221 | currentWeekday = 7 222 | } 223 | 224 | // 计算距离本周一的天数差 225 | daysToMonday := currentWeekday - 1 226 | 227 | // 计算本周一的凌晨（00:00:00） 228 | monday = now.AddDate(0, 0, -daysToMonday).Truncate(24 * time.Hour) 229 | 230 | // 计算本周日的23:59:59.999999999（当天最后一刻） 231 | sundayDate := monday.AddDate(0, 0, 6) // 周一加6天 = 周日（日期） 232 | sundayEnd = time.Date( 233 | sundayDate.Year(), 234 | sundayDate.Month(), 235 | sundayDate.Day(), 236 | 23, 59, 59, 999999999, // 时:分:秒:纳秒（当天最后一刻） 237 | now.Location(), // 保持与当前时间相同的时区 238 | ) 239 | 240 | return monday, sundayEnd 241 | } 242 | 243 | // GetTodayRange 按东八区（Asia/Shanghai）获取当天自然日范围（00:00:00 至 23:59:59.999999999） 244 | // 返回： 245 | // 246 | // start：当天00:00:00（东八区） 247 | // end：当天23:59:59.999999999（东八区） 248 | func GetTodayRange() (start, end time.Time) { 249 | // 指定时区为东八区（Asia/Shanghai） 250 | loc, _ := time.LoadLocation("Asia/Shanghai") 251 | now := time.Now().In(loc) 252 | 253 | // 计算东八当天凌晨（00:00:00） 254 | start = time.Date( 255 | now.Year(), 256 | now.Month(), 257 | now.Day(), 258 | 0, 0, 0, 0, 259 | loc, // 明确使用东八 260 | ) 261 | 262 | // 计算东八当天结束（23:59:59.999999999） 263 | end = time.Date( 264 | now.Year(), 265 | now.Month(), 266 | now.Day(), 267 | 23, 59, 59, 999999999, 268 | loc, // 明确使用东八 269 | ) 270 | 271 | return start, end 272 | } 273 | -------------------------------------------------------------------------------- /scan.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 一些扫描方法,一些website host方法 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "fmt" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | var ApplicationTerminalOut = true 16 | 17 | // ApplicationTerminalOutClose 关闭在终端打印的扫描信息 18 | func ApplicationTerminalOutClose() { 19 | ApplicationTerminalOut = false 20 | } 21 | 22 | type HostToolEr interface { 23 | Run() ([]string, int) 24 | } 25 | 26 | // HostScanUrl Host站点下 A标签 Url扫描， 从更目录开始扫描指定深度 get Url 应用函数 27 | type HostScanUrl struct { 28 | Host string 29 | Depth int // 页面深度 30 | UrlSet map[string]struct{} 31 | Count int 32 | MaxCount int64 33 | } 34 | 35 | // NewHostScanUrl 创建扫描站点 36 | func NewHostScanUrl(host string, depth int) *HostScanUrl { 37 | return &HostScanUrl{ 38 | Host: host, 39 | Depth: depth, 40 | UrlSet: make(map[string]struct{}), 41 | Count: 0, 42 | MaxCount: 0, 43 | } 44 | } 45 | 46 | // Run 执行扫描 47 | func (scan *HostScanUrl) Run() ([]string, int) { 48 | CloseLog() 49 | scan.do(scan.Host, 0) 50 | urls := make([]string, 0) 51 | for k := range scan.UrlSet { 52 | urls = append(urls, urlStr(k)) 53 | } 54 | if ApplicationTerminalOut { 55 | fmt.Printf("\n") 56 | } 57 | return urls, scan.Count 58 | } 59 | 60 | func (scan *HostScanUrl) do(caseUrl string, df int) { 61 | if len(caseUrl) < 1 { 62 | return 63 | } 64 | if df > scan.Depth { 65 | return 66 | } 67 | // 如果不是host下的域名 68 | if strings.Index(caseUrl, scan.Host) == -1 { 69 | if string(caseUrl[0]) == "/" { 70 | caseUrl = scan.Host + caseUrl 71 | goto G 72 | } 73 | return 74 | } 75 | G: 76 | if _, ok := scan.UrlSet[caseUrl]; ok { 77 | return 78 | } 79 | ctx, err := Get(caseUrl) 80 | if err != nil { 81 | Error(err) 82 | return 83 | } 84 | if ApplicationTerminalOut { 85 | fmt.Print(".") 86 | } 87 | df++ 88 | scan.UrlSet[caseUrl] = struct{}{} 89 | scan.Count++ 90 | a := RegHtmlA(ctx.Html) 91 | for _, v := range a { 92 | links := RegHtmlHrefTxt(v) 93 | if len(links) < 1 { 94 | continue 95 | } 96 | link := links[0] 97 | // 请求并验证 98 | scan.do(link, df) 99 | } 100 | } 101 | 102 | // HostScanExtLinks Host站点下的外链采集 应用函数 103 | type HostScanExtLinks struct { 104 | Host string 105 | } 106 | 107 | // NewHostScanExtLinks 创建站点链接采集，只支持get请求 108 | func NewHostScanExtLinks(host string) *HostScanExtLinks { 109 | return &HostScanExtLinks{ 110 | Host: host, 111 | } 112 | } 113 | 114 | func (scan *HostScanExtLinks) Run() ([]string, int) { 115 | var ( 116 | urlSet = make(map[string]struct{}) 117 | count int 118 | ) 119 | CloseLog() 120 | ctx, err := Get(scan.Host) 121 | if err != nil { 122 | Error(err) 123 | return []string{}, 0 124 | } 125 | a := RegHtmlA(ctx.Html) 126 | for _, v := range a { 127 | links := RegHtmlHrefTxt(v) 128 | if len(links) < 1 { 129 | continue 130 | } 131 | link := links[0] 132 | if strings.Index(link, scan.Host) == -1 && string(link[0]) != "/" && string(link[0]) != "#" { 133 | if _, ok := urlSet[link]; ok { 134 | continue 135 | } 136 | if UsefulUrl(link) { 137 | urlSet[link] = struct{}{} 138 | count++ 139 | } 140 | } 141 | } 142 | urls := make([]string, 0) 143 | for k := range urlSet { 144 | urls = append(urls, k) 145 | } 146 | return urls, count 147 | } 148 | 149 | // HostScanBadLink Host站点下 HTML Get Url 死链接扫描 应用函数 150 | type HostScanBadLink struct { 151 | Host string 152 | Depth int // 页面深度 153 | PageState map[string]int 154 | UrlSet map[string]struct{} // 检查页面重复 155 | } 156 | 157 | // NewHostScanBadLink 创建站点死链接扫描，只支持get请求 158 | func NewHostScanBadLink(host string, depth int) *HostScanBadLink { 159 | return &HostScanBadLink{ 160 | Host: host, 161 | Depth: depth, 162 | PageState: make(map[string]int), 163 | UrlSet: make(map[string]struct{}), 164 | } 165 | } 166 | 167 | func (scan *HostScanBadLink) Run() ([]string, int) { 168 | CloseLog() 169 | scan.do(scan.Host, 0) 170 | badUrls := make([]string, 0) 171 | for k, v := range scan.PageState { 172 | if v == 404 || v == 410 { 173 | badUrls = append(badUrls, k) 174 | } 175 | } 176 | if ApplicationTerminalOut { 177 | fmt.Printf("\n") 178 | } 179 | return badUrls, len(badUrls) 180 | } 181 | 182 | func (scan *HostScanBadLink) Result() map[string]int { 183 | res := make(map[string]int) 184 | for k, v := range scan.PageState { 185 | if v == 404 || v == 410 { 186 | res[k] = v 187 | } 188 | } 189 | return res 190 | } 191 | 192 | func (scan *HostScanBadLink) Report() map[string]int { 193 | return scan.PageState 194 | } 195 | 196 | func (scan *HostScanBadLink) do(caseUrl string, df int) { 197 | if df > scan.Depth { 198 | return 199 | } 200 | if len(caseUrl) < 1 { 201 | return 202 | } 203 | if string(caseUrl[0]) == "/" { 204 | caseUrl = scan.Host + caseUrl 205 | } 206 | if _, ok := scan.UrlSet[caseUrl]; ok { 207 | return 208 | } 209 | ctx, err := Get(caseUrl) 210 | if err != nil { 211 | ctx.StateCode = 404 212 | return 213 | } 214 | if ApplicationTerminalOut { 215 | fmt.Print(".") 216 | } 217 | df++ 218 | scan.UrlSet[caseUrl] = struct{}{} 219 | scan.PageState[caseUrl] = ctx.StateCode 220 | a := RegHtmlA(ctx.Html) 221 | for _, v := range a { 222 | links := RegHtmlHrefTxt(v) 223 | if len(links) < 1 { 224 | continue 225 | } 226 | link := links[0] 227 | // 请求并验证 228 | scan.do(link, df) 229 | } 230 | } 231 | 232 | // HostPageSpeedCheck Host站点下 HTML Get 测速 应用函数 233 | type HostPageSpeedCheck struct { 234 | Host string 235 | Depth int // 页面深度 236 | PageSpeed map[string]time.Duration 237 | UrlSet map[string]struct{} // 检查页面重复 238 | } 239 | 240 | // NewHostPageSpeedCheck 创建站点所有url测速，只支持get请求 241 | func NewHostPageSpeedCheck(host string, depth int) *HostPageSpeedCheck { 242 | return &HostPageSpeedCheck{ 243 | Host: host, 244 | Depth: depth, 245 | PageSpeed: make(map[string]time.Duration), 246 | UrlSet: make(map[string]struct{}), 247 | } 248 | } 249 | 250 | // Run int 单位 ms 251 | func (scan *HostPageSpeedCheck) Run() ([]string, int) { 252 | CloseLog() 253 | scan.do(scan.Host, 0) 254 | urls := make([]string, 0) 255 | for k, v := range scan.PageSpeed { 256 | urls = append(urls, fmt.Sprintf("%s:%v", k, v)) 257 | } 258 | if ApplicationTerminalOut { 259 | fmt.Printf("\n") 260 | } 261 | return urls, len(urls) 262 | } 263 | 264 | func (scan *HostPageSpeedCheck) Result() map[string]time.Duration { 265 | return scan.PageSpeed 266 | } 267 | 268 | func (scan *HostPageSpeedCheck) do(caseUrl string, df int) { 269 | if len(caseUrl) < 1 { 270 | return 271 | } 272 | if df > scan.Depth { 273 | return 274 | } 275 | if string(caseUrl[0]) == "/" { 276 | caseUrl = scan.Host + caseUrl 277 | } 278 | if _, ok := scan.UrlSet[caseUrl]; ok { 279 | return 280 | } 281 | if strings.Index(caseUrl, scan.Host) == -1 { 282 | return 283 | } 284 | ctx, err := Get(caseUrl) 285 | if err != nil { 286 | Error(err) 287 | return 288 | } 289 | if ApplicationTerminalOut { 290 | fmt.Print(".") 291 | } 292 | df++ 293 | scan.UrlSet[caseUrl] = struct{}{} 294 | scan.PageSpeed[caseUrl] = ctx.Ms 295 | a := RegHtmlA(ctx.Html) 296 | for _, v := range a { 297 | links := RegHtmlHrefTxt(v) 298 | if len(links) < 1 { 299 | continue 300 | } 301 | link := links[0] 302 | // 请求并验证 303 | scan.do(link, df) 304 | } 305 | } 306 | 307 | // AverageSpeed 平均时间 308 | func (scan *HostPageSpeedCheck) AverageSpeed() float64 { 309 | var ( 310 | n int64 = 0 311 | t int64 = 0 312 | ) 313 | for _, v := range scan.PageSpeed { 314 | n++ 315 | t += v.Milliseconds() 316 | } 317 | return float64(t) / float64(n) 318 | } 319 | 320 | // MaxSpeed 最高用时 321 | func (scan *HostPageSpeedCheck) MaxSpeed() int64 { 322 | var maxTmp int64 = 0 323 | for _, v := range scan.PageSpeed { 324 | if v.Milliseconds() > maxTmp { 325 | maxTmp = v.Milliseconds() 326 | } 327 | } 328 | return maxTmp 329 | } 330 | 331 | // MinSpeed 最低用时 332 | func (scan *HostPageSpeedCheck) MinSpeed() int64 { 333 | var minTmp int64 = 0 334 | for _, v := range scan.PageSpeed { 335 | if v.Milliseconds() < minTmp { 336 | minTmp = v.Milliseconds() 337 | } 338 | } 339 | return minTmp 340 | } 341 | 342 | // Report 报告 343 | func (scan *HostPageSpeedCheck) Report() map[string]string { 344 | report := make(map[string]string) 345 | report["avg"] = fmt.Sprintf("%f", scan.AverageSpeed()) 346 | report["max"] = fmt.Sprintf("%d", scan.MaxSpeed()) 347 | report["min"] = fmt.Sprintf("%d", scan.MinSpeed()) 348 | return report 349 | } 350 | -------------------------------------------------------------------------------- /gathertool.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 根据method提供对应的http请求方法，该库的请求方法会生成请求上下文，可以使用上下文的扩展功能来满足你当前的场景。 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "io" 13 | "mime/multipart" 14 | "net/http" 15 | "net/url" 16 | "os" 17 | "strings" 18 | ) 19 | 20 | // Get 执行GET请求，请求结果在 Context 对象里，请求错误会直接抛出 21 | // 22 | // caseUrl : 请求链接地址 23 | // 24 | // vs : 更多可选参数见 Req 25 | // 26 | // Example: 27 | // 28 | // ctx,err := Get("https://xxx.xxx") 29 | func Get(caseUrl string, vs ...any) (*Context, error) { 30 | ctx := NewGet(caseUrl, vs...) 31 | ctx.Do() 32 | return ctx, ctx.Err 33 | } 34 | 35 | // NewGet 定义一个GET请求的 Context ，但是不执行，执行需要调用 Context.Do 36 | // 37 | // caseUrl : 请求链接地址 38 | // 39 | // vs : 更多可选参数见 Req 40 | // 41 | // Example: 42 | // 43 | // NewGet("https://xxx.xxx").Do() 44 | func NewGet(caseUrl string, vs ...any) *Context { 45 | request, err := http.NewRequest(GET, urlStr(caseUrl), nil) 46 | if err != nil { 47 | Error(err) 48 | return &Context{ 49 | Err: reqErr(caseUrl, err), 50 | StateCode: RequestStateError, 51 | } 52 | } 53 | return Req(request, vs...) 54 | } 55 | 56 | // Post 执行POST请求，请求结果在 Context 对象里，请求错误会直接抛出 57 | // 58 | // caseUrl : 请求链接地址 59 | // 60 | // data : 请求的body 61 | // 62 | // contentType : 请求的content-type,如application/json; charset=UTF-8 63 | // 64 | // vs : 更多可选参数见 Req 65 | // 66 | // Example: 67 | // 68 | // ctx,err := Post("https://xxx.xxx", []byte("xx"), "application/json") 69 | func Post(caseUrl string, data []byte, contentType string, vs ...any) (*Context, error) { 70 | ctx := NewPost(caseUrl, data, contentType, vs...) 71 | ctx.Do() 72 | return ctx, ctx.Err 73 | } 74 | 75 | // NewPost 定义一个POST请求的 Context ，但是不执行，执行需要调用 Context.Do 76 | // 77 | // caseUrl : 请求链接地址 78 | // 79 | // data : 请求的body 80 | // 81 | // contentType : 请求的content-type,如application/json; charset=UTF-8 82 | // 83 | // vs : 更多可选参数见 Req 84 | // 85 | // Example: 86 | // 87 | // ctx := NewPost("https://xxx.xxx", []byte("xx"), "application/json") 88 | func NewPost(caseUrl string, data []byte, contentType string, vs ...any) *Context { 89 | request, err := http.NewRequest(POST, urlStr(caseUrl), bytes.NewBuffer(data)) 90 | if err != nil { 91 | Error(err) 92 | return &Context{ 93 | Err: reqErr(caseUrl, err), 94 | StateCode: RequestStateError, 95 | } 96 | } 97 | if contentType == "" { 98 | request.Header.Set("Content-Type", "application/json;") 99 | } else { 100 | request.Header.Set("Content-Type", contentType) 101 | } 102 | return Req(request, vs...) 103 | } 104 | 105 | // PostJson 执行Content-Type=json的POST请求，请求结果在 Context 对象里，请求错误会直接抛出 106 | // 107 | // caseUrl : 请求链接地址 108 | // 109 | // jsonStr : 请求body,应为json字符串 110 | // 111 | // vs : 更多可选参数见 Req 112 | // 113 | // Example: 114 | // 115 | // ctx,err := PostJson("https://xxx.xxx", `{"xx":"xx"}`) 116 | func PostJson(caseUrl string, jsonStr string, vs ...any) (*Context, error) { 117 | ctx := NewPost(caseUrl, []byte(jsonStr), "application/json; charset=UTF-8", vs...) 118 | ctx.Do() 119 | return ctx, ctx.Err 120 | } 121 | 122 | // PostForm 执行Content-Type=from-data的POST请求，请求结果在 Context 对象里，请求错误会直接抛出 123 | // 124 | // caseUrl : 请求链接地址 125 | // 126 | // data : 请求body,应为map[string]string 127 | // 128 | // vs : 更多可选参数见 Req 129 | // 130 | // Example: 131 | // 132 | // ctx,err := PostForm("https://xxx.xxx", map[string]string{"xx":"xx"}) 133 | func PostForm(caseUrl string, data map[string]string, vs ...any) (*Context, error) { 134 | postData := url.Values{} 135 | for k, v := range data { 136 | postData.Add(k, v) 137 | } 138 | request, err := http.NewRequest(POST, urlStr(caseUrl), strings.NewReader(postData.Encode())) 139 | if err != nil { 140 | Error(err) 141 | ctx := &Context{ 142 | Err: reqErr(caseUrl, err), 143 | StateCode: RequestStateError, 144 | } 145 | return ctx, ctx.Err 146 | } 147 | request.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") 148 | ctx := Req(request, vs...) 149 | ctx.Do() 150 | return ctx, ctx.Err 151 | } 152 | 153 | // PostFile 执行一个POST上传文件的请求，请求结果在 Context 对象里，请求错误会直接抛出 154 | // 155 | // caseUrl : 请求链接地址 156 | // 157 | // paramName : from-data的参数名 158 | // 159 | // filePath : 文件路径 160 | // 161 | // vs : 更多可选参数见 Req 162 | // 163 | // Example: 164 | // 165 | // ctx,err := PostFile("https://xxx.xxx","file","./test.txt") 166 | func PostFile(caseUrl, paramName, filePath string, vs ...any) (*Context, error) { 167 | f, err := os.Open(filePath) 168 | if err != nil { 169 | Error(err) 170 | ctx := &Context{ 171 | Err: reqErr(caseUrl, err), 172 | StateCode: RequestStateError, 173 | } 174 | return ctx, ctx.Err 175 | } 176 | defer func() { _ = f.Close() }() 177 | 178 | uploadBody := &bytes.Buffer{} 179 | writer := multipart.NewWriter(uploadBody) 180 | 181 | fWriter, err := writer.CreateFormFile("uploadFile", filePath) 182 | if err != nil { 183 | Info("copy file writer %v", err) 184 | } 185 | 186 | _, err = io.Copy(fWriter, f) 187 | if err != nil { 188 | Error(err) 189 | } 190 | 191 | fieldMap := map[string]string{ 192 | paramName: filePath, 193 | } 194 | 195 | for k, v := range fieldMap { 196 | _ = writer.WriteField(k, v) 197 | } 198 | 199 | err = writer.Close() 200 | if err != nil { 201 | Error(err) 202 | } 203 | 204 | request, err := http.NewRequest(POST, urlStr(caseUrl), uploadBody) 205 | if err != nil { 206 | Error(err) 207 | ctx := &Context{ 208 | Err: fmt.Errorf(caseUrl, " 请求错误， err: ", err.Error()), 209 | StateCode: RequestStateError, 210 | } 211 | return ctx, ctx.Err 212 | } 213 | 214 | request.Header.Add("Content-Type", writer.FormDataContentType()) 215 | ctx := Req(request, vs...) 216 | ctx.Do() 217 | return ctx, ctx.Err 218 | } 219 | 220 | // Put 执行一个PUT请求，请求结果在 Context 对象里，请求错误会直接抛出 221 | // 222 | // caseUrl : 请求链接地址 223 | // 224 | // data : 请求的body 225 | // 226 | // contentType : 请求的 content-type 227 | // 228 | // vs : 更多可选参数见 Req 229 | // 230 | // Example: 231 | // 232 | // ctx,err := Put("https://xxx.xxx",[]byte("xxx"),"") 233 | func Put(caseUrl string, data []byte, contentType string, vs ...any) (*Context, error) { 234 | ctx := NewPut(caseUrl, data, contentType, vs...) 235 | ctx.Do() 236 | return ctx, ctx.Err 237 | } 238 | 239 | // NewPut 定义一个PUT请求的 Context ，但是不执行，执行需要调用 Context.Do 240 | // 241 | // caseUrl : 请求链接地址 242 | // 243 | // data : 请求的body 244 | // 245 | // contentType : 请求的content-type,如application/json; charset=UTF-8 246 | // 247 | // vs : 更多可选参数见 Req 248 | // 249 | // Example: 250 | // 251 | // ctx := NewPut("https://xxx.xxx", []byte("xx"), "application/json") 252 | func NewPut(caseUrl string, data []byte, contentType string, vs ...any) *Context { 253 | request, err := http.NewRequest(PUT, urlStr(caseUrl), bytes.NewBuffer(data)) 254 | if err != nil { 255 | Error(err) 256 | return &Context{ 257 | Err: reqErr(caseUrl, err), 258 | StateCode: RequestStateError, 259 | } 260 | } 261 | request.Header.Set("Content-Type", contentType) 262 | return Req(request, vs...) 263 | } 264 | 265 | // Delete 执行一个DELETE请求，请求结果在 Context 对象里，请求错误会直接抛出 266 | func Delete(caseUrl string, vs ...any) (*Context, error) { 267 | ctx := NewDelete(caseUrl, vs...) 268 | ctx.Do() 269 | return ctx, ctx.Err 270 | } 271 | 272 | // NewDelete 定义一个DELETE请求的 Context ，但是不执行，执行需要调用 Context.Do 273 | func NewDelete(caseUrl string, vs ...any) *Context { 274 | request, err := http.NewRequest(DELETE, urlStr(caseUrl), nil) 275 | if err != nil { 276 | Error(err) 277 | return &Context{ 278 | Err: reqErr(caseUrl, err), 279 | StateCode: RequestStateError, 280 | } 281 | } 282 | return Req(request, vs...) 283 | } 284 | 285 | // Options 执行一个OPTIONS请求，请求结果在 Context 对象里，请求错误会直接抛出 286 | func Options(caseUrl string, vs ...any) (*Context, error) { 287 | ctx := NewOptions(caseUrl, vs...) 288 | ctx.Do() 289 | return ctx, ctx.Err 290 | } 291 | 292 | // NewOptions 定义一个OPTIONS请求的 Context ，但是不执行，执行需要调用 Context.Do 293 | func NewOptions(caseUrl string, vs ...any) *Context { 294 | request, err := http.NewRequest(OPTIONS, urlStr(caseUrl), nil) 295 | if err != nil { 296 | Error(err) 297 | return &Context{ 298 | Err: reqErr(caseUrl, err), 299 | StateCode: RequestStateError, 300 | } 301 | } 302 | return Req(request, vs...) 303 | } 304 | 305 | // Head 执行一个HEAD请求，请求结果在 Context 对象里，请求错误会直接抛出 306 | func Head(caseUrl string, vs ...any) (*Context, error) { 307 | request, err := http.NewRequest(HEAD, urlStr(caseUrl), nil) 308 | if err != nil { 309 | Error(err) 310 | return &Context{ 311 | Err: reqErr(caseUrl, err), 312 | StateCode: RequestStateError, 313 | }, err 314 | } 315 | ctx := Req(request, vs...) 316 | ctx.Do() 317 | return ctx, ctx.Err 318 | } 319 | 320 | // Patch 执行一个PATCH请求，请求结果在 Context 对象里，请求错误会直接抛出 321 | func Patch(caseUrl string, vs ...any) (*Context, error) { 322 | request, err := http.NewRequest(PATCH, urlStr(caseUrl), nil) 323 | if err != nil { 324 | Error(err) 325 | return &Context{ 326 | Err: reqErr(caseUrl, err), 327 | StateCode: RequestStateError, 328 | }, err 329 | } 330 | ctx := Req(request, vs...) 331 | ctx.Do() 332 | return ctx, ctx.Err 333 | } 334 | 335 | // Upload 执行GET下载请求，将下载的内容保存到指定路径终端会默认显示下载进度，并返回 Context 对象，请求错误会直接抛出 336 | // 337 | // caseUrl : 请求链接地址 338 | // 339 | // savePath : 下载内容保存路径 340 | // 341 | // vs : 更多可选参数见 Req 342 | // 343 | // Example: 344 | // 345 | // ctx := Upload("https://xxx.xxx", "./test.txt") 346 | func Upload(caseUrl, savePath string, vs ...any) (*Context, error) { 347 | ctx := NewGet(urlStr(caseUrl), vs) 348 | ctx.Upload(savePath) 349 | if ctx.Err != nil { 350 | return ctx, ctx.Err 351 | } 352 | return ctx, nil 353 | } 354 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 并发工作任务 TODO 测试 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "errors" 11 | "net/http" 12 | "runtime" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | // TodoQueue 任务队列 18 | type TodoQueue interface { 19 | Add(task *Task) error // 向队列中添加元素 20 | Poll() *Task // 移除队列中最前面的元素 21 | Clear() bool // 清空队列 22 | Size() int // 获取队列的元素个数 23 | IsEmpty() bool // 判断队列是否为空 24 | Print() // 打印 25 | } 26 | 27 | // Queue 队列 28 | type Queue struct { 29 | mux *sync.Mutex 30 | list []*Task 31 | } 32 | 33 | // Task 任务对象 34 | type Task struct { 35 | Url string 36 | JsonParam string 37 | HeaderMap *http.Header 38 | Data map[string]any // 上下文传递的数据 39 | Urls []*ReqUrl // 多步骤使用 40 | Type string // "", "upload", "do" 41 | SavePath string 42 | SaveDir string 43 | FileName string 44 | once *sync.Once 45 | } 46 | 47 | // NewTask 新建任务 48 | func NewTask() *Task { 49 | return &Task{ 50 | Data: make(map[string]any), 51 | Urls: make([]*ReqUrl, 0), 52 | once: &sync.Once{}, 53 | HeaderMap: &http.Header{}, 54 | } 55 | } 56 | 57 | func (task *Task) GetDataStr(key string) string { 58 | v, ok := task.Data[key] 59 | if ok { 60 | if vStr, yes := v.(string); yes { 61 | return vStr 62 | } 63 | } 64 | return "" 65 | } 66 | 67 | func (task *Task) AddData(key string, value any) *Task { 68 | task.once.Do(func() { 69 | task.Data = make(map[string]any) 70 | }) 71 | task.Data[key] = value 72 | return task 73 | } 74 | 75 | func (task *Task) SetUrl(urlStr string) *Task { 76 | task.Url = urlStr 77 | return task 78 | } 79 | 80 | func (task *Task) SetJsonParam(jsonStr string) *Task { 81 | task.JsonParam = jsonStr 82 | return task 83 | } 84 | 85 | func CrawlerTask(url, jsonParam string, vs ...any) *Task { 86 | header := &http.Header{} 87 | 88 | for _, v := range vs { 89 | switch vv := v.(type) { 90 | case http.Header: 91 | for key, values := range vv { 92 | for _, value := range values { 93 | header.Add(key, value) 94 | } 95 | } 96 | case *http.Header: 97 | for key, values := range *vv { 98 | for _, value := range values { 99 | header.Add(key, value) 100 | } 101 | } 102 | } 103 | } 104 | 105 | return &Task{ 106 | Url: url, 107 | JsonParam: jsonParam, 108 | HeaderMap: header, 109 | Data: make(map[string]any), 110 | Type: "do", 111 | } 112 | } 113 | 114 | // ReqUrl 单个请求地址对象 115 | type ReqUrl struct { 116 | Url string 117 | Method string 118 | Params map[string]any 119 | } 120 | 121 | // NewQueue 新建一个队列 122 | func NewQueue() TodoQueue { 123 | list := make([]*Task, 0) 124 | return &Queue{list: list, mux: &sync.Mutex{}} 125 | } 126 | 127 | // Add 向队列中添加元素 128 | func (q *Queue) Add(task *Task) error { 129 | q.mux.Lock() 130 | defer q.mux.Unlock() 131 | q.list = append(q.list, task) 132 | return nil 133 | } 134 | 135 | // Poll 移除队列中最前面的额元素 136 | func (q *Queue) Poll() *Task { 137 | q.mux.Lock() 138 | defer q.mux.Unlock() 139 | if q.IsEmpty() { 140 | return nil 141 | } 142 | 143 | first := q.list[0] 144 | q.list = q.list[1:] 145 | return first 146 | } 147 | 148 | func (q *Queue) Clear() bool { 149 | if q.IsEmpty() { 150 | return false 151 | } 152 | for i := 0; i < q.Size(); i++ { 153 | q.list[i].Url = "" 154 | } 155 | q.list = nil 156 | return true 157 | } 158 | 159 | func (q *Queue) Size() int { 160 | return len(q.list) 161 | } 162 | 163 | func (q *Queue) IsEmpty() bool { 164 | if len(q.list) == 0 { 165 | return true 166 | } 167 | return false 168 | } 169 | 170 | func (q *Queue) Print() { 171 | Info(q.list) 172 | } 173 | 174 | // UploadQueue 下载队列 175 | type UploadQueue struct { 176 | mux *sync.Mutex 177 | list []*Task 178 | } 179 | 180 | // NewUploadQueue 新建一个下载队列 181 | func NewUploadQueue() TodoQueue { 182 | list := make([]*Task, 0) 183 | return &UploadQueue{list: list, mux: &sync.Mutex{}} 184 | } 185 | 186 | // Add 向队列中添加元素 187 | func (q *UploadQueue) Add(task *Task) error { 188 | if task.Url == "" { 189 | return errors.New("task url is null") 190 | } 191 | if task.SavePath == "" && (task.SaveDir == "" || task.FileName == "") { 192 | return errors.New("save path is null") 193 | } 194 | q.mux.Lock() 195 | defer q.mux.Unlock() 196 | q.list = append(q.list, task) 197 | return nil 198 | } 199 | 200 | // Poll 移除队列中最前面的额元素 201 | func (q *UploadQueue) Poll() *Task { 202 | q.mux.Lock() 203 | defer q.mux.Unlock() 204 | if q.IsEmpty() { 205 | return nil 206 | } 207 | 208 | first := q.list[0] 209 | q.list = q.list[1:] 210 | return first 211 | } 212 | 213 | func (q *UploadQueue) Clear() bool { 214 | if q.IsEmpty() { 215 | return false 216 | } 217 | for i := 0; i < q.Size(); i++ { 218 | q.list[i].Url = "" 219 | } 220 | q.list = nil 221 | return true 222 | } 223 | 224 | func (q *UploadQueue) Size() int { 225 | return len(q.list) 226 | } 227 | 228 | func (q *UploadQueue) IsEmpty() bool { 229 | if len(q.list) == 0 { 230 | return true 231 | } 232 | return false 233 | } 234 | 235 | func (q *UploadQueue) Print() { 236 | Info(q.list) 237 | } 238 | 239 | // TODO 使用双向链表实现队列 240 | 241 | // StartJob 开始运行并发 242 | func StartJob(jobNumber int, queue TodoQueue, f func(task *Task)) { 243 | var wg sync.WaitGroup 244 | for job := 0; job < jobNumber; job++ { 245 | wg.Add(1) 246 | go func(i int) { 247 | Info("启动第", i, "个任务") 248 | defer wg.Done() 249 | for { 250 | if queue.IsEmpty() { 251 | break 252 | } 253 | task := queue.Poll() 254 | Info("第", i, "个任务取的值： ", task) 255 | f(task) 256 | } 257 | Info("第", i, "个任务结束！！") 258 | }(job) 259 | } 260 | wg.Wait() 261 | Info("执行完成！！！") 262 | } 263 | 264 | // Err2Retry 设置遇到错误执行 Retry 事件 265 | type Err2Retry bool 266 | 267 | // StartJobGet 并发执行Get,直到队列任务为空 268 | // jobNumber 并发数， 269 | // queue 全局队列， 270 | func StartJobGet(jobNumber int, queue TodoQueue, vs ...any) { 271 | 272 | var ( 273 | client *http.Client 274 | succeed SucceedFunc 275 | retry RetryFunc 276 | failed FailedFunc 277 | err2Retry Err2Retry 278 | sleep Sleep 279 | ) 280 | 281 | runtime.GOMAXPROCS(runtime.NumCPU()) 282 | 283 | for _, v := range vs { 284 | switch vv := v.(type) { 285 | case *http.Client: 286 | client = vv 287 | case SucceedFunc: 288 | succeed = vv 289 | case FailedFunc: 290 | failed = vv 291 | case RetryFunc: 292 | retry = vv 293 | case Err2Retry: 294 | err2Retry = vv 295 | case Sleep: 296 | sleep = vv 297 | Info("设置 sleep = ", sleep) 298 | } 299 | } 300 | 301 | var wg sync.WaitGroup 302 | for job := 0; job < jobNumber; job++ { 303 | wg.Add(1) 304 | go func(i int) { 305 | Info("启动第", i, "个任务") 306 | defer wg.Done() 307 | for { 308 | if queue.IsEmpty() { 309 | break 310 | } 311 | task := queue.Poll() 312 | Info("第", i, "个任务取的值： ", task) 313 | ctx := NewGet(task.Url, task) 314 | if client != nil { 315 | ctx.Client = client 316 | } 317 | if succeed != nil { 318 | ctx.SetSucceedFunc(succeed) 319 | } 320 | if retry != nil { 321 | ctx.SetRetryFunc(retry) 322 | } 323 | if failed != nil { 324 | ctx.SetFailedFunc(failed) 325 | } 326 | if err2Retry { 327 | ctx.OpenErr2Retry() 328 | } 329 | if sleep != 0 { 330 | time.Sleep(time.Duration(sleep) * time.Second) 331 | } 332 | switch task.Type { 333 | case "", "do": 334 | ctx.Do() 335 | case "upload": 336 | if task.SavePath == "" { 337 | task.SavePath = task.SaveDir + task.FileName 338 | } 339 | ctx.Upload(task.SavePath) 340 | default: 341 | ctx.Do() 342 | } 343 | } 344 | Info("第", i, "个任务结束！！") 345 | }(job) 346 | } 347 | wg.Wait() 348 | Info("执行完成！！！") 349 | } 350 | 351 | // StartJobPost 开始运行并发Post 352 | func StartJobPost(jobNumber int, queue TodoQueue, vs ...any) { 353 | var ( 354 | client *http.Client 355 | succeed SucceedFunc 356 | retry RetryFunc 357 | failed FailedFunc 358 | err2Retry Err2Retry 359 | sleep Sleep 360 | ) 361 | 362 | for _, v := range vs { 363 | switch vv := v.(type) { 364 | case *http.Client: 365 | client = vv 366 | case SucceedFunc: 367 | succeed = vv 368 | case FailedFunc: 369 | failed = vv 370 | case RetryFunc: 371 | retry = vv 372 | case Err2Retry: 373 | err2Retry = vv 374 | case Sleep: 375 | sleep = vv 376 | Info("设置 sleep = ", sleep) 377 | } 378 | } 379 | 380 | var wg sync.WaitGroup 381 | for job := 0; job < jobNumber; job++ { 382 | wg.Add(1) 383 | go func(i int) { 384 | Info("启动第", i, "个任务") 385 | defer wg.Done() 386 | for { 387 | if queue.IsEmpty() { 388 | break 389 | } 390 | task := queue.Poll() 391 | Info("第", i, "个任务取的值： ", task, task.HeaderMap) 392 | ctx := NewPost(task.Url, []byte(task.JsonParam), "application/json;", task) 393 | if client != nil { 394 | ctx.Client = client 395 | } 396 | if succeed != nil { 397 | ctx.SetSucceedFunc(succeed) 398 | } 399 | if retry != nil { 400 | ctx.SetRetryFunc(retry) 401 | } 402 | if failed != nil { 403 | ctx.SetFailedFunc(failed) 404 | } 405 | if err2Retry { 406 | ctx.OpenErr2Retry() 407 | } 408 | if sleep != 0 { 409 | ctx.sleep = sleep 410 | } 411 | switch task.Type { 412 | case "", "do": 413 | ctx.Do() 414 | case "upload": 415 | if task.SavePath == "" { 416 | task.SavePath = task.SaveDir + task.FileName 417 | } 418 | ctx.Upload(task.SavePath) 419 | default: 420 | ctx.Do() 421 | } 422 | 423 | } 424 | Info("第", i, "个任务结束！！") 425 | }(job) 426 | } 427 | wg.Wait() 428 | Info("执行完成！！！") 429 | } 430 | 431 | // CPUMax 多核执行 432 | func CPUMax() { 433 | runtime.GOMAXPROCS(runtime.NumCPU()) 434 | } 435 | 436 | // TODO: 优化并重构 437 | -------------------------------------------------------------------------------- /out_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 输出数据到文本相关的操作； 其中含csv的相关方法， excel，文本操作等 TODO 测试 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "archive/tar" 11 | "archive/zip" 12 | "crypto/md5" 13 | "encoding/csv" 14 | "encoding/hex" 15 | "fmt" 16 | "io" 17 | "os" 18 | "path" 19 | "path/filepath" 20 | "runtime" 21 | "strings" 22 | ) 23 | 24 | // Csv Csv格式文件 25 | type Csv struct { 26 | FileName string 27 | W *csv.Writer 28 | R *csv.Reader 29 | } 30 | 31 | // NewCSV 新创建一个csv对象 32 | func NewCSV(fileName string) (*Csv, error) { 33 | f, err := os.Create(fileName) 34 | if err != nil { 35 | Error(err.Error()) 36 | return nil, err 37 | } 38 | _, _ = f.WriteString("\xEF\xBB\xBF") 39 | csvObj := &Csv{FileName: fileName} 40 | csvObj.W = csv.NewWriter(f) 41 | csvObj.R = csv.NewReader(f) 42 | return csvObj, nil 43 | } 44 | 45 | func (c *Csv) Close() { 46 | c.Close() 47 | } 48 | 49 | func (c *Csv) Add(data []string) error { 50 | err := c.W.Write(data) 51 | if err != nil { 52 | return err 53 | } 54 | c.W.Flush() 55 | return nil 56 | } 57 | 58 | func (c *Csv) ReadAll() ([][]string, error) { 59 | return c.R.ReadAll() 60 | } 61 | 62 | // ReadCsvFile csv file -> [][]string 行列 63 | func ReadCsvFile(filename string) [][]string { 64 | file, err := os.Open(filename) 65 | if err != nil { 66 | panic(err) 67 | } 68 | defer func() { 69 | _ = file.Close() 70 | }() 71 | reader := csv.NewReader(file) 72 | reader.Comma = ',' 73 | allRecords, err := reader.ReadAll() 74 | if err != nil { 75 | panic(err) 76 | } 77 | return allRecords 78 | } 79 | 80 | func FileExists(name string) bool { 81 | if _, err := os.Stat(name); err != nil { 82 | if os.IsNotExist(err) { 83 | return false 84 | } 85 | } 86 | return true 87 | } 88 | 89 | // OutJsonFile 将data输出到json文件 90 | func OutJsonFile(data any, fileName string) error { 91 | var ( 92 | f *os.File 93 | err error 94 | ) 95 | if Exists(fileName) { //如果文件存在 96 | f, err = os.OpenFile(fileName, os.O_APPEND, 0666) //打开文件 97 | } else { 98 | f, err = os.Create(fileName) //创建文件 99 | } 100 | if err != nil { 101 | return err 102 | } 103 | str, err := Any2Json(data) 104 | if err != nil { 105 | return err 106 | } 107 | _, err = io.WriteString(f, str) 108 | if err != nil { 109 | return err 110 | } 111 | return nil 112 | } 113 | 114 | // GetAllFile 获取目录下的所有文件 115 | func GetAllFile(pathname string) ([]string, error) { 116 | s := make([]string, 0) 117 | rd, err := os.ReadDir(pathname) 118 | if err != nil { 119 | Error("read dir fail:", err) 120 | return s, err 121 | } 122 | for _, fi := range rd { 123 | if !fi.IsDir() { 124 | fullName := pathname + "/" + fi.Name() 125 | s = append(s, fullName) 126 | } 127 | } 128 | return s, nil 129 | } 130 | 131 | func subString(str string, start, end int) string { 132 | rs := []rune(str) 133 | length := len(rs) 134 | if start < 0 || start > length { 135 | Error("start is wrong") 136 | return "" 137 | } 138 | if end < start || end > length { 139 | Error("end is wrong") 140 | return "" 141 | } 142 | return string(rs[start:end]) 143 | } 144 | 145 | // DeCompressZIP zip解压文件 146 | func DeCompressZIP(zipFile, dest string) error { 147 | reader, err := zip.OpenReader(zipFile) 148 | if err != nil { 149 | return err 150 | } 151 | defer func() { 152 | _ = reader.Close() 153 | }() 154 | for _, file := range reader.File { 155 | rc, err := file.Open() 156 | if err != nil { 157 | return err 158 | } 159 | filename := dest + file.Name 160 | err = os.MkdirAll(subString(filename, 0, strings.LastIndex(filename, "/")), 0755) 161 | if err != nil { 162 | return err 163 | } 164 | w, err := os.Create(filename) 165 | if err != nil { 166 | return err 167 | } 168 | _, err = io.Copy(w, rc) 169 | if err != nil { 170 | return err 171 | } 172 | _ = w.Close() 173 | _ = rc.Close() 174 | } 175 | return nil 176 | } 177 | 178 | // DeCompressTAR tar 解压文件 179 | func DeCompressTAR(tarFile, dest string) error { 180 | file, err := os.Open(tarFile) 181 | if err != nil { 182 | Error(err) 183 | return err 184 | } 185 | defer func() { 186 | _ = file.Close() 187 | }() 188 | tr := tar.NewReader(file) 189 | for { 190 | hdr, err := tr.Next() 191 | if err == io.EOF { 192 | break 193 | } 194 | if err != nil { 195 | return err 196 | } 197 | filename := dest + hdr.Name 198 | err = os.MkdirAll(subString(filename, 0, strings.LastIndex(filename, "/")), 0755) 199 | if err != nil { 200 | return err 201 | } 202 | w, err := os.Create(filename) 203 | if err != nil { 204 | return err 205 | } 206 | _, err = io.Copy(w, tr) 207 | if err != nil { 208 | return err 209 | } 210 | _ = w.Close() 211 | } 212 | return nil 213 | } 214 | 215 | // DecompressionZipFile zip压缩文件 216 | func DecompressionZipFile(src, dest string) error { 217 | reader, err := zip.OpenReader(src) 218 | if err != nil { 219 | return err 220 | } 221 | defer func() { 222 | _ = reader.Close() 223 | }() 224 | for _, file := range reader.File { 225 | filePath := path.Join(dest, file.Name) 226 | if file.FileInfo().IsDir() { 227 | _ = os.MkdirAll(filePath, os.ModePerm) 228 | } else { 229 | if err = os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { 230 | return err 231 | } 232 | inFile, err := file.Open() 233 | if err != nil { 234 | return err 235 | } 236 | outFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) 237 | if err != nil { 238 | return err 239 | } 240 | _, err = io.Copy(outFile, inFile) 241 | if err != nil { 242 | return err 243 | } 244 | _ = inFile.Close() 245 | _ = outFile.Close() 246 | } 247 | } 248 | return nil 249 | } 250 | 251 | // CompressFiles 压缩很多文件 252 | // files 文件数组，可以是不同dir下的文件或者文件夹 253 | // dest 压缩文件存放地址 254 | func CompressFiles(files []string, dest string) error { 255 | d, _ := os.Create(dest) 256 | defer func() { 257 | _ = d.Close() 258 | }() 259 | w := zip.NewWriter(d) 260 | defer func() { 261 | _ = w.Close() 262 | }() 263 | for _, file := range files { 264 | err := compressFiles(file, "", w) 265 | if err != nil { 266 | return err 267 | } 268 | } 269 | return nil 270 | } 271 | 272 | func compressFiles(filePath string, prefix string, zw *zip.Writer) error { 273 | file, err := os.Open(filePath) 274 | if err != nil { 275 | return err 276 | } 277 | info, err := file.Stat() 278 | if err != nil { 279 | return err 280 | } 281 | if info.IsDir() { 282 | prefix = prefix + "/" + info.Name() 283 | fileInfos, err := file.Readdir(-1) 284 | if err != nil { 285 | return err 286 | } 287 | for _, fi := range fileInfos { 288 | f := file.Name() + "/" + fi.Name() 289 | err = compressFiles(f, prefix, zw) 290 | if err != nil { 291 | return err 292 | } 293 | } 294 | } else { 295 | header, err := zip.FileInfoHeader(info) 296 | if err != nil { 297 | return err 298 | } 299 | header.Name = prefix + "/" + header.Name 300 | writer, err := zw.CreateHeader(header) 301 | if err != nil { 302 | return err 303 | } 304 | _, err = io.Copy(writer, file) 305 | _ = file.Close() 306 | if err != nil { 307 | return err 308 | } 309 | } 310 | return nil 311 | } 312 | 313 | // CompressDirZip 压缩目录 314 | func CompressDirZip(src, outFile string) error { 315 | _ = os.RemoveAll(outFile) 316 | zipFile, err := os.Create(outFile) 317 | if err != nil { 318 | return err 319 | } 320 | defer func() { 321 | _ = zipFile.Close() 322 | }() 323 | archive := zip.NewWriter(zipFile) 324 | defer func() { 325 | _ = archive.Close() 326 | }() 327 | return filepath.Walk(src, func(path string, info os.FileInfo, _ error) error { 328 | if path == src { 329 | return nil 330 | } 331 | header, _ := zip.FileInfoHeader(info) 332 | header.Name = strings.TrimPrefix(path, src+`/`) 333 | if info.IsDir() { 334 | header.Name += `/` 335 | } else { 336 | header.Method = zip.Deflate 337 | } 338 | writer, _ := archive.CreateHeader(header) 339 | if !info.IsDir() { 340 | file, _ := os.Open(path) 341 | defer func() { 342 | _ = file.Close() 343 | }() 344 | _, _ = io.Copy(writer, file) 345 | } 346 | return nil 347 | }) 348 | } 349 | 350 | func Exists(path string) bool { 351 | if stat, err := os.Stat(path); stat != nil && !os.IsNotExist(err) { 352 | return true 353 | } 354 | return false 355 | } 356 | 357 | func IsDir(path string) bool { 358 | s, err := os.Stat(path) 359 | if err != nil { 360 | return false 361 | } 362 | return s.IsDir() 363 | } 364 | 365 | func IsFile(path string) bool { 366 | s, err := os.Stat(path) 367 | if err != nil { 368 | return false 369 | } 370 | return !s.IsDir() 371 | } 372 | 373 | // GetNowPath 获取当前运行路径 374 | func GetNowPath() string { 375 | pathData, err := os.Getwd() 376 | if err != nil { 377 | Error(err) 378 | } 379 | if runtime.GOOS == "windows" { 380 | return strings.Replace(pathData, "\\", "/", -1) 381 | } 382 | return pathData 383 | } 384 | 385 | // FileMd5sum 文件 Md5 386 | func FileMd5sum(fileName string) string { 387 | fin, err := os.OpenFile(fileName, os.O_RDONLY, 0644) 388 | if err != nil { 389 | Info(fileName, err) 390 | return "" 391 | } 392 | defer func() { 393 | _ = fin.Close() 394 | }() 395 | Buf, bufErr := os.ReadFile(fileName) 396 | if bufErr != nil { 397 | Info(fileName, bufErr) 398 | return "" 399 | } 400 | m := md5.Sum(Buf) 401 | return hex.EncodeToString(m[:16]) 402 | } 403 | 404 | // PathExists 目录不存在则创建 405 | func PathExists(path string) { 406 | _, err := os.Stat(path) 407 | if err != nil && os.IsNotExist(err) { 408 | _ = os.MkdirAll(path, 0777) 409 | } 410 | } 411 | 412 | // FileSizeFormat 字节的单位转换 保留两位小数 413 | func FileSizeFormat(fileSize int64) (size string) { 414 | if fileSize < 1024 { 415 | //return strconv.FormatInt(fileSize, 10) + "B" 416 | return fmt.Sprintf("%.2fB", float64(fileSize)/float64(1)) 417 | } else if fileSize < (1024 * 1024) { 418 | return fmt.Sprintf("%.2fKB", float64(fileSize)/float64(1024)) 419 | } else if fileSize < (1024 * 1024 * 1024) { 420 | return fmt.Sprintf("%.2fMB", float64(fileSize)/float64(1024*1024)) 421 | } else if fileSize < (1024 * 1024 * 1024 * 1024) { 422 | return fmt.Sprintf("%.2fGB", float64(fileSize)/float64(1024*1024*1024)) 423 | } else if fileSize < (1024 * 1024 * 1024 * 1024 * 1024) { 424 | return fmt.Sprintf("%.2fTB", float64(fileSize)/float64(1024*1024*1024*1024)) 425 | } else { //if fileSize < (1024 * 1024 * 1024 * 1024 * 1024 * 1024) 426 | return fmt.Sprintf("%.2fEB", float64(fileSize)/float64(1024*1024*1024*1024*1024)) 427 | } 428 | } 429 | 430 | // AbPathByCaller 获取当前执行文件绝对路径（go run） 431 | func AbPathByCaller() string { 432 | var abPath string 433 | _, filename, _, ok := runtime.Caller(0) 434 | if ok { 435 | abPath = path.Dir(filename) 436 | } 437 | return path.Join(abPath, "../../../") 438 | } 439 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= 2 | github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= 3 | github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 8 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 9 | github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= 10 | github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= 11 | github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= 12 | github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= 13 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 14 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 15 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 16 | github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= 17 | github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 18 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 19 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 20 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 21 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= 22 | github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= 23 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 24 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 25 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 26 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 27 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 28 | github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= 29 | github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 30 | github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= 31 | github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= 32 | github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= 33 | github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= 34 | github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= 35 | github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= 36 | github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= 37 | github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= 38 | github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= 39 | github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= 40 | github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= 41 | github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= 42 | github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= 43 | github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= 44 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 45 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 46 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 47 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 48 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= 49 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= 50 | github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= 51 | github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= 52 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 53 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 54 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 55 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 56 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= 57 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 58 | github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= 59 | github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= 60 | github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= 61 | github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= 62 | github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= 63 | github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= 64 | github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= 65 | github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= 66 | github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 67 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 68 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 69 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 70 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 71 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 72 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 73 | github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= 74 | github.com/xuri/efp v0.0.0-20220603152613-6918739fd470/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= 75 | github.com/xuri/excelize/v2 v2.7.0 h1:Hri/czwyRCW6f6zrCDWXcXKshlq4xAZNpNOpdfnFhEw= 76 | github.com/xuri/excelize/v2 v2.7.0/go.mod h1:ebKlRoS+rGyLMyUx3ErBECXs/HNYqyj+PbkkKRK5vSI= 77 | github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 h1:OAmKAfT06//esDdpi/DZ8Qsdt4+M5+ltca05dA5bG2M= 78 | github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= 79 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 80 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 81 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 82 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 83 | golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= 84 | golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= 85 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 86 | golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 h1:Lj6HJGCSn5AjxRAH2+r35Mir4icalbqku+CLUtjnvXY= 87 | golang.org/x/image v0.0.0-20220902085622-e7cb96979f69/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= 88 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 89 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 90 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 91 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 92 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 93 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 94 | golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= 95 | golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= 96 | golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= 97 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 98 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 99 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 100 | golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= 101 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 102 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 103 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 104 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 105 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 106 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 107 | golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 108 | golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= 109 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 110 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 111 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 112 | golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= 113 | golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= 114 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 115 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 116 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 117 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 118 | golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 119 | golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= 120 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 121 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 122 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 123 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 124 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 125 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= 126 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= 127 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 128 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 129 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= 130 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= 131 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 132 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 133 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 134 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 135 | -------------------------------------------------------------------------------- /data_structure.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 包含了常见数据结构的实用方法，有切片，map,set等等 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "errors" 12 | "fmt" 13 | "math/rand" 14 | "reflect" 15 | "sort" 16 | "sync" 17 | "time" 18 | ) 19 | 20 | var randObj = rand.New(rand.NewSource(time.Now().UnixNano())) 21 | var TableIsNULL = fmt.Errorf("table is null") 22 | 23 | func SliceContains[V comparable](a []V, v V) bool { 24 | l := len(a) 25 | if l == 0 { 26 | return false 27 | } 28 | for i := 0; i < l; i++ { 29 | if a[i] == v { 30 | return true 31 | } 32 | } 33 | return false 34 | } 35 | 36 | func SliceDeduplicate[V comparable](a []V) []V { 37 | l := len(a) 38 | if l < 2 { 39 | return a 40 | } 41 | seen := make(map[V]struct{}) 42 | j := 0 43 | for i := 0; i < l; i++ { 44 | if _, ok := seen[a[i]]; ok { 45 | continue 46 | } 47 | seen[a[i]] = struct{}{} 48 | a[j] = a[i] 49 | j++ 50 | } 51 | return a[:j] 52 | } 53 | 54 | func SliceDel[V comparable](a []V, i int) []V { 55 | l := len(a) 56 | if l == 0 { 57 | return a 58 | } 59 | if i < 0 || i > l-1 { 60 | return a 61 | } 62 | return append(a[:i], a[i+1:]...) 63 | } 64 | 65 | type number interface { 66 | int | int8 | int16 | int32 | int64 | 67 | uint | uint8 | uint16 | uint32 | uint64 | 68 | float32 | float64 69 | } 70 | 71 | func SliceMax[V number](a []V) V { 72 | l := len(a) 73 | if l == 0 { 74 | var none V 75 | return none 76 | } 77 | maxV := a[0] 78 | for k := 1; k < l; k++ { 79 | if a[k] > maxV { 80 | maxV = a[k] 81 | } 82 | } 83 | return maxV 84 | } 85 | 86 | func SliceMin[V number](a []V) V { 87 | l := len(a) 88 | if l == 0 { 89 | return 0 90 | } 91 | minV := a[0] 92 | for k := 1; k < l; k++ { 93 | if a[k] < minV { 94 | minV = a[k] 95 | } 96 | } 97 | return minV 98 | } 99 | 100 | func SlicePop[V comparable](a []V) (V, []V) { 101 | if len(a) == 0 { 102 | var none V 103 | return none, a 104 | } 105 | return a[len(a)-1], a[:len(a)-1] 106 | } 107 | 108 | func SliceReverse[V comparable](a []V) []V { 109 | l := len(a) 110 | if l == 0 { 111 | return a 112 | } 113 | for s, e := 0, len(a)-1; s < e; { 114 | a[s], a[e] = a[e], a[s] 115 | s++ 116 | e-- 117 | } 118 | return a 119 | } 120 | 121 | func SliceShuffle[V comparable](a []V) []V { 122 | l := len(a) 123 | if l <= 1 { 124 | return a 125 | } 126 | randObj.Shuffle(l, func(i, j int) { 127 | a[i], a[j] = a[j], a[i] 128 | }) 129 | return a 130 | } 131 | 132 | func SliceCopy[V comparable](a []V) []V { 133 | return append(a[:0:0], a...) 134 | } 135 | 136 | // OrderMap 固定顺序map 137 | type OrderMap[K, V comparable] struct { 138 | mux sync.Mutex // TODO 优化 使用读写锁 139 | data map[K]V 140 | keyList []K // TODO 优化 使用链表 141 | size int 142 | } 143 | 144 | // NewOrderMap ues: NewOrderMap[K, V]() 145 | func NewOrderMap[K, V comparable]() *OrderMap[K, V] { 146 | obj := &OrderMap[K, V]{ 147 | mux: sync.Mutex{}, 148 | data: make(map[K]V), 149 | keyList: make([]K, 0), 150 | size: 0, 151 | } 152 | return obj 153 | } 154 | 155 | func (m *OrderMap[K, V]) Add(key K, value V) *OrderMap[K, V] { 156 | m.mux.Lock() 157 | defer m.mux.Unlock() 158 | if _, ok := m.data[key]; ok { 159 | m.data[key] = value 160 | return m 161 | } 162 | m.keyList = append(m.keyList, key) 163 | m.size++ 164 | m.data[key] = value 165 | return m 166 | } 167 | 168 | func (m *OrderMap[K, V]) Get(key K) V { 169 | m.mux.Lock() 170 | defer m.mux.Unlock() 171 | return m.data[key] 172 | } 173 | 174 | func (m *OrderMap[K, V]) Del(key K) *OrderMap[K, V] { 175 | m.mux.Lock() 176 | defer m.mux.Unlock() 177 | if _, ok := m.data[key]; ok { 178 | delete(m.data, key) 179 | for i := 0; i < m.size; i++ { 180 | if m.keyList[i] == key { 181 | m.keyList = append(m.keyList[:i], m.keyList[i+1:]...) 182 | m.size-- 183 | return m 184 | } 185 | } 186 | } 187 | return m 188 | } 189 | 190 | func (m *OrderMap[K, V]) Len() int { 191 | return m.size 192 | } 193 | 194 | func (m *OrderMap[K, V]) KeyList() []K { 195 | return m.keyList 196 | } 197 | 198 | func (m *OrderMap[K, V]) AddMap(data map[K]V) *OrderMap[K, V] { 199 | for k, v := range data { 200 | m.Add(k, v) 201 | } 202 | return m 203 | } 204 | 205 | func (m *OrderMap[K, V]) Range(f func(k K, v V)) *OrderMap[K, V] { 206 | for i := 0; i < m.size; i++ { 207 | f(m.keyList[i], m.data[m.keyList[i]]) 208 | } 209 | return m 210 | } 211 | 212 | // RangeAt Range 遍历map含顺序id 213 | func (m *OrderMap[K, V]) RangeAt(f func(id int, k K, v V)) *OrderMap[K, V] { 214 | for i := 0; i < m.size; i++ { 215 | f(i, m.keyList[i], m.data[m.keyList[i]]) 216 | } 217 | return m 218 | } 219 | 220 | // CheckValue 查看map是否存在指定的值 221 | func (m *OrderMap[K, V]) CheckValue(value V) bool { 222 | m.mux.Lock() 223 | defer m.mux.Unlock() 224 | for i := 0; i < m.size; i++ { 225 | if m.data[m.keyList[i]] == value { 226 | return true 227 | } 228 | } 229 | return false 230 | } 231 | 232 | // Reverse map反序 233 | func (m *OrderMap[K, V]) Reverse() *OrderMap[K, V] { 234 | for i, j := 0, len(m.keyList)-1; i < j; i, j = i+1, j-1 { 235 | m.keyList[i], m.keyList[j] = m.keyList[j], m.keyList[i] 236 | } 237 | return m 238 | } 239 | 240 | func (m *OrderMap[K, V]) Json() (string, error) { 241 | return Any2Json(m.data) 242 | } 243 | 244 | func (m *OrderMap[K, V]) DebugPrint() { 245 | m.RangeAt(func(id int, k K, v V) { 246 | DebugF("item:%d key:%v value:%v", id, k, v) 247 | }) 248 | } 249 | 250 | // Insert 插入值指定位置 251 | func (m *OrderMap[K, V]) Insert(k K, v V, position int) error { 252 | m.Add(k, v) 253 | return m.Move(k, position) 254 | } 255 | 256 | // Move 值移动指定位置操作 257 | func (m *OrderMap[K, V]) Move(k K, position int) error { 258 | if position >= m.size { 259 | return errors.New("position >= map len") 260 | } 261 | has := false 262 | for i := 0; i < m.size; i++ { 263 | if m.keyList[i] == k { 264 | m.keyList = append(m.keyList[0:i], m.keyList[i+1:]...) 265 | has = true 266 | break 267 | } 268 | } 269 | if has { 270 | m.keyList = append(m.keyList[:position+1], m.keyList[position:]...) 271 | m.keyList[position] = k 272 | } 273 | return nil 274 | } 275 | 276 | func (m *OrderMap[K, V]) GetAtPosition(position int) (K, V, error) { 277 | if position >= m.size { 278 | var ( 279 | k K 280 | v V 281 | ) 282 | return k, v, errors.New("position >= map len") 283 | } 284 | k := m.keyList[position] 285 | v := m.data[k] 286 | return k, v, nil 287 | } 288 | 289 | // Pop 首位读取并移除 290 | func (m *OrderMap[K, V]) Pop() (K, V, error) { 291 | if m.size < 1 { 292 | var ( 293 | k K 294 | v V 295 | ) 296 | return k, v, errors.New("map size is 0") 297 | } 298 | k := m.keyList[0] 299 | v := m.data[k] 300 | m.Del(k) 301 | return k, v, nil 302 | } 303 | 304 | // BackPop 末尾读取并移除 305 | func (m *OrderMap[K, V]) BackPop() (K, V, error) { 306 | if m.size < 1 { 307 | var ( 308 | k K 309 | v V 310 | ) 311 | return k, v, errors.New("map size is 0") 312 | } 313 | k := m.keyList[m.size-1] 314 | v := m.data[k] 315 | m.Del(k) 316 | return k, v, nil 317 | } 318 | 319 | // Shuffle 洗牌 320 | func (m *OrderMap[K, V]) Shuffle() { 321 | if m.size <= 1 { 322 | return 323 | } 324 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 325 | r.Shuffle(m.size, func(i, j int) { 326 | m.keyList[i], m.keyList[j] = m.keyList[j], m.keyList[i] 327 | }) 328 | } 329 | 330 | // SortDesc 根据k排序 desc 331 | func (m *OrderMap[K, V]) SortDesc() { 332 | sort.Slice(m.keyList, func(i, j int) bool { 333 | if i > j { 334 | return true 335 | } 336 | return false 337 | }) 338 | } 339 | 340 | // SortAsc 根据k排序 asc 341 | func (m *OrderMap[K, V]) SortAsc() { 342 | sort.Slice(m.keyList, func(i, j int) bool { 343 | if i < j { 344 | return true 345 | } 346 | return false 347 | }) 348 | } 349 | 350 | func (m *OrderMap[K, V]) CopyMap() map[K]V { 351 | temp := make(map[K]V, m.size) 352 | m.Range(func(k K, v V) { 353 | temp[k] = v 354 | }) 355 | return temp 356 | } 357 | 358 | // MysqlNewTable 给mysql提供创建新的固定map顺序为字段的表 359 | func (m *OrderMap[K, V]) MysqlNewTable(db Mysql, table string) error { 360 | var ( 361 | createSql bytes.Buffer 362 | line = m.Len() 363 | ) 364 | if table == "" { 365 | return TableIsNULL 366 | } 367 | if line < 1 { 368 | return fmt.Errorf("fiedls len is 0") 369 | } 370 | if db.DB == nil { 371 | _ = db.Conn() 372 | } 373 | createSql.WriteString("CREATE TABLE ") 374 | createSql.WriteString(table) 375 | createSql.WriteString(" ( temp_id int(11) NOT NULL AUTO_INCREMENT, ") 376 | m.Range(func(k K, v V) { 377 | createSql.WriteString(Any2String(k)) 378 | createSql.WriteString(" ") 379 | createSql.WriteString(dataType2Mysql(v)) 380 | createSql.WriteString(", ") 381 | }) 382 | createSql.WriteString("PRIMARY KEY (temp_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;") 383 | _, err := db.DB.Exec(createSql.String()) 384 | if db.log { 385 | Info("[Sql] Exec : " + createSql.String()) 386 | if err != nil { 387 | Error("[Sql] Error : " + err.Error()) 388 | } 389 | } 390 | if db.allTN == nil { 391 | _ = db.allTableName() 392 | } 393 | db.allTN.add(table) 394 | return nil 395 | } 396 | 397 | // MysqlInsert 给mysql提供固定顺序map写入 398 | func (m *OrderMap[K, V]) MysqlInsert(db Mysql, table string) error { 399 | var ( 400 | line = m.Len() 401 | fieldDataMap = make(map[string]any) 402 | ) 403 | if table == "" { 404 | return TableIsNULL 405 | } 406 | if line < 1 { 407 | return fmt.Errorf("fiedls len is 0") 408 | } 409 | if db.DB == nil { 410 | _ = db.Conn() 411 | } 412 | if db.allTN == nil { 413 | _ = db.allTableName() 414 | } 415 | if !db.allTN.isHave(table) { 416 | err := m.MysqlNewTable(db, table) 417 | if err != nil { 418 | return err 419 | } 420 | } 421 | m.Range(func(k K, v V) { 422 | fieldDataMap[Any2String(k)] = v 423 | }) 424 | return db.insert(table, fieldDataMap) 425 | } 426 | 427 | type Set map[any]struct{} 428 | 429 | func NewSet() Set { 430 | return make(Set) 431 | } 432 | 433 | func (s Set) Has(key any) bool { 434 | _, ok := s[key] 435 | return ok 436 | } 437 | 438 | func (s Set) Add(key any) { 439 | s[key] = struct{}{} 440 | } 441 | 442 | func (s Set) Delete(key any) { 443 | delete(s, key) 444 | } 445 | 446 | func (s Set) DebugPrint() { 447 | for k := range s { 448 | Debug(k) 449 | } 450 | } 451 | 452 | type Stack[K comparable] struct { 453 | data map[int]K 454 | } 455 | 456 | func NewStack[K comparable]() *Stack[K] { 457 | return &Stack[K]{ 458 | data: make(map[int]K), 459 | } 460 | } 461 | 462 | func (s *Stack[K]) Push(data K) { 463 | s.data[len(s.data)] = data 464 | } 465 | 466 | func (s *Stack[K]) Pop() K { 467 | if len(s.data) < 1 { 468 | var k K 469 | return k 470 | } 471 | l := len(s.data) - 1 472 | k := s.data[l] 473 | delete(s.data, l) 474 | return k 475 | } 476 | 477 | func (s *Stack[K]) DebugPrint() { 478 | for i := 0; i < len(s.data); i++ { 479 | Debug(s.data) 480 | } 481 | } 482 | 483 | func MapCopy[K, V comparable](data map[K]V) (copy map[K]V) { 484 | copy = make(map[K]V, len(data)) 485 | for k, v := range data { 486 | copy[k] = v 487 | } 488 | return 489 | } 490 | 491 | func MapMergeCopy[K, V comparable](src ...map[K]V) (copy map[K]V) { 492 | copy = make(map[K]V) 493 | for _, m := range src { 494 | for k, v := range m { 495 | copy[k] = v 496 | } 497 | } 498 | return 499 | } 500 | 501 | // Map2Slice Eg: {"K1": "v1", "K2": "v2"} => ["K1", "v1", "K2", "v2"] 502 | func Map2Slice(data any) []any { 503 | var ( 504 | reflectValue = reflect.ValueOf(data) 505 | reflectKind = reflectValue.Kind() 506 | ) 507 | for reflectKind == reflect.Ptr { 508 | reflectValue = reflectValue.Elem() 509 | reflectKind = reflectValue.Kind() 510 | } 511 | switch reflectKind { 512 | case reflect.Map: 513 | array := make([]any, 0) 514 | for _, key := range reflectValue.MapKeys() { 515 | array = append(array, key.Interface()) 516 | array = append(array, reflectValue.MapIndex(key).Interface()) 517 | } 518 | return array 519 | default: 520 | return nil 521 | } 522 | } 523 | 524 | // Slice2Map ["K1", "v1", "K2", "v2"] => {"K1": "v1", "K2": "v2"} 525 | // ["K1", "v1", "K2"] => nil 526 | func Slice2Map(slice any) map[any]any { 527 | var ( 528 | reflectValue = reflect.ValueOf(slice) 529 | reflectKind = reflectValue.Kind() 530 | ) 531 | for reflectKind == reflect.Ptr { 532 | reflectValue = reflectValue.Elem() 533 | reflectKind = reflectValue.Kind() 534 | } 535 | switch reflectKind { 536 | case reflect.Slice, reflect.Array: 537 | length := reflectValue.Len() 538 | if length%2 != 0 { 539 | return nil 540 | } 541 | data := make(map[any]any) 542 | for i := 0; i < reflectValue.Len(); i += 2 { 543 | data[reflectValue.Index(i).Interface()] = reflectValue.Index(i + 1).Interface() 544 | } 545 | return data 546 | default: 547 | return nil 548 | } 549 | } 550 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## 简介 2 | - gathertool是golang脚本化开发集成库，目的是提高爬虫，数据处理，接口测试，模拟请求等对应场景脚本程序开发的效率； 3 | - gathertool也是一款轻量级爬虫库，特色是抽象了请求事件，通俗点理解就是对请求过程状态进行事件处理。 4 | - gathertool也是接口测试&压力测试库，在接口测试脚本开发上有明显的效率优势， 5 | - gathertool还集成了对三方中间件的操作，DB操作等。 6 | 7 | --- 8 | 9 | ## 使用 10 | - go get github.com/mangenotwork/gathertool 11 | 12 | --- 13 | 14 | ## 介绍 15 | gathertool是一个高度封装工具库，包含了http/s的请求，Mysql数据库方法，数据类型处理方法，数据提取方法，websocket相关方法， 16 | TCP|UDP相关方法，NoSql相关方法，开发常用方法等; 可以用于爬虫程序，接口&压力测试程序，常见网络协议调试程序，数据提取与存储程序等； 17 | 18 | - gathertool的请求特点: 会在请求阶段执行各个事件如请求失败后的重试事件,请求前后的事件，请求成功事件等等, 可以根据请求状态码自定义这些事件； 19 | 20 | - gathertool还拥有很好的可扩展性， 适配传入任意自定义http请求对象， 能适配各种代理对象等等； 21 | 22 | - gathertool还拥有抓取数据存储功能, 存储到mysql；配合 https://github.com/mangenotwork/dbHelper 可存储到更多的数据存储上; 还有很多创新的地方文档会根据具体方法进行介绍； 23 | 24 | - gathertool还封装了消息队列接口，支持Nsq,Kafka,rabbitmq,redis等消息队列中间件 25 | 26 | ## 文档： 27 | [开发文档](https://github.com/mangenotwork/gathertool/blob/main/_doc/develop.md) 28 | 29 | [pkg.go.dev](https://pkg.go.dev/github.com/mangenotwork/gathertool) 30 | 31 | 32 | ## 开始使用 33 | > go get github.com/mangenotwork/gathertool 34 | 35 | 36 | #### 简单的get请求 37 | ```go 38 | import gt "github.com/mangenotwork/gathertool" 39 | 40 | func main(){ 41 | ctx, err := gt.Get("https://www.baidu.com") 42 | if err != nil { 43 | log.Println(err) 44 | } 45 | log.Println(ctx.RespBodyString) 46 | } 47 | ``` 48 | 49 | #### 含请求事件请求 50 | 51 | - StartFunc 开始事件，请求开始前执行的方法类型 52 | 53 | - SucceedFunc 成功事件，请求成功后执行的方法类型 54 | 55 | - FailedFunc 失败事件，请求失败执行的方法类型 56 | 57 | - RetryFunc 重试事件，重试请求前执行的方法类型，是否重试来自上次请求的状态码来确定，见StatusCodeMap; 58 | 59 | - EndFunc 结束事件，请求流程结束后执行的方法类型 60 | 61 | ```go 62 | import gt "github.com/mangenotwork/gathertool" 63 | 64 | func main(){ 65 | 66 | gt.NewGet(`http://192.168.0.1`).SetStartFunc(func(ctx *gt.Context){ 67 | log.Println("请求前： 添加代理等等操作") 68 | ctx.Client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)} 69 | } 70 | ).SetSucceedFunc(func(ctx *gt.Context){ 71 | log.Println("请求成功： 处理数据或存储等等") 72 | log.Println(ctx.RespBodyString()) 73 | } 74 | ).SetFailedFunc(func(ctx *gt.Context){ 75 | log.Println("请求失败： 记录失败等等") 76 | log.Println(ctx.Err) 77 | } 78 | ).SetRetryFunc(func(ctx *gt.Context){ 79 | log.Println("请求重试： 更换代理或添加等待时间等等") 80 | ctx.Client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)} 81 | } 82 | ).SetEndFunc(func(ctx *gt.Context){ 83 | log.Println("请求结束： 记录结束，处理数据等等") 84 | log.Println(ctx) 85 | } 86 | ).Do() 87 | 88 | } 89 | ``` 90 | 91 | #### 事件方法复用 92 | ```go 93 | func main(){ 94 | gt.NewGet(`http://192.168.0.1`).SetSucceedFunc(succeed).SetFailedFunc(failed).SetRetryFunc(retry).Do() 95 | gt.NewGet(`http://www.baidu.com`).SetSucceedFunc(baiduSucceed).SetFailedFunc(failed).SetRetryFunc(retry).Do() 96 | } 97 | // 请求成功： 处理数据或存储等等 98 | func succeed(ctx *gt.Context){ 99 | log.Println(ctx.RespBodyString()) 100 | } 101 | // 请求失败： 记录失败等等 102 | func failed(ctx *gt.Context){ 103 | log.Println(ctx.Err) 104 | } 105 | // 请求重试： 更换代理或添加等待时间等等 106 | func retry(ctx *gt.Context){ 107 | ctx.Client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)} 108 | } 109 | // 百度请求成功后 110 | func baiduSucceed(ctx *gt.Context){ 111 | log.Println(ctx.RespBodyString()) 112 | } 113 | ``` 114 | 115 | #### post请求 116 | ```go 117 | // FormData 118 | postData := gt.FormData{ 119 | "aa":"aa", 120 | } 121 | 122 | // header 123 | header := gt.NewHeader(map[string]string{ 124 | "Accept":"*/*", 125 | "X-MicrosoftAjax":"Delta=true", 126 | "Accept-Encoding":"gzip, deflate", 127 | "XMLHttpRequest":"XMLHttpRequest", 128 | "Content-Type":"application/x-www-form-urlencoded; charset=UTF-8", 129 | }) 130 | 131 | // cookie 132 | cookie := gt.Cookie{ 133 | "aa":"a" 134 | } 135 | 136 | // 随机休眠 2~6秒 137 | sleep := gt.SetSleep(2,6) 138 | c := gt.NewPostForm(caseUrl, postData, header, cookie, sleep) 139 | c.Do() 140 | html := c.RespBodyString() 141 | log.Print(html) 142 | 143 | ``` 144 | 145 | #### 数据存储到mysql 146 | ```go 147 | var ( 148 | host = "192.168.0.100" 149 | port = 3306 150 | user = "root" 151 | password = "root123" 152 | dbName = "dbName" 153 | db,_ = gt.NewMysql(host, port, user, password, dbName) 154 | ) 155 | 156 | //.... 执行抓取 157 | data1 := "data1" 158 | data2 := "data2" 159 | 160 | inputdata := map[string]interface{} { 161 | "data1" : data1, 162 | "data2" : data2, 163 | } 164 | 165 | tableName := "data" 166 | db.Spider2022DB.InsertAt(tableName, inputdata) 167 | ``` 168 | 169 | #### HTML数据提取 170 | ```go 171 | func main(){ 172 | date := "2022-07-05" 173 | caseUrl := "***" 174 | ctx, _ := gt.Get(fmt.Sprintf(caseUrl, date)) 175 | datas, _ := gt.GetPointHTML(ctx.Html, "div", "id", "domestic") 176 | Data(datas, date, "内期表", "备注：内期表=国内期货主力合约表") 177 | datas, _ = gt.GetPointHTML(ctx.Html, "div", "id", "overseas") 178 | Data(datas, date, "外期表", "备注：外期表=国外期货主力合约表") 179 | } 180 | 181 | func Data(datas []string, date, typeName, note string) { 182 | for _, data := range datas { 183 | table, _ := gt.GetPointHTML(data, "table", "id", "fdata") 184 | if len(table) > 0 { 185 | trList := gt.RegHtmlTr(table[0]) 186 | jys := "" 187 | for _, tr := range trList { 188 | td := gt.RegHtmlTd(tr) 189 | log.Println("td = ", td, len(td)) 190 | if len(td) == 1 { 191 | jys = gt.RegHtmlTdTxt(td[0])[0] 192 | continue 193 | } 194 | name := gt.RegHtmlTdTxt(td[0])[0] 195 | if strings.Index(name, "商品名称") != -1 { 196 | continue 197 | } 198 | zlhy := gt.RegHtmlTdTxt(td[1])[0] 199 | jsj := gt.RegHtmlTdTxt(td[2])[0] 200 | zd := gt.RegDelHtml(gt.RegHtmlTdTxt(td[3])[0]) 201 | cjj := gt.RegHtmlTdTxt(td[4])[0] 202 | ccl := gt.RegHtmlTdTxt(td[5])[0] 203 | dw := gt.RegHtmlTdTxt(td[6])[0] 204 | log.Println("日期 = ", date) 205 | log.Println("机构 = ", jys) 206 | log.Println("商品名称 = ", name) 207 | log.Println("主力合约 = ", zlhy) 208 | log.Println("结算价 = ", jsj) 209 | log.Println("涨跌 = ", zd) 210 | log.Println("成交量 = ", cjj) 211 | log.Println("持仓量 = ", ccl) 212 | log.Println("单位 = ", dw) 213 | } 214 | } 215 | } 216 | } 217 | ``` 218 | 219 | #### Json数据提取 220 | ```go 221 | func main(){ 222 | txt := `{ 223 | "reason":"查询成功!", 224 | "result":{ 225 | "city":"苏州", 226 | "realtime":{ 227 | "temperature":"17", 228 | "humidity":"69", 229 | "info":"阴", 230 | "wid":"02", 231 | "direct":"东风", 232 | "power":"2级", 233 | "aqi":"30" 234 | }, 235 | "future":[ 236 | { 237 | "date":"2021-10-25", 238 | "temperature":"12\/21℃", 239 | "weather":"多云", 240 | "wid":{ 241 | "day":"01", 242 | "night":"01" 243 | }, 244 | "direct":"东风" 245 | }, 246 | { 247 | "date":"2021-10-26", 248 | "temperature":"13\/21℃", 249 | "weather":"多云", 250 | "wid":{ 251 | "day":"01", 252 | "night":"01" 253 | }, 254 | "direct":"东风转东北风" 255 | }, 256 | { 257 | "date":"2021-10-27", 258 | "temperature":"13\/22℃", 259 | "weather":"多云", 260 | "wid":{ 261 | "day":"01", 262 | "night":"01" 263 | }, 264 | "direct":"东北风" 265 | } 266 | ] 267 | }, 268 | "error_code":0 269 | }` 270 | 271 | jx1 := "/result/future/[0]/date" 272 | jx2 := "/result/future/[0]" 273 | jx3 := "/result/future" 274 | 275 | log.Println(gt.JsonFind(txt, jx1)) 276 | log.Println(gt.JsonFind2Json(txt, jx2)) 277 | log.Println(gt.JsonFind2Json(txt, jx3)) 278 | log.Println(gt.JsonFind2Map(txt, jx2)) 279 | log.Println(gt.JsonFind2Arr(txt, jx3)) 280 | 281 | } 282 | ``` 283 | 284 | ...... 285 | 286 | ## 实例 287 | - [Get请求](https://github.com/mangenotwork/gathertool/tree/main/_examples/get) 288 | - [阳光高考招生章程抓取](https://github.com/mangenotwork/gathertool/tree/main/_examples/get_yggk) 289 | - [ip地址信息抓取](https://github.com/mangenotwork/gathertool/tree/main/_examples/ip_bczs_cn) 290 | - [压力测试](https://github.com/mangenotwork/gathertool/tree/main/_examples/stress_testing) 291 | - [文件下载](https://github.com/mangenotwork/gathertool/tree/main/_examples/upload_file) 292 | - [无登录微博抓取](https://github.com/mangenotwork/gathertool/tree/main/_examples/weibo) 293 | - [百度题库抓取](https://github.com/mangenotwork/gathertool/tree/main/_examples/baidu_tk) 294 | - [搭建http/s代理与抓包](https://github.com/mangenotwork/gathertool/tree/main/_examples/intercept) 295 | - [搭建socket5代理](https://github.com/mangenotwork/gathertool/tree/main/_examples/socket5) 296 | - [商品报价信息抓取](https://github.com/mangenotwork/gathertool/tree/main/_examples/baojia) 297 | - [期货信息抓取](https://github.com/mangenotwork/gathertool/tree/main/_examples/qihuo) 298 | - [行业信息网行业分类](https://github.com/mangenotwork/gathertool/tree/main/_examples/cnlinfo) 299 | 300 | ## JetBrains 开源证书支持 301 | 302 | `gathertool` 项目一直以来都是在 JetBrains 公司旗下的 `GoLand` 集成开发环境中进行开发，基于 **free JetBrains Open Source license(s)** 正版免费授权，在此表达我的谢意。 303 | 304 | 305 | 306 | 307 | ## 使用注意&不足之处 308 | 1. Mysql相关方法封装使用的字符串拼接，如抓取无需关心Sql注入这类场景放心使用，生产环境使用会出现安全风险； 309 | 2. 变量类型转换使用反射，性能方面会差一些，追求性能请使用其他； 310 | 311 | 312 | ## 经验积累 313 | 从2022年v0.3.4版本到至今已经应用在我的实际工作中，给边边角角的工作带来了巨大的脚步编写效率，经过不断的积累和总结质量也在不断的提示；感兴趣的伙伴可以fork代码， 314 | 将你的宝贵经验提交进来。 315 | 316 | ## 配合使用 317 | - 数据存储与读取 https://github.com/mangenotwork/dbHelper 318 | - 图片处理 https://github.com/mangenotwork/imgHelper 319 | 320 | ## TODO 321 | - Redis连接方法改为连接池, 干掉redis队列 322 | - 关闭重试 323 | - 提供通过html定位加正则获取内容的方法 324 | - 引入请求工作与队列的功能，在广度爬取场景使用 325 | 326 | ## BUG 327 | - 固定map调用不了 .\main.go:124:16: undefined: gt.OrderMap 328 | - panic 329 | ```azure 330 | panic: runtime error: index out of range [1] with length 1 331 | 332 | goroutine 1 [running]: 333 | github.com/mangenotwork/gathertool.RegFindAllTxt({0xb042f8?, 0x1?}, {0xc00020c000, 0x3fe5}) 334 | D:/go/pkg/mod/github.com/mangenotwork/gathertool@v0.4.7/reg.go:32 +0x135 335 | ``` 336 | 337 | 338 | ## 三方引用 感谢这些开源项目 339 | - github.com/Shopify/sarama 340 | - github.com/dgrijalva/jwt-go 341 | - github.com/go-sql-driver/mysql 342 | - github.com/nsqio/go-nsq 343 | - github.com/streadway/amqp 344 | - github.com/xuri/excelize/v2 345 | - golang.org/x/crypto 346 | - golang.org/x/net 347 | - golang.org/x/text 348 | - gopkg.in/gomail.v2 349 | - gopkg.in/yaml.v3 350 | 351 | ## 里程碑 352 | 353 | #### v0.4.8 354 | ``` 355 | 1. 升级到go版本1.19并进行优化 356 | 2. 多类型重复的方法改为泛型，减轻代码量 357 | 3. 增加测试代码和整理实例 358 | 4. 优化项目文件结构，规范库的注释 359 | 5. 更新和优化文档 360 | 6. 修复 panic 361 | 7. 修复问题: 362 | - Any2Float64 不支持字符串转float64 363 | - 下载的函数名错误 Upload 执行GET下载请求 364 | - 缺少url提取文件名，文件后缀的函数 365 | - 类型转换缺少空值判断 366 | 8. 新增: 367 | - 正则提取增加 section, dl, dt, dd 标签 368 | - 去掉 gorm.io/gorm, mongo-driver, github.com/gomodule/redigo 369 | - html解析需要引入属性值包含关系 370 | - 下载资源需要打印下载url, 以及检查下载大小，如果是0则抛出错误 371 | ``` 372 | 373 | #### v0.4.7 374 | ``` 375 | 1. Redis连接方法改为连接池 376 | 2. 增加关闭重试方法 377 | 3. 增加Whois查询 378 | 4. 测试与优化 379 | ``` 380 | 381 | #### v0.4.4 ~ v0.4.6 382 | ``` 383 | 1. 移除 grpc相关方法 384 | 2. 新增DNS查询 385 | 3. 新增证书信息获取 386 | 4. 新增Url扫描 387 | 5. 新增邮件发送 388 | 6. 优化ICMP Ping的方法 389 | ``` 390 | 391 | #### v0.4.3 392 | ``` 393 | 1. 新增网站链接采集应用场景的方法 394 | 2. 修复v0.4.2的json导入bug 395 | 3. 修复redis封装方法的bug 396 | 4. 请求日志添加请求地址信息 397 | 5. 优化抓取实例 398 | ``` 399 | 400 | #### v0.4.2 401 | ``` 402 | 新增: 403 | 1. redis, nsq, rabbitmq, kafka 消息队列方法 404 | 2. 新增开发文档 405 | 3. 新增redis相关方法 406 | ``` 407 | 408 | #### v0.4.1 409 | ``` 410 | 新增: 411 | 1. 文件相关处理 412 | 2. 文件压缩解压 413 | 3. 新增抓取实列 _examples/cnlinfo 414 | ``` 415 | 416 | #### v0.2.1 ~ v0.3.9 417 | ``` 418 | 旧的迭代版本，不稳定，请使用 > v0.3.4的版本 419 | - 新增代理与抓包 420 | - 启动一个代理服务，并抓包对数据包进行处理 421 | - 新增socket5代理 422 | 423 | v0.3.6 424 | 新增: 425 | 1. func SimpleServer(addr string, g *grpc.Server) : grpc服务端 426 | 2. func NewTask() *Task : task的实例方法 427 | 3. func (task *Task) SetUrl(urlStr string) *Task & func (task *Task) SetJsonParam(jsonStr string) *Task 428 | 429 | 修复: 430 | 1. Context对象，Task为空的问题 431 | 432 | v0.3.7 433 | 新增: 434 | 1. 新增html解析，指定html内容提取 435 | 2. 新增抓取实例 436 | 3. 优化部分方法 437 | 438 | v0.3.9 439 | 新增: 440 | 1. 新增配置，支持yaml 441 | 2. 优化部分方法 442 | ``` 443 | 444 | 445 | 446 | -------------------------------------------------------------------------------- /crypto_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 加密解码相关封装方法 TODO 测试 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "crypto/aes" 12 | "crypto/cipher" 13 | "crypto/des" 14 | "crypto/hmac" 15 | "crypto/md5" 16 | "crypto/rand" 17 | "crypto/sha1" 18 | "crypto/sha256" 19 | "crypto/sha512" 20 | "encoding/base64" 21 | "errors" 22 | "fmt" 23 | "hash" 24 | "io" 25 | 26 | "github.com/dgrijalva/jwt-go" 27 | "golang.org/x/crypto/pbkdf2" 28 | ) 29 | 30 | const ( 31 | CBC = "CBC" 32 | ECB = "ECB" 33 | CFB = "CFB" 34 | CTR = "CTR" 35 | ) 36 | 37 | // AES AES interface 38 | type AES interface { 39 | Encrypt(str, key []byte) ([]byte, error) 40 | Decrypt(str, key []byte) ([]byte, error) 41 | } 42 | 43 | // DES DES interface 44 | type DES interface { 45 | Encrypt(str, key []byte) ([]byte, error) 46 | Decrypt(str, key []byte) ([]byte, error) 47 | } 48 | 49 | // NewAES : use NewAES(AES_CBC) 50 | func NewAES(typeName string, arg ...[]byte) AES { 51 | iv := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6} 52 | if len(arg) != 0 { 53 | iv = arg[0] 54 | } 55 | switch typeName { 56 | case "cbc", "Cbc", CBC: 57 | return &cbcObj{ 58 | cryptoType: "aes", 59 | iv: iv, 60 | } 61 | case "ecb", "Ecb", ECB: 62 | return &ecbObj{ 63 | cryptoType: "aes", 64 | } 65 | case "cfb", "Cfb", CFB: 66 | return &cfbObj{ 67 | cryptoType: "aes", 68 | } 69 | case "ctr", "Ctr", CTR: 70 | return &ctrObj{ 71 | cryptoType: "aes", 72 | count: iv, 73 | } 74 | default: 75 | return &cbcObj{ 76 | iv: iv, 77 | } 78 | } 79 | } 80 | 81 | // NewDES : use NewDES(DES_CBC) 82 | func NewDES(typeName string, arg ...[]byte) DES { 83 | iv := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6} 84 | if len(arg) != 0 { 85 | iv = arg[0] 86 | } 87 | switch typeName { 88 | case "cbc", "Cbc", CBC: 89 | return &cbcObj{ 90 | cryptoType: "des", 91 | iv: iv, 92 | } 93 | case "ecb", "Ecb", ECB: 94 | return &ecbObj{ 95 | cryptoType: "des", 96 | } 97 | case "cfb", "Cfb", CFB: 98 | return &cfbObj{ 99 | cryptoType: "des", 100 | } 101 | case "ctr", "Ctr", CTR: 102 | return &ctrObj{ 103 | count: iv, 104 | cryptoType: "des", 105 | } 106 | default: 107 | return &cbcObj{ 108 | iv: iv, 109 | cryptoType: "des", 110 | } 111 | } 112 | } 113 | 114 | // CBC : 密码分组链接模式（Cipher Block Chaining (CBC)） default 115 | type cbcObj struct { 116 | cryptoType string 117 | iv []byte 118 | } 119 | 120 | func (cbc *cbcObj) getBlock(key []byte) (block cipher.Block, err error) { 121 | if cbc.cryptoType == "aes" { 122 | block, err = aes.NewCipher(key) 123 | } 124 | if cbc.cryptoType == "des" { 125 | block, err = des.NewCipher(key) 126 | } 127 | return 128 | } 129 | 130 | // Encrypt AES CBC Encrypt 131 | func (cbc *cbcObj) Encrypt(str, key []byte) ([]byte, error) { 132 | block, err := cbc.getBlock(key) 133 | if err != nil { 134 | Error("[" + cbc.cryptoType + "-CBC] ERROR:" + err.Error()) 135 | return []byte(""), err 136 | } 137 | blockSize := block.BlockSize() 138 | originData := cbc.pkcs5Padding(str, blockSize) 139 | blockMode := cipher.NewCBCEncrypter(block, cbc.iv) 140 | cryptData := make([]byte, len(originData)) 141 | blockMode.CryptBlocks(cryptData, originData) 142 | P2E() 143 | return cryptData, nil 144 | } 145 | 146 | // Decrypt AES CBC Decrypt 147 | func (cbc *cbcObj) Decrypt(str, key []byte) ([]byte, error) { 148 | block, err := cbc.getBlock(key) 149 | if err != nil { 150 | Error("[" + cbc.cryptoType + "-CBC] ERROR:" + err.Error()) 151 | return []byte(""), err 152 | } 153 | blockMode := cipher.NewCBCDecrypter(block, cbc.iv) 154 | originStr := make([]byte, len(str)) 155 | blockMode.CryptBlocks(originStr, str) 156 | P2E() 157 | return cbc.pkcs5UnPadding(originStr), nil 158 | } 159 | 160 | func (cbc *cbcObj) pkcs5Padding(ciphertext []byte, blockSize int) []byte { 161 | padding := blockSize - len(ciphertext)%blockSize 162 | padText := bytes.Repeat([]byte{byte(padding)}, padding) 163 | return append(ciphertext, padText...) 164 | } 165 | 166 | func (cbc *cbcObj) pkcs5UnPadding(origData []byte) []byte { 167 | length := len(origData) 168 | unPadDing := int(origData[length-1]) 169 | return origData[:(length - unPadDing)] 170 | } 171 | 172 | // ECB : 电码本模式（Electronic Codebook Book (ECB)） 173 | type ecbObj struct { 174 | cryptoType string 175 | } 176 | 177 | func (ecb *ecbObj) getBlock(key []byte) (block cipher.Block, err error) { 178 | if ecb.cryptoType == "aes" { 179 | block, err = aes.NewCipher(ecb.generateKey(key)) 180 | } 181 | if ecb.cryptoType == "des" { 182 | block, err = des.NewCipher(key) 183 | } 184 | return 185 | } 186 | 187 | // Encrypt AES ECB Encrypt 188 | func (ecb *ecbObj) Encrypt(str, key []byte) ([]byte, error) { 189 | block, err := ecb.getBlock(key) 190 | if err != nil { 191 | Error("[" + ecb.cryptoType + "-ECB] ERROR:" + err.Error()) 192 | return []byte(""), err 193 | } 194 | blockSize := block.BlockSize() 195 | if ecb.cryptoType == "aes" { 196 | str = ecb.pkcs5PaddingAes(str, blockSize) 197 | } 198 | if ecb.cryptoType == "des" { 199 | str = ecb.pkcs5PaddingDes(str, blockSize) 200 | } 201 | 202 | //返回加密结果 203 | encryptData := make([]byte, len(str)) 204 | //存储每次加密的数据 205 | tmpData := make([]byte, blockSize) 206 | //分组分块加密 207 | for index := 0; index < len(str); index += blockSize { 208 | block.Encrypt(tmpData, str[index:index+blockSize]) 209 | copy(encryptData, tmpData) 210 | } 211 | P2E() 212 | return encryptData, nil 213 | } 214 | 215 | func (ecb *ecbObj) pkcs5PaddingDes(ciphertext []byte, blockSize int) []byte { 216 | padding := blockSize - len(ciphertext)%blockSize 217 | padText := bytes.Repeat([]byte{byte(padding)}, padding) 218 | return append(ciphertext, padText...) 219 | } 220 | 221 | func (ecb *ecbObj) pkcs5PaddingAes(ciphertext []byte, blockSize int) []byte { 222 | padding := blockSize - len(ciphertext)%blockSize 223 | if padding != 0 { 224 | ciphertext = append(ciphertext, bytes.Repeat([]byte{byte(0)}, padding)...) 225 | } 226 | return ciphertext 227 | } 228 | 229 | // Decrypt AES ECB Decrypt 230 | func (ecb *ecbObj) Decrypt(str, key []byte) ([]byte, error) { 231 | block, err := ecb.getBlock(key) 232 | if err != nil { 233 | Error("[" + ecb.cryptoType + "-ECB] ERROR:" + err.Error()) 234 | return []byte(""), err 235 | } 236 | blockSize := block.BlockSize() 237 | //返回加密结果 238 | decryptData := make([]byte, len(str)) 239 | //存储每次加密的数据 240 | tmpData := make([]byte, blockSize) 241 | 242 | //分组分块加密 243 | for index := 0; index < len(str); index += blockSize { 244 | block.Decrypt(tmpData, str[index:index+blockSize]) 245 | copy(decryptData, tmpData) 246 | } 247 | 248 | if ecb.cryptoType == "des" { 249 | return ecb.pkcs5UnPadding(decryptData), nil 250 | } 251 | P2E() 252 | return ecb.unPadding(decryptData), nil 253 | } 254 | 255 | func (ecb *ecbObj) generateKey(key []byte) (genKey []byte) { 256 | genKey = make([]byte, 16) 257 | copy(genKey, key) 258 | for i := 16; i < len(key); { 259 | for j := 0; j < 16 && i < len(key); j, i = j+1, i+1 { 260 | genKey[j] ^= key[i] 261 | } 262 | } 263 | return genKey 264 | } 265 | 266 | func (ecb *ecbObj) unPadding(src []byte) []byte { 267 | for i := len(src) - 1; ; i-- { 268 | if src[i] != 0 { 269 | return src[:i+1] 270 | } 271 | } 272 | } 273 | 274 | func (ecb *ecbObj) pkcs5UnPadding(origData []byte) []byte { 275 | length := len(origData) 276 | unPadding := int(origData[length-1]) 277 | return origData[:(length - unPadding)] 278 | } 279 | 280 | // 密码反馈模式（Cipher FeedBack (CFB)） 281 | type cfbObj struct { 282 | cryptoType string 283 | } 284 | 285 | func (cfb *cfbObj) getBlock(key []byte) (block cipher.Block, err error) { 286 | if cfb.cryptoType == "aes" { 287 | block, err = aes.NewCipher(key) 288 | } 289 | if cfb.cryptoType == "des" { 290 | block, err = des.NewCipher(key) 291 | } 292 | return 293 | } 294 | 295 | // Encrypt AES CFB Encrypt 296 | func (cfb *cfbObj) Encrypt(str, key []byte) ([]byte, error) { 297 | P2E() 298 | block, err := cfb.getBlock(key) 299 | if err != nil { 300 | Error("[" + cfb.cryptoType + "-CFB] ERROR:" + err.Error()) 301 | return nil, err 302 | } 303 | 304 | if cfb.cryptoType == "aes" { 305 | encrypted := make([]byte, aes.BlockSize+len(str)) 306 | iv := encrypted[:aes.BlockSize] 307 | if _, err := io.ReadFull(rand.Reader, iv); err != nil { 308 | return nil, err 309 | } 310 | stream := cipher.NewCFBEncrypter(block, iv) 311 | stream.XORKeyStream(encrypted[aes.BlockSize:], str) 312 | return encrypted, nil 313 | } 314 | 315 | if cfb.cryptoType == "des" { 316 | encrypted := make([]byte, des.BlockSize+len(str)) 317 | iv := encrypted[:des.BlockSize] 318 | if _, err := io.ReadFull(rand.Reader, iv); err != nil { 319 | return nil, err 320 | } 321 | stream := cipher.NewCFBEncrypter(block, iv) 322 | stream.XORKeyStream(encrypted[des.BlockSize:], str) 323 | return encrypted, nil 324 | } 325 | return nil, nil 326 | } 327 | 328 | // Decrypt AES CFB Decrypt 329 | func (cfb *cfbObj) Decrypt(str, key []byte) ([]byte, error) { 330 | P2E() 331 | block, err := cfb.getBlock(key) 332 | if err != nil { 333 | Error("[" + cfb.cryptoType + "-CFB] ERROR:" + err.Error()) 334 | return nil, err 335 | } 336 | 337 | iv := make([]byte, 0) 338 | if cfb.cryptoType == "aes" { 339 | if len(str) < aes.BlockSize { 340 | return nil, errors.New("ciphertext too short") 341 | } 342 | iv = str[:aes.BlockSize] 343 | str = str[aes.BlockSize:] 344 | } 345 | 346 | if cfb.cryptoType == "des" { 347 | if len(str) < des.BlockSize { 348 | return nil, errors.New("ciphertext too short") 349 | } 350 | iv = str[:des.BlockSize] 351 | str = str[des.BlockSize:] 352 | } 353 | 354 | stream := cipher.NewCFBDecrypter(block, iv) 355 | stream.XORKeyStream(str, str) 356 | return str, nil 357 | } 358 | 359 | // 计算器模式（Counter (CTR)） 360 | type ctrObj struct { 361 | count []byte //指定计数器,长度必须等于block的块尺寸 362 | cryptoType string 363 | } 364 | 365 | func (ctr *ctrObj) getBlock(key []byte) (block cipher.Block, err error) { 366 | if ctr.cryptoType == "aes" { 367 | block, err = aes.NewCipher(key) 368 | } 369 | if ctr.cryptoType == "des" { 370 | block, err = des.NewCipher(key) 371 | if len(ctr.count) > des.BlockSize { 372 | ctr.count = ctr.count[0:des.BlockSize] 373 | } 374 | } 375 | return 376 | } 377 | 378 | // Encrypt AES CTR Encrypt 379 | func (ctr *ctrObj) Encrypt(str, key []byte) ([]byte, error) { 380 | return ctr.crypto(str, key) 381 | } 382 | 383 | // Decrypt AES CTR Decrypt 384 | func (ctr *ctrObj) Decrypt(str, key []byte) ([]byte, error) { 385 | return ctr.crypto(str, key) 386 | } 387 | 388 | func (ctr *ctrObj) crypto(str, key []byte) ([]byte, error) { 389 | P2E() 390 | block, err := ctr.getBlock(key) 391 | if err != nil { 392 | Error("[AES-CTR] ERROR:" + err.Error()) 393 | return []byte(""), err 394 | } 395 | //指定分组模式 396 | blockMode := cipher.NewCTR(block, ctr.count) 397 | //执行加密、解密操作 398 | res := make([]byte, len(str)) 399 | blockMode.XORKeyStream(res, str) 400 | //返回明文或密文 401 | return res, nil 402 | } 403 | 404 | // 输出反馈模式（Output FeedBack (OFB)） 405 | type ofbObj struct{} 406 | 407 | func hmacFunc(h func() hash.Hash, str, key []byte) string { 408 | mac := hmac.New(h, key) 409 | mac.Write(str) 410 | res := base64.StdEncoding.EncodeToString(mac.Sum(nil)) 411 | return res 412 | } 413 | 414 | // HmacMD5 hmac md5 415 | func HmacMD5(str, key string) string { 416 | return hmacFunc(md5.New, []byte(str), []byte(key)) 417 | } 418 | 419 | // HmacSHA1 hmac sha1 420 | func HmacSHA1(str, key string) string { 421 | return hmacFunc(sha1.New, []byte(str), []byte(key)) 422 | } 423 | 424 | // HmacSHA256 hmac sha256 425 | func HmacSHA256(str, key string) string { 426 | return hmacFunc(sha256.New, []byte(str), []byte(key)) 427 | } 428 | 429 | // HmacSHA512 hmac sha512 430 | func HmacSHA512(str, key string) string { 431 | return hmacFunc(sha512.New, []byte(str), []byte(key)) 432 | } 433 | 434 | func pbkdf2Func(h func() hash.Hash, str, salt []byte, iterations, keySize int) []byte { 435 | return pbkdf2.Key(str, salt, iterations, keySize, h) 436 | } 437 | 438 | func PBKDF2(str, salt []byte, iterations, keySize int) []byte { 439 | return pbkdf2Func(sha256.New, str, salt, iterations, keySize) 440 | } 441 | 442 | // jwtEncrypt 443 | func jwtEncrypt(token *jwt.Token, data map[string]any, secret string) (string, error) { 444 | claims := make(jwt.MapClaims) 445 | for k, v := range data { 446 | claims[k] = v 447 | } 448 | token.Claims = claims 449 | return token.SignedString([]byte(secret)) 450 | } 451 | 452 | // JwtEncrypt jwt Encrypt 453 | func JwtEncrypt(data map[string]any, secret, method string) (string, error) { 454 | switch method { 455 | case "256": 456 | return jwtEncrypt(jwt.New(jwt.SigningMethodHS256), data, secret) 457 | case "384": 458 | return jwtEncrypt(jwt.New(jwt.SigningMethodHS384), data, secret) 459 | case "512": 460 | return jwtEncrypt(jwt.New(jwt.SigningMethodHS512), data, secret) 461 | } 462 | return "", fmt.Errorf("未知method; method= 256 or 384 or 512 ") 463 | } 464 | 465 | func JwtEncrypt256(data map[string]any, secret string) (string, error) { 466 | token := jwt.New(jwt.SigningMethodHS256) 467 | return jwtEncrypt(token, data, secret) 468 | } 469 | 470 | func JwtEncrypt384(data map[string]any, secret string) (string, error) { 471 | token := jwt.New(jwt.SigningMethodHS384) 472 | return jwtEncrypt(token, data, secret) 473 | } 474 | 475 | func JwtEncrypt512(data map[string]any, secret string) (string, error) { 476 | token := jwt.New(jwt.SigningMethodHS512) 477 | return jwtEncrypt(token, data, secret) 478 | } 479 | 480 | // JwtDecrypt jwt decrypt 481 | func JwtDecrypt(tokenString, secret string) (data map[string]any, err error) { 482 | data = make(map[string]any) 483 | var secretFunc = func() jwt.Keyfunc { //按照这样的规则解析 484 | return func(t *jwt.Token) (any, error) { 485 | return []byte(secret), nil 486 | } 487 | } 488 | token, err := jwt.Parse(tokenString, secretFunc()) 489 | if err != nil { 490 | err = fmt.Errorf("未知Token") 491 | return 492 | } 493 | claim, ok := token.Claims.(jwt.MapClaims) 494 | if !ok { 495 | return 496 | } 497 | if !token.Valid { 498 | // 令牌错误 499 | return 500 | } 501 | for k, v := range claim { 502 | data[k] = v 503 | } 504 | return 505 | } 506 | -------------------------------------------------------------------------------- /string_helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 字符串，字符编码等等相关的操作方法 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "compress/gzip" 12 | "crypto/md5" 13 | "encoding/base64" 14 | "encoding/binary" 15 | "encoding/gob" 16 | "encoding/hex" 17 | "fmt" 18 | "golang.org/x/text/encoding" 19 | "golang.org/x/text/encoding/ianaindex" 20 | "golang.org/x/text/encoding/simplifiedchinese" 21 | "golang.org/x/text/transform" 22 | "io" 23 | "log" 24 | "strconv" 25 | "strings" 26 | "unicode/utf8" 27 | ) 28 | 29 | // CleaningStr 清理字符串前后空白，回车，换行符号 30 | func CleaningStr(str string) string { 31 | str = strings.Replace(str, "\n", "", -1) 32 | str = strings.Replace(str, "\r", "", -1) 33 | str = strings.Replace(str, "\\n", "", -1) 34 | str = strings.TrimSpace(str) 35 | str = StrDeleteSpace(str) 36 | return str 37 | } 38 | 39 | // StrDeleteSpace 删除字符串前后的空格 40 | func StrDeleteSpace(str string) string { 41 | strList := []byte(str) 42 | spaceCount, count := 0, len(strList) 43 | for i := 0; i <= len(strList)-1; i++ { 44 | if strList[i] == 32 { 45 | spaceCount++ 46 | } else { 47 | break 48 | } 49 | } 50 | 51 | strList = strList[spaceCount:] 52 | spaceCount, count = 0, len(strList) 53 | for i := count - 1; i >= 0; i-- { 54 | if strList[i] == 32 { 55 | spaceCount++ 56 | } else { 57 | break 58 | } 59 | } 60 | 61 | return string(strList[:count-spaceCount]) 62 | } 63 | 64 | // EncodeByte 任意类型编码为[]byte 65 | func EncodeByte(v any) []byte { 66 | switch value := v.(type) { 67 | case int, int8, int16, int32: 68 | return Int2Byte(value.(int)) 69 | case int64: 70 | return Int642Byte(value) 71 | case string: 72 | return Str2Byte(value) 73 | case bool: 74 | return Bool2Byte(value) 75 | case float32: 76 | return Float322Byte(value) 77 | case float64: 78 | return Float642Byte(value) 79 | } 80 | return []byte("") 81 | } 82 | 83 | // DecodeByte decode byte 84 | func DecodeByte(b []byte) ([]byte, error) { 85 | rse := make([]byte, 0) 86 | buf := bytes.NewBuffer(b) 87 | err := binary.Read(buf, binary.BigEndian, rse) 88 | return rse, err 89 | } 90 | 91 | func deepCopy[T any](dst, src T) error { 92 | var buf bytes.Buffer 93 | if err := gob.NewEncoder(&buf).Encode(src); err != nil { 94 | return err 95 | } 96 | return gob.NewDecoder(bytes.NewBuffer(buf.Bytes())).Decode(dst) 97 | } 98 | 99 | // DeepCopy 深copy 100 | func DeepCopy[T any](dst, src T) error { 101 | return deepCopy(dst, src) 102 | } 103 | 104 | // PanicToError panic -> error 105 | func PanicToError(fn func()) (err error) { 106 | defer func() { 107 | if r := recover(); r != nil { 108 | err = fmt.Errorf("panic error: %v", r) 109 | } 110 | }() 111 | fn() 112 | return 113 | } 114 | 115 | // P2E panic -> error 116 | func P2E() { 117 | defer func() { 118 | if r := recover(); r != nil { 119 | Error("Panic error: %v", r) 120 | } 121 | }() 122 | } 123 | 124 | // Charset 字符集类型 125 | type Charset string 126 | 127 | const ( 128 | UTF8 = Charset("UTF-8") 129 | GB18030 = Charset("GB18030") 130 | GBK = Charset("GBK") 131 | GB2312 = Charset("GB2312") 132 | ) 133 | 134 | // ConvertByte2String 编码转换 135 | func ConvertByte2String(byte []byte, charset Charset) string { 136 | var str string 137 | switch charset { 138 | case GB18030: 139 | var decodeBytes, _ = simplifiedchinese.GB18030.NewDecoder().Bytes(byte) 140 | str = string(decodeBytes) 141 | 142 | case GBK: 143 | var decodeBytes, _ = simplifiedchinese.GBK.NewDecoder().Bytes(byte) 144 | str = string(decodeBytes) 145 | 146 | case GB2312: 147 | var decodeBytes, _ = simplifiedchinese.HZGB2312.NewDecoder().Bytes(byte) 148 | str = string(decodeBytes) 149 | 150 | case UTF8: 151 | fallthrough 152 | 153 | default: 154 | str = string(byte) 155 | } 156 | 157 | return str 158 | } 159 | 160 | // UnicodeDec Unicode编码 161 | func UnicodeDec(raw string) string { 162 | str, err := strconv.Unquote(strings.Replace(strconv.Quote(raw), `\\u`, `\u`, -1)) 163 | if err != nil { 164 | return "" 165 | } 166 | return str 167 | } 168 | 169 | // UnicodeDecByte Unicode编码输出[]byte 170 | func UnicodeDecByte(raw []byte) []byte { 171 | rawStr := string(raw) 172 | return []byte(UnicodeDec(rawStr)) 173 | } 174 | 175 | // UnescapeUnicode Unicode 转码 176 | func UnescapeUnicode(raw []byte) ([]byte, error) { 177 | str, err := strconv.Unquote(strings.Replace(strconv.Quote(string(raw)), `\\u`, `\u`, -1)) 178 | if err != nil { 179 | return nil, err 180 | } 181 | return []byte(str), nil 182 | } 183 | 184 | // Base64Encode base64 编码 185 | func Base64Encode(str string) string { 186 | return base64.StdEncoding.EncodeToString([]byte(str)) 187 | } 188 | 189 | // Base64Decode base64 解码 190 | func Base64Decode(str string) (string, error) { 191 | b, err := base64.StdEncoding.DecodeString(str) 192 | return string(b), err 193 | } 194 | 195 | // Base64UrlEncode base64 url 编码 196 | func Base64UrlEncode(str string) string { 197 | return base64.URLEncoding.EncodeToString([]byte(str)) 198 | } 199 | 200 | // Base64UrlDecode base64 url 解码 201 | func Base64UrlDecode(str string) (string, error) { 202 | b, err := base64.URLEncoding.DecodeString(str) 203 | return string(b), err 204 | } 205 | 206 | func convert(dstCharset string, srcCharset string, src string) (dst string, err error) { 207 | if dstCharset == srcCharset { 208 | return src, nil 209 | } 210 | dst = src 211 | 212 | if srcCharset != "UTF-8" { 213 | if e := getEncoding(srcCharset); e != nil { 214 | tmp, err := io.ReadAll( 215 | transform.NewReader(bytes.NewReader([]byte(src)), e.NewDecoder()), 216 | ) 217 | if err != nil { 218 | return "", err 219 | } 220 | src = string(tmp) 221 | } else { 222 | return dst, err 223 | } 224 | } 225 | 226 | if dstCharset != "UTF-8" { 227 | if e := getEncoding(dstCharset); e != nil { 228 | tmp, err := io.ReadAll( 229 | transform.NewReader(bytes.NewReader([]byte(src)), e.NewEncoder()), 230 | ) 231 | if err != nil { 232 | return "", err 233 | } 234 | dst = string(tmp) 235 | } else { 236 | return dst, err 237 | } 238 | } else { 239 | dst = src 240 | } 241 | return dst, nil 242 | } 243 | 244 | var ( 245 | // Alias for charsets. 246 | charsetAlias = map[string]string{ 247 | "HZGB2312": "HZ-GB-2312", 248 | "hzgb2312": "HZ-GB-2312", 249 | "GB2312": "HZ-GB-2312", 250 | "gb2312": "HZ-GB-2312", 251 | } 252 | ) 253 | 254 | func getEncoding(charset string) encoding.Encoding { 255 | if c, ok := charsetAlias[charset]; ok { 256 | charset = c 257 | } 258 | enc, err := ianaindex.MIB.Encoding(charset) 259 | if err != nil { 260 | log.Println(err) 261 | } 262 | return enc 263 | } 264 | 265 | // ToUTF8 to utf8 266 | func ToUTF8(srcCharset string, src string) (dst string, err error) { 267 | return convert("UTF-8", srcCharset, src) 268 | } 269 | 270 | // UTF8To utf8 to 271 | func UTF8To(dstCharset string, src string) (dst string, err error) { 272 | return convert(dstCharset, "UTF-8", src) 273 | } 274 | 275 | // ToUTF16 to utf16 276 | func ToUTF16(srcCharset string, src string) (dst string, err error) { 277 | return convert("UTF-16", srcCharset, src) 278 | } 279 | 280 | // UTF16To utf16 to 281 | func UTF16To(dstCharset string, src string) (dst string, err error) { 282 | return convert(dstCharset, "UTF-16", src) 283 | } 284 | 285 | // ToBIG5 to big5 286 | func ToBIG5(srcCharset string, src string) (dst string, err error) { 287 | return convert("big5", srcCharset, src) 288 | } 289 | 290 | // BIG5To big to 291 | func BIG5To(dstCharset string, src string) (dst string, err error) { 292 | return convert(dstCharset, "big5", src) 293 | } 294 | 295 | // ToGDK to gdk 296 | func ToGDK(srcCharset string, src string) (dst string, err error) { 297 | return convert("gbk", srcCharset, src) 298 | } 299 | 300 | // GDKTo gdk to 301 | func GDKTo(dstCharset string, src string) (dst string, err error) { 302 | return convert(dstCharset, "gbk", src) 303 | } 304 | 305 | // ToGB18030 to gb18030 306 | func ToGB18030(srcCharset string, src string) (dst string, err error) { 307 | return convert("gb18030", srcCharset, src) 308 | } 309 | 310 | // GB18030To gb18030 to 311 | func GB18030To(dstCharset string, src string) (dst string, err error) { 312 | return convert(dstCharset, "gb18030", src) 313 | } 314 | 315 | // ToGB2312 to gb2312 316 | func ToGB2312(srcCharset string, src string) (dst string, err error) { 317 | return convert("GB2312", srcCharset, src) 318 | } 319 | 320 | // GB2312To gb2312 to 321 | func GB2312To(dstCharset string, src string) (dst string, err error) { 322 | return convert(dstCharset, "GB2312", src) 323 | } 324 | 325 | // ToHZGB2312 to hzgb2312 326 | func ToHZGB2312(srcCharset string, src string) (dst string, err error) { 327 | return convert("HZGB2312", srcCharset, src) 328 | } 329 | 330 | // HZGB2312To hzgb2312 to 331 | func HZGB2312To(dstCharset string, src string) (dst string, err error) { 332 | return convert(dstCharset, "HZGB2312", src) 333 | } 334 | 335 | // IF 三元表达式 336 | func IF[T any](condition bool, a, b T) T { 337 | if condition { 338 | return a 339 | } 340 | return b 341 | } 342 | 343 | // ReplaceAllToOne 批量统一替换字符串 344 | func ReplaceAllToOne(str string, from []string, to string) string { 345 | arr := make([]string, len(from)*2) 346 | for i, s := range from { 347 | arr[i*2] = s 348 | arr[i*2+1] = to 349 | } 350 | r := strings.NewReplacer(arr...) 351 | return r.Replace(str) 352 | } 353 | 354 | // 字节换算 355 | const ( 356 | KiB = 1024 357 | MiB = KiB * 1024 358 | GiB = MiB * 1024 359 | TiB = GiB * 1024 360 | ) 361 | 362 | // HumanFriendlyTraffic 字节换算 363 | func HumanFriendlyTraffic(bytes uint64) string { 364 | if bytes <= KiB { 365 | return fmt.Sprintf("%d B", bytes) 366 | } 367 | if bytes <= MiB { 368 | return fmt.Sprintf("%.2f KiB", float32(bytes)/KiB) 369 | } 370 | if bytes <= GiB { 371 | return fmt.Sprintf("%.2f MiB", float32(bytes)/MiB) 372 | } 373 | if bytes <= TiB { 374 | return fmt.Sprintf("%.2f GiB", float32(bytes)/GiB) 375 | } 376 | return fmt.Sprintf("%.2f TiB", float32(bytes)/TiB) 377 | } 378 | 379 | // StrToSize 字节换算 380 | func StrToSize(sizeStr string) int64 { 381 | i := 0 382 | for ; i < len(sizeStr); i++ { 383 | if sizeStr[i] == '.' || (sizeStr[i] >= '0' && sizeStr[i] <= '9') { 384 | continue 385 | } else { 386 | break 387 | } 388 | } 389 | var ( 390 | unit = sizeStr[i:] 391 | number, _ = strconv.ParseFloat(sizeStr[:i], 64) 392 | ) 393 | if unit == "" { 394 | return int64(number) 395 | } 396 | switch strings.ToLower(unit) { 397 | case "b", "bytes": 398 | return int64(number) 399 | case "k", "kb", "ki", "kib", "kilobyte": 400 | return int64(number * 1024) 401 | case "m", "mb", "mi", "mib", "mebibyte": 402 | return int64(number * 1024 * 1024) 403 | case "g", "gb", "gi", "gib", "gigabyte": 404 | return int64(number * 1024 * 1024 * 1024) 405 | case "t", "tb", "ti", "tib", "terabyte": 406 | return int64(number * 1024 * 1024 * 1024 * 1024) 407 | case "p", "pb", "pi", "pib", "petabyte": 408 | return int64(number * 1024 * 1024 * 1024 * 1024 * 1024) 409 | case "e", "eb", "ei", "eib", "exabyte": 410 | return int64(number * 1024 * 1024 * 1024 * 1024 * 1024 * 1024) 411 | case "z", "zb", "zi", "zib", "zettabyte": 412 | return int64(number * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024) 413 | case "y", "yb", "yi", "yib", "yottabyte": 414 | return int64(number * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024) 415 | case "bb", "brontobyte": 416 | return int64(number * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024) 417 | } 418 | return -1 419 | } 420 | 421 | // GzipCompress Gzip压缩 422 | func GzipCompress(src []byte) []byte { 423 | var in bytes.Buffer 424 | w := gzip.NewWriter(&in) 425 | _, _ = w.Write(src) 426 | _ = w.Close() 427 | return in.Bytes() 428 | } 429 | 430 | // GzipDecompress Gzip解压 431 | func GzipDecompress(src []byte) []byte { 432 | dst := make([]byte, 0) 433 | br := bytes.NewReader(src) 434 | gr, err := gzip.NewReader(br) 435 | if err != nil { 436 | return dst 437 | } 438 | defer func() { 439 | _ = gr.Close() 440 | }() 441 | tmp, err := io.ReadAll(gr) 442 | if err != nil { 443 | return dst 444 | } 445 | dst = tmp 446 | return dst 447 | } 448 | 449 | // ConvertStr2GBK 将utf-8编码的字符串转换为GBK编码 450 | func ConvertStr2GBK(str string) string { 451 | ret, err := simplifiedchinese.GBK.NewEncoder().String(str) 452 | if err != nil { 453 | ret = str 454 | } 455 | return ret 456 | } 457 | 458 | // ConvertGBK2Str 将GBK编码的字符串转换为utf-8编码 459 | func ConvertGBK2Str(gbkStr string) string { 460 | ret, err := simplifiedchinese.GBK.NewDecoder().String(gbkStr) 461 | if err != nil { 462 | ret = gbkStr 463 | } 464 | return ret 465 | } 466 | 467 | // ByteToGBK byte -> gbk byte 468 | func ByteToGBK(strBuf []byte) []byte { 469 | if IsUtf8(strBuf) { 470 | if GBKBuf, err := simplifiedchinese.GBK.NewEncoder().Bytes(strBuf); err == nil { 471 | if IsUtf8(GBKBuf) { 472 | return GBKBuf 473 | } 474 | } 475 | if GB18030Buf, err := simplifiedchinese.GB18030.NewEncoder().Bytes(strBuf); err == nil { 476 | if IsUtf8(GB18030Buf) { 477 | return GB18030Buf 478 | } 479 | } 480 | if HZGB2312Buf, err := simplifiedchinese.HZGB2312.NewEncoder().Bytes(strBuf); err == nil { 481 | if IsUtf8(HZGB2312Buf) { 482 | return HZGB2312Buf 483 | } 484 | } 485 | return strBuf 486 | } else { 487 | return strBuf 488 | } 489 | } 490 | 491 | // ByteToUTF8 byte -> utf8 byte 492 | func ByteToUTF8(strBuf []byte) []byte { 493 | if IsUtf8(strBuf) { 494 | return strBuf 495 | } else { 496 | if GBKBuf, err := simplifiedchinese.GBK.NewDecoder().Bytes(strBuf); err == nil { 497 | if IsUtf8(GBKBuf) { 498 | return GBKBuf 499 | } 500 | } 501 | if GB18030Buf, err := simplifiedchinese.GB18030.NewDecoder().Bytes(strBuf); err == nil { 502 | if IsUtf8(GB18030Buf) { 503 | return GB18030Buf 504 | } 505 | } 506 | if HZGB2312Buf, err := simplifiedchinese.HZGB2312.NewDecoder().Bytes(strBuf); err == nil { 507 | fmt.Println("3") 508 | if IsUtf8(HZGB2312Buf) { 509 | return HZGB2312Buf 510 | } 511 | } 512 | return strBuf 513 | } 514 | } 515 | 516 | // IsUtf8 是否是utf8编码 517 | func IsUtf8(buf []byte) bool { 518 | return utf8.Valid(buf) 519 | } 520 | 521 | // Get16MD5Encode 返回一个16位md5加密后的字符串 522 | func Get16MD5Encode(data string) string { 523 | return GetMD5Encode(data)[8:24] 524 | } 525 | 526 | // GetMD5Encode 获取Md5编码 527 | func GetMD5Encode(data string) string { 528 | h := md5.New() 529 | h.Write([]byte(data)) 530 | return hex.EncodeToString(h.Sum(nil)) 531 | } 532 | -------------------------------------------------------------------------------- /conversion.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Description : 数据类型转换 3 | * Author : ManGe 4 | * Mail : 2912882908@qq.com 5 | **/ 6 | 7 | package gathertool 8 | 9 | import ( 10 | "bytes" 11 | "encoding/binary" 12 | "encoding/json" 13 | "fmt" 14 | "math" 15 | "net" 16 | "reflect" 17 | "strconv" 18 | "strings" 19 | "unsafe" 20 | ) 21 | 22 | // Json2Map 数据类型转换 json -> map 23 | func Json2Map(str string) (map[string]any, error) { 24 | var tempMap map[string]any 25 | err := json.Unmarshal([]byte(str), &tempMap) 26 | return tempMap, err 27 | } 28 | 29 | // Any2Map 数据类型转换 any -> map[string]any 30 | func Any2Map(data any) map[string]any { 31 | if v, ok := data.(map[string]any); ok { 32 | return v 33 | } 34 | 35 | if reflect.ValueOf(data).Kind() == reflect.String { 36 | dataMap, err := Json2Map(data.(string)) 37 | if err == nil { 38 | return dataMap 39 | } 40 | } 41 | return nil 42 | } 43 | 44 | // Any2String 数据类型转换 any -> string 45 | func Any2String(i any) string { 46 | if i == nil { 47 | return "" 48 | } 49 | 50 | if reflect.ValueOf(i).Kind() == reflect.String { 51 | return i.(string) 52 | } 53 | 54 | var buf bytes.Buffer 55 | stringValue(reflect.ValueOf(i), 0, &buf) 56 | return buf.String() 57 | } 58 | 59 | func stringValue(v reflect.Value, indent int, buf *bytes.Buffer) { 60 | for v.Kind() == reflect.Ptr { 61 | v = v.Elem() 62 | } 63 | switch v.Kind() { 64 | case reflect.Struct: 65 | buf.WriteString("{\n") 66 | for i := 0; i < v.Type().NumField(); i++ { 67 | ft := v.Type().Field(i) 68 | fv := v.Field(i) 69 | if ft.Name[0:1] == strings.ToLower(ft.Name[0:1]) { 70 | continue 71 | } 72 | if (fv.Kind() == reflect.Ptr || fv.Kind() == reflect.Slice) && fv.IsNil() { 73 | continue 74 | } 75 | buf.WriteString(strings.Repeat(" ", indent+2)) 76 | buf.WriteString(ft.Name + ": ") 77 | if tag := ft.Tag.Get("sensitive"); tag == "true" { 78 | buf.WriteString("") 79 | } else { 80 | stringValue(fv, indent+2, buf) 81 | } 82 | buf.WriteString(",\n") 83 | } 84 | buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") 85 | 86 | case reflect.Slice: 87 | nl, id, id2 := "", "", "" 88 | if v.Len() > 3 { 89 | nl, id, id2 = "\n", strings.Repeat(" ", indent), strings.Repeat(" ", indent+2) 90 | } 91 | buf.WriteString("[" + nl) 92 | for i := 0; i < v.Len(); i++ { 93 | buf.WriteString(id2) 94 | stringValue(v.Index(i), indent+2, buf) 95 | 96 | if i < v.Len()-1 { 97 | buf.WriteString("," + nl) 98 | } 99 | } 100 | buf.WriteString(nl + id + "]") 101 | 102 | case reflect.Map: 103 | buf.WriteString("{\n") 104 | for i, k := range v.MapKeys() { 105 | buf.WriteString(strings.Repeat(" ", indent+2)) 106 | buf.WriteString(k.String() + ": ") 107 | stringValue(v.MapIndex(k), indent+2, buf) 108 | if i < v.Len()-1 { 109 | buf.WriteString(",\n") 110 | } 111 | } 112 | buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") 113 | 114 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 115 | buf.WriteString(strconv.FormatInt(v.Int(), 10)) 116 | 117 | case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: 118 | buf.WriteString(strconv.FormatUint(v.Uint(), 10)) 119 | 120 | case reflect.Float32, reflect.Float64: 121 | result := fmt.Sprintf("%f", v.Float()) 122 | // 去除result末尾的0 123 | for strings.HasSuffix(result, "0") { 124 | result = strings.TrimSuffix(result, "0") 125 | } 126 | if strings.HasSuffix(result, ".") { 127 | result = strings.TrimSuffix(result, ".") 128 | } 129 | buf.WriteString(result) 130 | 131 | default: 132 | format := "%v" 133 | switch v.Interface().(type) { 134 | case string: 135 | format = "%q" 136 | } 137 | _, _ = fmt.Fprintf(buf, format, v.Interface()) 138 | } 139 | } 140 | 141 | // Any2Int 数据类型转换 any -> int 142 | func Any2Int(data any) int { 143 | var t2 int 144 | switch data.(type) { 145 | case uint: 146 | t2 = int(data.(uint)) 147 | break 148 | case int8: 149 | t2 = int(data.(int8)) 150 | break 151 | case uint8: 152 | t2 = int(data.(uint8)) 153 | break 154 | case int16: 155 | t2 = int(data.(int16)) 156 | break 157 | case uint16: 158 | t2 = int(data.(uint16)) 159 | break 160 | case int32: 161 | t2 = int(data.(int32)) 162 | break 163 | case uint32: 164 | t2 = int(data.(uint32)) 165 | break 166 | case int64: 167 | t2 = int(data.(int64)) 168 | break 169 | case uint64: 170 | t2 = int(data.(uint64)) 171 | break 172 | case float32: 173 | t2 = int(data.(float32)) 174 | break 175 | case float64: 176 | t2 = int(data.(float64)) 177 | break 178 | case string: 179 | t2, _ = strconv.Atoi(data.(string)) 180 | break 181 | default: 182 | t2 = data.(int) 183 | break 184 | } 185 | return t2 186 | } 187 | 188 | // Any2Int64 数据类型转换 any -> int64 189 | func Any2Int64(data any) int64 { 190 | return int64(Any2Int(data)) 191 | } 192 | 193 | // Any2Arr 数据类型转换 any -> []any 194 | func Any2Arr(data any) []any { 195 | if v, ok := data.([]any); ok { 196 | return v 197 | } 198 | return nil 199 | } 200 | 201 | // Any2Float64 数据类型转换 any -> float64 202 | func Any2Float64(data any) float64 { 203 | if v, ok := data.(float64); ok { 204 | return v 205 | } 206 | if v, ok := data.(float32); ok { 207 | return float64(v) 208 | } 209 | return 0 210 | } 211 | 212 | // Any2Strings 数据类型转换 any -> []string 213 | func Any2Strings(data any) []string { 214 | listValue, ok := data.([]any) 215 | if !ok { 216 | return nil 217 | } 218 | keyStringValues := make([]string, len(listValue)) 219 | for i, arg := range listValue { 220 | keyStringValues[i] = arg.(string) 221 | } 222 | return keyStringValues 223 | } 224 | 225 | // Any2Json 数据类型转换 any -> json string 226 | func Any2Json(data any) (string, error) { 227 | jsonStr, err := json.Marshal(data) 228 | return string(jsonStr), err 229 | } 230 | 231 | // Int2Hex 数据类型转换 int -> hex 232 | func Int2Hex(i int64) string { 233 | return fmt.Sprintf("%x", i) 234 | } 235 | 236 | // Int642Hex 数据类型转换 int64 -> hex 237 | func Int642Hex(i int64) string { 238 | return fmt.Sprintf("%x", i) 239 | } 240 | 241 | // Hex2Int 数据类型转换 hex -> int 242 | func Hex2Int(s string) int { 243 | n, err := strconv.ParseUint(s, 16, 8) 244 | if err != nil { 245 | Error(err) 246 | return 0 247 | } 248 | 249 | n2 := uint8(n) 250 | return int(*(*int8)(unsafe.Pointer(&n2))) 251 | } 252 | 253 | // Hex2Int64 数据类型转换 hex -> int 254 | func Hex2Int64(s string) int64 { 255 | n, err := strconv.ParseUint(s, 16, 8) 256 | if err != nil { 257 | Error(err) 258 | return 0 259 | } 260 | 261 | n2 := uint8(n) 262 | return int64(*(*int8)(unsafe.Pointer(&n2))) 263 | } 264 | 265 | // Str2Int64 数据类型转换 string -> int64 266 | func Str2Int64(str string) int64 { 267 | i, err := strconv.ParseInt(str, 10, 64) 268 | if err != nil { 269 | return 0 270 | } 271 | return i 272 | } 273 | 274 | // Str2Int 数据类型转换 string -> int 275 | func Str2Int(str string) int { 276 | i, err := strconv.Atoi(str) 277 | if err != nil { 278 | return 0 279 | } 280 | return i 281 | } 282 | 283 | // Str2Int32 数据类型转换 string -> int32 284 | func Str2Int32(str string) int32 { 285 | i, err := strconv.Atoi(str) 286 | if err != nil { 287 | return 0 288 | } 289 | return int32(i) 290 | } 291 | 292 | // Str2Float64 数据类型转换 string -> float64 293 | func Str2Float64(str string) float64 { 294 | i, err := strconv.ParseFloat(str, 64) 295 | if err != nil { 296 | return 0 297 | } 298 | return i 299 | } 300 | 301 | // Str2Float32 数据类型转换 string -> float32 302 | func Str2Float32(str string) float32 { 303 | i, err := strconv.ParseFloat(str, 32) 304 | if err != nil { 305 | return 0 306 | } 307 | return float32(i) 308 | } 309 | 310 | // Uint82Str 数据类型转换 []uint8 -> string 311 | func Uint82Str(bs []uint8) string { 312 | ba := make([]byte, 0) 313 | for _, b := range bs { 314 | ba = append(ba, b) 315 | } 316 | return string(ba) 317 | } 318 | 319 | // Str2Byte 数据类型转换 string -> []byte 320 | func Str2Byte(s string) []byte { 321 | x := (*[2]uintptr)(unsafe.Pointer(&s)) 322 | h := [3]uintptr{x[0], x[1], x[1]} 323 | return *(*[]byte)(unsafe.Pointer(&h)) 324 | } 325 | 326 | // Byte2Str 数据类型转换 []byte -> string 327 | func Byte2Str(b []byte) string { 328 | return *(*string)(unsafe.Pointer(&b)) 329 | } 330 | 331 | // Bool2Byte 数据类型转换 bool -> []byte 332 | func Bool2Byte(b bool) []byte { 333 | if b == true { 334 | return []byte{1} 335 | } 336 | return []byte{0} 337 | } 338 | 339 | // Byte2Bool 数据类型转换 []byte -> bool 340 | func Byte2Bool(b []byte) bool { 341 | if len(b) == 0 || bytes.Compare(b, make([]byte, len(b))) == 0 { 342 | return false 343 | } 344 | return true 345 | } 346 | 347 | // Int2Byte 数据类型转换 int -> []byte 348 | func Int2Byte(i int) []byte { 349 | b := make([]byte, 4) 350 | binary.LittleEndian.PutUint32(b, uint32(i)) 351 | return b 352 | } 353 | 354 | // Byte2Int 数据类型转换 []byte -> int 355 | func Byte2Int(b []byte) int { 356 | return int(binary.LittleEndian.Uint32(b)) 357 | } 358 | 359 | // Int642Byte 数据类型转换 int64 -> []byte 360 | func Int642Byte(i int64) []byte { 361 | b := make([]byte, 8) 362 | binary.LittleEndian.PutUint64(b, uint64(i)) 363 | return b 364 | } 365 | 366 | // Byte2Int64 数据类型转换 []byte -> int64 367 | func Byte2Int64(b []byte) int64 { 368 | return int64(binary.LittleEndian.Uint64(b)) 369 | } 370 | 371 | // Float322Byte 数据类型转换 float32 -> []byte 372 | func Float322Byte(f float32) []byte { 373 | b := make([]byte, 4) 374 | binary.LittleEndian.PutUint32(b, Float322Uint32(f)) 375 | return b 376 | } 377 | 378 | // Float322Uint32 数据类型转换 float32 -> uint32 379 | func Float322Uint32(f float32) uint32 { 380 | return math.Float32bits(f) 381 | } 382 | 383 | // Byte2Float32 数据类型转换 []byte -> float32 384 | func Byte2Float32(b []byte) float32 { 385 | return math.Float32frombits(binary.LittleEndian.Uint32(b)) 386 | } 387 | 388 | // Float642Byte 数据类型转换 float64 -> []byte 389 | func Float642Byte(f float64) []byte { 390 | b := make([]byte, 8) 391 | binary.LittleEndian.PutUint64(b, Float642Uint64(f)) 392 | return b 393 | } 394 | 395 | // Float642Uint64 数据类型转换 float64 -> uint64 396 | func Float642Uint64(f float64) uint64 { 397 | return math.Float64bits(f) 398 | } 399 | 400 | // Byte2Float64 数据类型转换 []byte -> float64 401 | func Byte2Float64(b []byte) float64 { 402 | return math.Float64frombits(binary.LittleEndian.Uint64(b)) 403 | } 404 | 405 | // Struct2Map 数据类型转换 Struct -> map 406 | // 参数说明: 407 | // - hasValue=true表示字段值不管是否存在都转换成map 408 | // - hasValue=false表示字段为空或者不为0则转换成map 409 | func Struct2Map(obj any, hasValue bool) (map[string]any, error) { 410 | mp := make(map[string]any) 411 | value := reflect.ValueOf(obj).Elem() 412 | typeOf := reflect.TypeOf(obj).Elem() 413 | for i := 0; i < value.NumField(); i++ { 414 | switch value.Field(i).Kind() { 415 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, 416 | reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: 417 | if hasValue { 418 | if value.Field(i).Int() != 0 { 419 | mp[typeOf.Field(i).Name] = value.Field(i).Int() 420 | } 421 | } else { 422 | mp[typeOf.Field(i).Name] = value.Field(i).Int() 423 | } 424 | 425 | case reflect.String: 426 | if hasValue { 427 | if len(value.Field(i).String()) != 0 { 428 | mp[typeOf.Field(i).Name] = value.Field(i).String() 429 | } 430 | } else { 431 | mp[typeOf.Field(i).Name] = value.Field(i).String() 432 | } 433 | 434 | case reflect.Float32, reflect.Float64: 435 | if hasValue { 436 | if len(value.Field(i).String()) != 0 { 437 | mp[typeOf.Field(i).Name] = value.Field(i).Float() 438 | } 439 | } else { 440 | mp[typeOf.Field(i).Name] = value.Field(i).Float() 441 | } 442 | 443 | case reflect.Bool: 444 | if hasValue { 445 | if len(value.Field(i).String()) != 0 { 446 | mp[typeOf.Field(i).Name] = value.Field(i).Bool() 447 | } 448 | } else { 449 | mp[typeOf.Field(i).Name] = value.Field(i).Bool() 450 | } 451 | 452 | default: 453 | return mp, fmt.Errorf("数据类型不匹配") 454 | } 455 | } 456 | return mp, nil 457 | } 458 | 459 | // Byte2Bit 数据类型转换 []byte -> []uint8 (bit) 460 | func Byte2Bit(b []byte) []uint8 { 461 | bits := make([]uint8, 0) 462 | for _, v := range b { 463 | bits = bits2Uint(bits, uint(v), 8) 464 | } 465 | return bits 466 | } 467 | 468 | func bits2Uint(bits []uint8, ui uint, l int) []uint8 { 469 | a := make([]uint8, l) 470 | for i := l - 1; i >= 0; i-- { 471 | a[i] = uint8(ui & 1) 472 | ui >>= 1 473 | } 474 | if bits != nil { 475 | return append(bits, a...) 476 | } 477 | return a 478 | } 479 | 480 | // Bit2Byte 数据类型转换 []uint8 -> []byte 481 | func Bit2Byte(b []uint8) []byte { 482 | if len(b)%8 != 0 { 483 | for i := 0; i < len(b)%8; i++ { 484 | b = append(b, 0) 485 | } 486 | } 487 | by := make([]byte, 0) 488 | for i := 0; i < len(b); i += 8 { 489 | by = append(b, byte(bitsToUint(b[i:i+8]))) 490 | } 491 | return by 492 | } 493 | 494 | func bitsToUint(bits []uint8) uint { 495 | v := uint(0) 496 | for _, i := range bits { 497 | v = v<<1 | uint(i) 498 | } 499 | return v 500 | } 501 | 502 | // ByteToBinaryString 数据类型转换 字节 -> 二进制字符串 503 | func ByteToBinaryString(data byte) (str string) { 504 | var a byte 505 | for i := 0; i < 8; i++ { 506 | a = data 507 | data <<= 1 508 | data >>= 1 509 | switch a { 510 | case data: 511 | str += "0" 512 | default: 513 | str += "1" 514 | } 515 | data <<= 1 516 | } 517 | return str 518 | } 519 | 520 | // IP2Binary 数据类型转换 IP str -> binary int64 521 | func IP2Binary(ip string) string { 522 | rse := IP2Int64(ip) 523 | return strconv.FormatInt(rse, 2) 524 | } 525 | 526 | // UInt32ToIP 数据类型转换 uint32 -> net.IP 527 | func UInt32ToIP(ip uint32) net.IP { 528 | var b [4]byte 529 | b[0] = byte(ip & 0xFF) 530 | b[1] = byte((ip >> 8) & 0xFF) 531 | b[2] = byte((ip >> 16) & 0xFF) 532 | b[3] = byte((ip >> 24) & 0xFF) 533 | return net.IPv4(b[3], b[2], b[1], b[0]) 534 | } 535 | 536 | // IP2Int64 数据类型转换 IP str -> int64 537 | func IP2Int64(ip string) int64 { 538 | address := net.ParseIP(ip) 539 | if address == nil { 540 | Error("ip地址不正确") 541 | return 0 542 | } 543 | bits := strings.Split(ip, ".") 544 | b0, b1, b2, b3 := 0, 0, 0, 0 545 | if len(bits) >= 1 { 546 | b0, _ = strconv.Atoi(bits[0]) 547 | } 548 | if len(bits) >= 2 { 549 | b1, _ = strconv.Atoi(bits[1]) 550 | } 551 | if len(bits) >= 3 { 552 | b2, _ = strconv.Atoi(bits[2]) 553 | } 554 | if len(bits) >= 4 { 555 | b3, _ = strconv.Atoi(bits[3]) 556 | } 557 | var sum int64 558 | sum += int64(b0) << 24 559 | sum += int64(b1) << 16 560 | sum += int64(b2) << 8 561 | sum += int64(b3) 562 | return sum 563 | } 564 | --------------------------------------------------------------------------------