element with collection-product-card class call callback
73 | c.OnHTML(`a.collection-product-card`, func(e *colly.HTMLElement) {
74 | // Activate detailCollector if the link contains "coursera.org/learn"
75 | courseURL := e.Request.AbsoluteURL(e.Attr("href"))
76 | if strings.Index(courseURL, "coursera.org/learn") != -1 {
77 | detailCollector.Visit(courseURL)
78 | }
79 | })
80 |
81 | // Extract details of the course
82 | detailCollector.OnHTML(`div[id=rendered-content]`, func(e *colly.HTMLElement) {
83 | log.Println("Course found", e.Request.URL)
84 | title := e.ChildText(".banner-title")
85 | if title == "" {
86 | log.Println("No title found", e.Request.URL)
87 | }
88 | course := Course{
89 | Title: title,
90 | URL: e.Request.URL.String(),
91 | Description: e.ChildText("div.content"),
92 | Creator: e.ChildText("li.banner-instructor-info > a > div > div > span"),
93 | Rating: e.ChildText("span.number-rating"),
94 | }
95 | // Iterate over div components and add details to course
96 | e.ForEach(".AboutCourse .ProductGlance > div", func(_ int, el *colly.HTMLElement) {
97 | svgTitle := strings.Split(el.ChildText("div:nth-child(1) svg title"), " ")
98 | lastWord := svgTitle[len(svgTitle)-1]
99 | switch lastWord {
100 | // svg Title: Available Languages
101 | case "languages":
102 | course.Language = el.ChildText("div:nth-child(2) > div:nth-child(1)")
103 | // svg Title: Mixed/Beginner/Intermediate/Advanced Level
104 | case "Level":
105 | course.Level = el.ChildText("div:nth-child(2) > div:nth-child(1)")
106 | // svg Title: Hours to complete
107 | case "complete":
108 | course.Commitment = el.ChildText("div:nth-child(2) > div:nth-child(1)")
109 | }
110 | })
111 | courses = append(courses, course)
112 | })
113 |
114 | // Start scraping on http://coursera.com/browse
115 | c.Visit("https://coursera.org/browse")
116 |
117 | enc := json.NewEncoder(file)
118 | enc.SetIndent("", " ")
119 |
120 | // Dump json to the standard output
121 | enc.Encode(courses)
122 | }
123 |
--------------------------------------------------------------------------------
/_examples/cryptocoinmarketcap/cryptocoinmarketcap.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/csv"
5 | "log"
6 | "os"
7 |
8 | "github.com/gocolly/colly/v2"
9 | )
10 |
11 | func main() {
12 | fName := "cryptocoinmarketcap.csv"
13 | file, err := os.Create(fName)
14 | if err != nil {
15 | log.Fatalf("Cannot create file %q: %s\n", fName, err)
16 | return
17 | }
18 | defer file.Close()
19 | writer := csv.NewWriter(file)
20 | defer writer.Flush()
21 |
22 | // Write CSV header
23 | writer.Write([]string{"Name", "Symbol", "Market Cap (USD)", "Price (USD)", "Circulating Supply (USD)", "Volume (24h)", "Change (1h)", "Change (24h)", "Change (7d)"})
24 |
25 | // Instantiate default collector
26 | c := colly.NewCollector()
27 |
28 | c.OnHTML("tbody tr", func(e *colly.HTMLElement) {
29 | writer.Write([]string{
30 | e.ChildText(".cmc-table__column-name"),
31 | e.ChildText(".cmc-table__cell--sort-by__symbol"),
32 | e.ChildText(".cmc-table__cell--sort-by__market-cap"),
33 | e.ChildText(".cmc-table__cell--sort-by__price"),
34 | e.ChildText(".cmc-table__cell--sort-by__circulating-supply"),
35 | e.ChildText(".cmc-table__cell--sort-by__volume-24-h"),
36 | e.ChildText(".cmc-table__cell--sort-by__percent-change-1-h"),
37 | e.ChildText(".cmc-table__cell--sort-by__percent-change-24-h"),
38 | e.ChildText(".cmc-table__cell--sort-by__percent-change-7-d"),
39 | })
40 | })
41 |
42 | c.Visit("https://coinmarketcap.com/all/views/all/")
43 |
44 | log.Printf("Scraping finished, check file %q for results\n", fName)
45 | }
46 |
--------------------------------------------------------------------------------
/_examples/error_handling/error_handling.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | )
8 |
9 | func main() {
10 | // Create a collector
11 | c := colly.NewCollector()
12 |
13 | // Set HTML callback
14 | // Won't be called if error occurs
15 | c.OnHTML("*", func(e *colly.HTMLElement) {
16 | fmt.Println(e)
17 | })
18 |
19 | // Set error handler
20 | c.OnError(func(r *colly.Response, err error) {
21 | fmt.Println("Request URL:", r.Request.URL, "failed with response:", r, "\nError:", err)
22 | })
23 |
24 | // Start scraping
25 | c.Visit("https://definitely-not-a.website/")
26 | }
27 |
--------------------------------------------------------------------------------
/_examples/factba.se/factbase.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "os"
7 | "strconv"
8 |
9 | "github.com/gocolly/colly/v2"
10 | )
11 |
12 | var baseSearchURL = "https://factba.se/json/json-transcript.php?q=&f=&dt=&p="
13 | var baseTranscriptURL = "https://factba.se/transcript/"
14 |
15 | type result struct {
16 | Slug string `json:"slug"`
17 | Date string `json:"date"`
18 | }
19 |
20 | type results struct {
21 | Data []*result `json:"data"`
22 | }
23 |
24 | type transcript struct {
25 | Speaker string
26 | Text string
27 | }
28 |
29 | func main() {
30 | c := colly.NewCollector(
31 | colly.AllowedDomains("factba.se"),
32 | )
33 |
34 | d := c.Clone()
35 |
36 | d.OnHTML("body", func(e *colly.HTMLElement) {
37 | t := make([]transcript, 0)
38 | e.ForEach(".topic-media-row", func(_ int, el *colly.HTMLElement) {
39 | t = append(t, transcript{
40 | Speaker: el.ChildText(".speaker-label"),
41 | Text: el.ChildText(".transcript-text-block"),
42 | })
43 | })
44 | jsonData, err := json.MarshalIndent(t, "", " ")
45 | if err != nil {
46 | return
47 | }
48 | os.WriteFile(colly.SanitizeFileName(e.Request.Ctx.Get("date")+"_"+e.Request.Ctx.Get("slug"))+".json", jsonData, 0644)
49 | })
50 |
51 | stop := false
52 | c.OnResponse(func(r *colly.Response) {
53 | rs := &results{}
54 | err := json.Unmarshal(r.Body, rs)
55 | if err != nil || len(rs.Data) == 0 {
56 | stop = true
57 | return
58 | }
59 | for _, res := range rs.Data {
60 | u := baseTranscriptURL + res.Slug
61 | ctx := colly.NewContext()
62 | ctx.Put("date", res.Date)
63 | ctx.Put("slug", res.Slug)
64 | d.Request("GET", u, nil, ctx, nil)
65 | }
66 | })
67 |
68 | for i := 1; i < 1000; i++ {
69 | if stop {
70 | break
71 | }
72 | if err := c.Visit(baseSearchURL + strconv.Itoa(i)); err != nil {
73 | fmt.Println("Error:", err)
74 | break
75 | }
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/_examples/google_groups/google_groups.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "flag"
6 | "log"
7 | "os"
8 | "strings"
9 |
10 | "github.com/gocolly/colly/v2"
11 | )
12 |
13 | // Mail is the container of a single e-mail
14 | type Mail struct {
15 | Title string
16 | Link string
17 | Author string
18 | Date string
19 | Message string
20 | }
21 |
22 | func main() {
23 | var groupName string
24 | flag.StringVar(&groupName, "group", "hspbp", "Google Groups group name")
25 | flag.Parse()
26 |
27 | threads := make(map[string][]Mail)
28 |
29 | threadCollector := colly.NewCollector()
30 | mailCollector := colly.NewCollector()
31 |
32 | // Collect threads
33 | threadCollector.OnHTML("tr", func(e *colly.HTMLElement) {
34 | ch := e.DOM.Children()
35 | author := ch.Eq(1).Text()
36 | // deleted topic
37 | if author == "" {
38 | return
39 | }
40 |
41 | title := ch.Eq(0).Text()
42 | link, _ := ch.Eq(0).Children().Eq(0).Attr("href")
43 | // fix link to point to the pure HTML version of the thread
44 | link = strings.Replace(link, ".com/d/topic", ".com/forum/?_escaped_fragment_=topic", 1)
45 | date := ch.Eq(2).Text()
46 |
47 | log.Printf("Thread found: %s %q %s %s\n", link, title, author, date)
48 | mailCollector.Visit(link)
49 | })
50 |
51 | // Visit next page
52 | threadCollector.OnHTML("body > a[href]", func(e *colly.HTMLElement) {
53 | log.Println("Next page link found:", e.Attr("href"))
54 | e.Request.Visit(e.Attr("href"))
55 | })
56 |
57 | // Extract mails
58 | mailCollector.OnHTML("body", func(e *colly.HTMLElement) {
59 | // Find subject
60 | threadSubject := e.ChildText("h2")
61 | if _, ok := threads[threadSubject]; !ok {
62 | threads[threadSubject] = make([]Mail, 0, 8)
63 | }
64 |
65 | // Extract mails
66 | e.ForEach("table tr", func(_ int, el *colly.HTMLElement) {
67 | mail := Mail{
68 | Title: el.ChildText("td:nth-of-type(1)"),
69 | Link: el.ChildAttr("td:nth-of-type(1)", "href"),
70 | Author: el.ChildText("td:nth-of-type(2)"),
71 | Date: el.ChildText("td:nth-of-type(3)"),
72 | Message: el.ChildText("td:nth-of-type(4)"),
73 | }
74 | threads[threadSubject] = append(threads[threadSubject], mail)
75 | })
76 |
77 | // Follow next page link
78 | if link, found := e.DOM.Find("> a[href]").Attr("href"); found {
79 | e.Request.Visit(link)
80 | } else {
81 | log.Printf("Thread %q done\n", threadSubject)
82 | }
83 | })
84 |
85 | threadCollector.Visit("https://groups.google.com/forum/?_escaped_fragment_=forum/" + groupName)
86 |
87 | enc := json.NewEncoder(os.Stdout)
88 | enc.SetIndent("", " ")
89 |
90 | // Dump json to the standard output
91 | enc.Encode(threads)
92 | }
93 |
--------------------------------------------------------------------------------
/_examples/hackernews_comments/hackernews_comments.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "flag"
6 | "log"
7 | "os"
8 | "strconv"
9 | "strings"
10 |
11 | "github.com/gocolly/colly/v2"
12 | )
13 |
14 | type comment struct {
15 | Author string `selector:"a.hnuser"`
16 | URL string `selector:".age a[href]" attr:"href"`
17 | Comment string `selector:".comment"`
18 | Replies []*comment
19 | depth int
20 | }
21 |
22 | func main() {
23 | var itemID string
24 | flag.StringVar(&itemID, "id", "", "hackernews post id")
25 | flag.Parse()
26 |
27 | if itemID == "" {
28 | log.Println("Hackernews post id required")
29 | os.Exit(1)
30 | }
31 |
32 | comments := make([]*comment, 0)
33 |
34 | // Instantiate default collector
35 | c := colly.NewCollector()
36 |
37 | // Extract comment
38 | c.OnHTML(".comment-tree tr.athing", func(e *colly.HTMLElement) {
39 | width, err := strconv.Atoi(e.ChildAttr("td.ind img", "width"))
40 | if err != nil {
41 | return
42 | }
43 | // hackernews uses 40px spacers to indent comment replies,
44 | // so we have to divide the width with it to get the depth
45 | // of the comment
46 | depth := width / 40
47 | c := &comment{
48 | Replies: make([]*comment, 0),
49 | depth: depth,
50 | }
51 | e.Unmarshal(c)
52 | c.Comment = strings.TrimSpace(c.Comment[:len(c.Comment)-5])
53 | if depth == 0 {
54 | comments = append(comments, c)
55 | return
56 | }
57 | parent := comments[len(comments)-1]
58 | // append comment to its parent
59 | for i := 0; i < depth-1; i++ {
60 | parent = parent.Replies[len(parent.Replies)-1]
61 | }
62 | parent.Replies = append(parent.Replies, c)
63 | })
64 |
65 | c.Visit("https://news.ycombinator.com/item?id=" + itemID)
66 |
67 | enc := json.NewEncoder(os.Stdout)
68 | enc.SetIndent("", " ")
69 |
70 | // Dump json to the standard output
71 | enc.Encode(comments)
72 | }
73 |
--------------------------------------------------------------------------------
/_examples/instagram/instagram.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "crypto/md5"
5 | "encoding/json"
6 | "fmt"
7 | "log"
8 | "net/url"
9 | "os"
10 | "regexp"
11 | "strings"
12 |
13 | "github.com/gocolly/colly/v2"
14 | )
15 |
16 | // "id": user id, "after": end cursor
17 | const nextPageURL string = `https://www.instagram.com/graphql/query/?query_hash=%s&variables=%s`
18 | const nextPagePayload string = `{"id":"%s","first":50,"after":"%s"}`
19 |
20 | var requestID string
21 | var requestIds [][]byte
22 | var queryIdPattern = regexp.MustCompile(`queryId:".{32}"`)
23 |
24 | type pageInfo struct {
25 | EndCursor string `json:"end_cursor"`
26 | NextPage bool `json:"has_next_page"`
27 | }
28 |
29 | type mainPageData struct {
30 | Rhxgis string `json:"rhx_gis"`
31 | EntryData struct {
32 | ProfilePage []struct {
33 | Graphql struct {
34 | User struct {
35 | Id string `json:"id"`
36 | Media struct {
37 | Edges []struct {
38 | Node struct {
39 | ImageURL string `json:"display_url"`
40 | ThumbnailURL string `json:"thumbnail_src"`
41 | IsVideo bool `json:"is_video"`
42 | Date int `json:"date"`
43 | Dimensions struct {
44 | Width int `json:"width"`
45 | Height int `json:"height"`
46 | } `json:"dimensions"`
47 | } `json::node"`
48 | } `json:"edges"`
49 | PageInfo pageInfo `json:"page_info"`
50 | } `json:"edge_owner_to_timeline_media"`
51 | } `json:"user"`
52 | } `json:"graphql"`
53 | } `json:"ProfilePage"`
54 | } `json:"entry_data"`
55 | }
56 |
57 | type nextPageData struct {
58 | Data struct {
59 | User struct {
60 | Container struct {
61 | PageInfo pageInfo `json:"page_info"`
62 | Edges []struct {
63 | Node struct {
64 | ImageURL string `json:"display_url"`
65 | ThumbnailURL string `json:"thumbnail_src"`
66 | IsVideo bool `json:"is_video"`
67 | Date int `json:"taken_at_timestamp"`
68 | Dimensions struct {
69 | Width int `json:"width"`
70 | Height int `json:"height"`
71 | }
72 | }
73 | } `json:"edges"`
74 | } `json:"edge_owner_to_timeline_media"`
75 | }
76 | } `json:"data"`
77 | }
78 |
79 | func main() {
80 | if len(os.Args) != 2 {
81 | log.Println("Missing account name argument")
82 | os.Exit(1)
83 | }
84 |
85 | var actualUserId string
86 | instagramAccount := os.Args[1]
87 | outputDir := fmt.Sprintf("./instagram_%s/", instagramAccount)
88 |
89 | c := colly.NewCollector(
90 | //colly.CacheDir("./_instagram_cache/"),
91 | colly.UserAgent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"),
92 | )
93 |
94 | c.OnRequest(func(r *colly.Request) {
95 | r.Headers.Set("X-Requested-With", "XMLHttpRequest")
96 | r.Headers.Set("Referer", "https://www.instagram.com/"+instagramAccount)
97 | if r.Ctx.Get("gis") != "" {
98 | gis := fmt.Sprintf("%s:%s", r.Ctx.Get("gis"), r.Ctx.Get("variables"))
99 | h := md5.New()
100 | h.Write([]byte(gis))
101 | gisHash := fmt.Sprintf("%x", h.Sum(nil))
102 | r.Headers.Set("X-Instagram-GIS", gisHash)
103 | }
104 | })
105 |
106 | c.OnHTML("html", func(e *colly.HTMLElement) {
107 | d := c.Clone()
108 | d.OnResponse(func(r *colly.Response) {
109 | requestIds = queryIdPattern.FindAll(r.Body, -1)
110 | requestID = string(requestIds[1][9:41])
111 | })
112 | requestIDURL := e.Request.AbsoluteURL(e.ChildAttr(`link[as="script"]`, "href"))
113 | d.Visit(requestIDURL)
114 |
115 | dat := e.ChildText("body > script:first-of-type")
116 | jsonData := dat[strings.Index(dat, "{") : len(dat)-1]
117 | data := &mainPageData{}
118 | err := json.Unmarshal([]byte(jsonData), data)
119 | if err != nil {
120 | log.Fatal(err)
121 | }
122 |
123 | log.Println("saving output to ", outputDir)
124 | os.MkdirAll(outputDir, os.ModePerm)
125 | page := data.EntryData.ProfilePage[0]
126 | actualUserId = page.Graphql.User.Id
127 | for _, obj := range page.Graphql.User.Media.Edges {
128 | // skip videos
129 | if obj.Node.IsVideo {
130 | continue
131 | }
132 | c.Visit(obj.Node.ImageURL)
133 | }
134 | nextPageVars := fmt.Sprintf(nextPagePayload, actualUserId, page.Graphql.User.Media.PageInfo.EndCursor)
135 | e.Request.Ctx.Put("variables", nextPageVars)
136 | if page.Graphql.User.Media.PageInfo.NextPage {
137 | u := fmt.Sprintf(
138 | nextPageURL,
139 | requestID,
140 | url.QueryEscape(nextPageVars),
141 | )
142 | log.Println("Next page found", u)
143 | e.Request.Ctx.Put("gis", data.Rhxgis)
144 | e.Request.Visit(u)
145 | }
146 | })
147 |
148 | c.OnError(func(r *colly.Response, e error) {
149 | log.Println("error:", e, r.Request.URL, string(r.Body))
150 | })
151 |
152 | c.OnResponse(func(r *colly.Response) {
153 | if strings.Index(r.Headers.Get("Content-Type"), "image") > -1 {
154 | r.Save(outputDir + r.FileName())
155 | return
156 | }
157 |
158 | if strings.Index(r.Headers.Get("Content-Type"), "json") == -1 {
159 | return
160 | }
161 |
162 | data := &nextPageData{}
163 | err := json.Unmarshal(r.Body, data)
164 | if err != nil {
165 | log.Fatal(err)
166 | }
167 |
168 | for _, obj := range data.Data.User.Container.Edges {
169 | // skip videos
170 | if obj.Node.IsVideo {
171 | continue
172 | }
173 | c.Visit(obj.Node.ImageURL)
174 | }
175 | if data.Data.User.Container.PageInfo.NextPage {
176 | nextPageVars := fmt.Sprintf(nextPagePayload, actualUserId, data.Data.User.Container.PageInfo.EndCursor)
177 | r.Request.Ctx.Put("variables", nextPageVars)
178 | u := fmt.Sprintf(
179 | nextPageURL,
180 | requestID,
181 | url.QueryEscape(nextPageVars),
182 | )
183 | log.Println("Next page found", u)
184 | r.Request.Visit(u)
185 | }
186 | })
187 |
188 | c.Visit("https://instagram.com/" + instagramAccount)
189 | }
190 |
--------------------------------------------------------------------------------
/_examples/local_files/html/child_page/one.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Document
8 |
9 |
10 | Child Page One
11 |
12 |
--------------------------------------------------------------------------------
/_examples/local_files/html/child_page/three.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Document
8 |
9 |
10 | Child Page Three
11 |
12 |
--------------------------------------------------------------------------------
/_examples/local_files/html/child_page/two.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Document
8 |
9 |
10 | Child Page Two
11 |
12 |
--------------------------------------------------------------------------------
/_examples/local_files/html/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Document
8 |
9 |
10 | Index.html
11 |
16 |
17 |
--------------------------------------------------------------------------------
/_examples/local_files/local_files.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net/http"
6 | "os"
7 | "path/filepath"
8 |
9 | "github.com/gocolly/colly/v2"
10 | )
11 |
12 | func main() {
13 | dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
14 | if err != nil {
15 | panic(err)
16 | }
17 |
18 | t := &http.Transport{}
19 | t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
20 |
21 | c := colly.NewCollector()
22 | c.WithTransport(t)
23 |
24 | pages := []string{}
25 |
26 | c.OnHTML("h1", func(e *colly.HTMLElement) {
27 | pages = append(pages, e.Text)
28 | })
29 |
30 | c.OnHTML("a", func(e *colly.HTMLElement) {
31 | c.Visit("file://" + dir + "/html" + e.Attr("href"))
32 | })
33 |
34 | fmt.Println("file://" + dir + "/html/index.html")
35 | c.Visit("file://" + dir + "/html/index.html")
36 | c.Wait()
37 | for i, p := range pages {
38 | fmt.Printf("%d : %s\n", i, p)
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/_examples/login/login.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 |
6 | "github.com/gocolly/colly/v2"
7 | )
8 |
9 | func main() {
10 | // create a new collector
11 | c := colly.NewCollector()
12 |
13 | // authenticate
14 | err := c.Post("http://example.com/login", map[string]string{"username": "admin", "password": "admin"})
15 | if err != nil {
16 | log.Fatal(err)
17 | }
18 |
19 | // attach callbacks after login
20 | c.OnResponse(func(r *colly.Response) {
21 | log.Println("response received", r.StatusCode)
22 | })
23 |
24 | // start scraping
25 | c.Visit("https://example.com/")
26 | }
27 |
--------------------------------------------------------------------------------
/_examples/max_depth/max_depth.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | )
8 |
9 | func main() {
10 | // Instantiate default collector
11 | c := colly.NewCollector(
12 | // MaxDepth is 1, so only the links on the scraped page
13 | // is visited, and no further links are followed
14 | colly.MaxDepth(1),
15 | )
16 |
17 | // On every a element which has href attribute call callback
18 | c.OnHTML("a[href]", func(e *colly.HTMLElement) {
19 | link := e.Attr("href")
20 | // Print link
21 | fmt.Println(link)
22 | // Visit link found on page
23 | e.Request.Visit(link)
24 | })
25 |
26 | // Start scraping on https://en.wikipedia.org
27 | c.Visit("https://en.wikipedia.org/")
28 | }
29 |
--------------------------------------------------------------------------------
/_examples/multipart/asciimoo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gocolly/colly/3a490c99cf2a7493271f151949590baae6a72538/_examples/multipart/asciimoo.jpg
--------------------------------------------------------------------------------
/_examples/multipart/multipart.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "net/http"
7 | "os"
8 | "time"
9 |
10 | "github.com/gocolly/colly/v2"
11 | )
12 |
13 | func generateFormData() map[string][]byte {
14 | f, _ := os.Open("gocolly.jpg")
15 | defer f.Close()
16 |
17 | imgData, _ := io.ReadAll(f)
18 |
19 | return map[string][]byte{
20 | "firstname": []byte("one"),
21 | "lastname": []byte("two"),
22 | "email": []byte("onetwo@example.com"),
23 | "file": imgData,
24 | }
25 | }
26 |
27 | func setupServer() {
28 | var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
29 | fmt.Println("received request")
30 | err := r.ParseMultipartForm(10000000)
31 | if err != nil {
32 | fmt.Println("server: Error")
33 | w.WriteHeader(500)
34 | w.Write([]byte("Internal Server Error"))
35 | return
36 | }
37 | w.WriteHeader(200)
38 | fmt.Println("server: OK")
39 | w.Write([]byte("Success"))
40 | }
41 |
42 | go http.ListenAndServe(":8080", handler)
43 | }
44 |
45 | func main() {
46 | // Start a single route http server to post an image to.
47 | setupServer()
48 |
49 | c := colly.NewCollector(colly.AllowURLRevisit(), colly.MaxDepth(5))
50 |
51 | // On every a element which has href attribute call callback
52 | c.OnHTML("html", func(e *colly.HTMLElement) {
53 | fmt.Println(e.Text)
54 | time.Sleep(1 * time.Second)
55 | e.Request.PostMultipart("http://localhost:8080/", generateFormData())
56 | })
57 |
58 | // Before making a request print "Visiting ..."
59 | c.OnRequest(func(r *colly.Request) {
60 | fmt.Println("Posting gocolly.jpg to", r.URL.String())
61 | })
62 |
63 | // Start scraping
64 | c.PostMultipart("http://localhost:8080/", generateFormData())
65 | c.Wait()
66 | }
67 |
--------------------------------------------------------------------------------
/_examples/openedx_courses/openedx_courses.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "strings"
7 | "time"
8 |
9 | "github.com/gocolly/colly/v2"
10 | )
11 |
12 | // DATE_FORMAT default format date used in openedx
13 | const DATE_FORMAT = "02 Jan, 2006"
14 |
15 | // Course store openedx course data
16 | type Course struct {
17 | CourseID string
18 | Run string
19 | Name string
20 | Number string
21 | StartDate *time.Time
22 | EndDate *time.Time
23 | URL string
24 | }
25 |
26 | func main() {
27 | // Instantiate default collector
28 | c := colly.NewCollector(
29 | // Using IndonesiaX as sample
30 | colly.AllowedDomains("indonesiax.co.id", "www.indonesiax.co.id"),
31 |
32 | // Cache responses to prevent multiple download of pages
33 | // even if the collector is restarted
34 | colly.CacheDir("./cache"),
35 | )
36 |
37 | courses := make([]Course, 0, 200)
38 |
39 | // On every a element which has href attribute call callback
40 | c.OnHTML("a[href]", func(e *colly.HTMLElement) {
41 | link := e.Attr("href")
42 | if !strings.HasPrefix(link, "/courses/") {
43 | return
44 | }
45 | // start scraping the page under the link found
46 | e.Request.Visit(link)
47 | })
48 |
49 | c.OnHTML("div[class=main-container]", func(e *colly.HTMLElement) {
50 | if e.DOM.Find("section#course-info").Length() == 0 {
51 | return
52 | }
53 | title := strings.Split(e.ChildText(".course-info__title"), "\n")[0]
54 | course_id := e.ChildAttr("input[name=course_id]", "value")
55 | texts := e.ChildTexts("span[data-datetime]")
56 | start_date, _ := time.Parse(DATE_FORMAT, texts[0])
57 | end_date, _ := time.Parse(DATE_FORMAT, texts[1])
58 | var run string
59 | if len(strings.Split(course_id, "_")) > 1 {
60 | run = strings.Split(course_id, "_")[1]
61 | }
62 | course := Course{
63 | CourseID: course_id,
64 | Run: run,
65 | Name: title,
66 | Number: e.ChildText("span.course-number"),
67 | StartDate: &start_date,
68 | EndDate: &end_date,
69 | URL: fmt.Sprintf("/courses/%s/about", course_id),
70 | }
71 | courses = append(courses, course)
72 | })
73 |
74 | // Start scraping on https://openedxdomain/courses
75 | c.Visit("https://www.indonesiax.co.id/courses")
76 |
77 | // Convert results to JSON data if the scraping job has finished
78 | jsonData, err := json.MarshalIndent(courses, "", " ")
79 | if err != nil {
80 | panic(err)
81 | }
82 |
83 | // Dump json to the standard output (can be redirected to a file)
84 | fmt.Println(string(jsonData))
85 | }
86 |
--------------------------------------------------------------------------------
/_examples/parallel/parallel.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | )
8 |
9 | func main() {
10 | // Instantiate default collector
11 | c := colly.NewCollector(
12 | // MaxDepth is 2, so only the links on the scraped page
13 | // and links on those pages are visited
14 | colly.MaxDepth(2),
15 | colly.Async(),
16 | )
17 |
18 | // Limit the maximum parallelism to 2
19 | // This is necessary if the goroutines are dynamically
20 | // created to control the limit of simultaneous requests.
21 | //
22 | // Parallelism can be controlled also by spawning fixed
23 | // number of go routines.
24 | c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 2})
25 |
26 | // On every a element which has href attribute call callback
27 | c.OnHTML("a[href]", func(e *colly.HTMLElement) {
28 | link := e.Attr("href")
29 | // Print link
30 | fmt.Println(link)
31 | // Visit link found on page on a new thread
32 | e.Request.Visit(link)
33 | })
34 |
35 | // Start scraping on https://en.wikipedia.org
36 | c.Visit("https://en.wikipedia.org/")
37 | // Wait until threads are finished
38 | c.Wait()
39 | }
40 |
--------------------------------------------------------------------------------
/_examples/proxy_switcher/proxy_switcher.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "log"
6 |
7 | "github.com/gocolly/colly/v2"
8 | "github.com/gocolly/colly/v2/proxy"
9 | )
10 |
11 | func main() {
12 | // Instantiate default collector
13 | c := colly.NewCollector(colly.AllowURLRevisit())
14 |
15 | // Rotate two socks5 proxies
16 | rp, err := proxy.RoundRobinProxySwitcher("socks5://127.0.0.1:1337", "socks5://127.0.0.1:1338")
17 | if err != nil {
18 | log.Fatal(err)
19 | }
20 | c.SetProxyFunc(rp)
21 |
22 | // Print the response
23 | c.OnResponse(func(r *colly.Response) {
24 | log.Printf("Proxy Address: %s\n", r.Request.ProxyURL)
25 | log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
26 | })
27 |
28 | // Fetch httpbin.org/ip five times
29 | for i := 0; i < 5; i++ {
30 | c.Visit("https://httpbin.org/ip")
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/_examples/queue/queue.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | "github.com/gocolly/colly/v2/queue"
8 | )
9 |
10 | func main() {
11 | url := "https://httpbin.org/delay/1"
12 |
13 | // Instantiate default collector
14 | c := colly.NewCollector(colly.AllowURLRevisit())
15 |
16 | // create a request queue with 2 consumer threads
17 | q, _ := queue.New(
18 | 2, // Number of consumer threads
19 | &queue.InMemoryQueueStorage{MaxSize: 10000}, // Use default queue storage
20 | )
21 |
22 | c.OnRequest(func(r *colly.Request) {
23 | fmt.Println("visiting", r.URL)
24 | if r.ID < 15 {
25 | r2, err := r.New("GET", fmt.Sprintf("%s?x=%v", url, r.ID), nil)
26 | if err == nil {
27 | q.AddRequest(r2)
28 | }
29 | }
30 | })
31 |
32 | for i := 0; i < 5; i++ {
33 | // Add URLs to the queue
34 | q.AddURL(fmt.Sprintf("%s?n=%d", url, i))
35 | }
36 | // Consume URLs
37 | q.Run(c)
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/_examples/random_delay/random_delay.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "time"
6 |
7 | "github.com/gocolly/colly/v2"
8 | "github.com/gocolly/colly/v2/debug"
9 | )
10 |
11 | func main() {
12 | url := "https://httpbin.org/delay/2"
13 |
14 | // Instantiate default collector
15 | c := colly.NewCollector(
16 | // Attach a debugger to the collector
17 | colly.Debugger(&debug.LogDebugger{}),
18 | colly.Async(),
19 | )
20 |
21 | // Limit the number of threads started by colly to two
22 | // when visiting links which domains' matches "*httpbin.*" glob
23 | c.Limit(&colly.LimitRule{
24 | DomainGlob: "*httpbin.*",
25 | Parallelism: 2,
26 | RandomDelay: 5 * time.Second,
27 | })
28 |
29 | // Start scraping in four threads on https://httpbin.org/delay/2
30 | for i := 0; i < 4; i++ {
31 | c.Visit(fmt.Sprintf("%s?n=%d", url, i))
32 | }
33 | // Start scraping on https://httpbin.org/delay/2
34 | c.Visit(url)
35 | // Wait until threads are finished
36 | c.Wait()
37 | }
38 |
--------------------------------------------------------------------------------
/_examples/rate_limit/rate_limit.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | "github.com/gocolly/colly/v2/debug"
8 | )
9 |
10 | func main() {
11 | url := "https://httpbin.org/delay/2"
12 |
13 | // Instantiate default collector
14 | c := colly.NewCollector(
15 | // Turn on asynchronous requests
16 | colly.Async(),
17 | // Attach a debugger to the collector
18 | colly.Debugger(&debug.LogDebugger{}),
19 | )
20 |
21 | // Limit the number of threads started by colly to two
22 | // when visiting links which domains' matches "*httpbin.*" glob
23 | c.Limit(&colly.LimitRule{
24 | DomainGlob: "*httpbin.*",
25 | Parallelism: 2,
26 | //Delay: 5 * time.Second,
27 | })
28 |
29 | // Start scraping in five threads on https://httpbin.org/delay/2
30 | for i := 0; i < 5; i++ {
31 | c.Visit(fmt.Sprintf("%s?n=%d", url, i))
32 | }
33 | // Wait until threads are finished
34 | c.Wait()
35 | }
36 |
--------------------------------------------------------------------------------
/_examples/reddit/reddit.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "time"
7 |
8 | "github.com/gocolly/colly/v2"
9 | )
10 |
11 | type item struct {
12 | StoryURL string
13 | Source string
14 | comments string
15 | CrawledAt time.Time
16 | Comments string
17 | Title string
18 | }
19 |
20 | func main() {
21 | stories := []item{}
22 | // Instantiate default collector
23 | c := colly.NewCollector(
24 | // Visit only domains: old.reddit.com
25 | colly.AllowedDomains("old.reddit.com"),
26 | // Parallelism
27 | colly.Async(true),
28 | )
29 |
30 | // On every a element which has .top-matter attribute call callback
31 | // This class is unique to the div that holds all information about a story
32 | c.OnHTML(".top-matter", func(e *colly.HTMLElement) {
33 | temp := item{}
34 | temp.StoryURL = e.ChildAttr("a[data-event-action=title]", "href")
35 | temp.Source = "https://old.reddit.com/r/programming/"
36 | temp.Title = e.ChildText("a[data-event-action=title]")
37 | temp.Comments = e.ChildAttr("a[data-event-action=comments]", "href")
38 | temp.CrawledAt = time.Now()
39 | stories = append(stories, temp)
40 | })
41 |
42 | // On every span tag with the class next-button
43 | c.OnHTML("span.next-button", func(h *colly.HTMLElement) {
44 | t := h.ChildAttr("a", "href")
45 | c.Visit(t)
46 | })
47 |
48 | // Set max Parallelism and introduce a Random Delay
49 | c.Limit(&colly.LimitRule{
50 | Parallelism: 2,
51 | RandomDelay: 5 * time.Second,
52 | })
53 |
54 | // Before making a request print "Visiting ..."
55 | c.OnRequest(func(r *colly.Request) {
56 | fmt.Println("Visiting", r.URL.String())
57 |
58 | })
59 |
60 | // Crawl all reddits the user passes in
61 | reddits := os.Args[1:]
62 | for _, reddit := range reddits {
63 | c.Visit(reddit)
64 |
65 | }
66 |
67 | c.Wait()
68 | fmt.Println(stories)
69 |
70 | }
71 |
--------------------------------------------------------------------------------
/_examples/request_context/request_context.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | )
8 |
9 | func main() {
10 | // Instantiate default collector
11 | c := colly.NewCollector()
12 |
13 | // Before making a request put the URL with
14 | // the key of "url" into the context of the request
15 | c.OnRequest(func(r *colly.Request) {
16 | r.Ctx.Put("url", r.URL.String())
17 | })
18 |
19 | // After making a request get "url" from
20 | // the context of the request
21 | c.OnResponse(func(r *colly.Response) {
22 | fmt.Println(r.Ctx.Get("url"))
23 | })
24 |
25 | // Start scraping on https://en.wikipedia.org
26 | c.Visit("https://en.wikipedia.org/")
27 | }
28 |
--------------------------------------------------------------------------------
/_examples/scraper_server/scraper_server.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "log"
6 | "net/http"
7 |
8 | "github.com/gocolly/colly/v2"
9 | )
10 |
11 | type pageInfo struct {
12 | StatusCode int
13 | Links map[string]int
14 | }
15 |
16 | func handler(w http.ResponseWriter, r *http.Request) {
17 | URL := r.URL.Query().Get("url")
18 | if URL == "" {
19 | log.Println("missing URL argument")
20 | return
21 | }
22 | log.Println("visiting", URL)
23 |
24 | c := colly.NewCollector()
25 |
26 | p := &pageInfo{Links: make(map[string]int)}
27 |
28 | // count links
29 | c.OnHTML("a[href]", func(e *colly.HTMLElement) {
30 | link := e.Request.AbsoluteURL(e.Attr("href"))
31 | if link != "" {
32 | p.Links[link]++
33 | }
34 | })
35 |
36 | // extract status code
37 | c.OnResponse(func(r *colly.Response) {
38 | log.Println("response received", r.StatusCode)
39 | p.StatusCode = r.StatusCode
40 | })
41 | c.OnError(func(r *colly.Response, err error) {
42 | log.Println("error:", r.StatusCode, err)
43 | p.StatusCode = r.StatusCode
44 | })
45 |
46 | c.Visit(URL)
47 |
48 | // dump results
49 | b, err := json.Marshal(p)
50 | if err != nil {
51 | log.Println("failed to serialize response:", err)
52 | return
53 | }
54 | w.Header().Add("Content-Type", "application/json")
55 | w.Write(b)
56 | }
57 |
58 | func main() {
59 | // example usage: curl -s 'http://127.0.0.1:7171/?url=http://go-colly.org/'
60 | addr := ":7171"
61 |
62 | http.HandleFunc("/", handler)
63 |
64 | log.Println("listening on", addr)
65 | log.Fatal(http.ListenAndServe(addr, nil))
66 | }
67 |
--------------------------------------------------------------------------------
/_examples/shopify_sitemap/shopify_sitemap.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/gocolly/colly/v2"
7 | )
8 |
9 | func main() {
10 | // Array containing all the known URLs in a sitemap
11 | knownUrls := []string{}
12 |
13 | // Create a Collector specifically for Shopify
14 | c := colly.NewCollector(colly.AllowedDomains("www.shopify.com"))
15 |
16 | // Create a callback on the XPath query searching for the URLs
17 | c.OnXML("//urlset/url/loc", func(e *colly.XMLElement) {
18 | knownUrls = append(knownUrls, e.Text)
19 | })
20 |
21 | // Start the collector
22 | c.Visit("https://www.shopify.com/sitemap.xml")
23 |
24 | fmt.Println("All known URLs:")
25 | for _, url := range knownUrls {
26 | fmt.Println("\t", url)
27 | }
28 | fmt.Println("Collected", len(knownUrls), "URLs")
29 | }
30 |
--------------------------------------------------------------------------------
/_examples/url_filter/url_filter.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 |
7 | "github.com/gocolly/colly/v2"
8 | )
9 |
10 | func main() {
11 | // Instantiate default collector
12 | c := colly.NewCollector(
13 | // Visit only root url and urls which start with "e" or "h" on httpbin.org
14 | colly.URLFilters(
15 | regexp.MustCompile("http://httpbin\\.org/(|e.+)$"),
16 | regexp.MustCompile("http://httpbin\\.org/h.+"),
17 | ),
18 | )
19 |
20 | // On every a element which has href attribute call callback
21 | c.OnHTML("a[href]", func(e *colly.HTMLElement) {
22 | link := e.Attr("href")
23 | // Print link
24 | fmt.Printf("Link found: %q -> %s\n", e.Text, link)
25 | // Visit link found on page
26 | // Only those links are visited which are matched by any of the URLFilter regexps
27 | c.Visit(e.Request.AbsoluteURL(link))
28 | })
29 |
30 | // Before making a request print "Visiting ..."
31 | c.OnRequest(func(r *colly.Request) {
32 | fmt.Println("Visiting", r.URL.String())
33 | })
34 |
35 | // Start scraping on http://httpbin.org
36 | c.Visit("http://httpbin.org/")
37 | }
38 |
--------------------------------------------------------------------------------
/_examples/xkcd_store/xkcd_store.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/csv"
5 | "log"
6 | "os"
7 |
8 | "github.com/gocolly/colly/v2"
9 | )
10 |
11 | func main() {
12 | fName := "xkcd_store_items.csv"
13 | file, err := os.Create(fName)
14 | if err != nil {
15 | log.Fatalf("Cannot create file %q: %s\n", fName, err)
16 | return
17 | }
18 | defer file.Close()
19 | writer := csv.NewWriter(file)
20 | defer writer.Flush()
21 | // Write CSV header
22 | writer.Write([]string{"Name", "Price", "URL", "Image URL"})
23 |
24 | // Instantiate default collector
25 | c := colly.NewCollector(
26 | // Allow requests only to store.xkcd.com
27 | colly.AllowedDomains("store.xkcd.com"),
28 | )
29 |
30 | // Extract product details
31 | c.OnHTML(".product-grid-item", func(e *colly.HTMLElement) {
32 | writer.Write([]string{
33 | e.ChildAttr("a", "title"),
34 | e.ChildText("span"),
35 | e.Request.AbsoluteURL(e.ChildAttr("a", "href")),
36 | "https:" + e.ChildAttr("img", "src"),
37 | })
38 | })
39 |
40 | // Find and visit next page links
41 | c.OnHTML(`.next a[href]`, func(e *colly.HTMLElement) {
42 | e.Request.Visit(e.Attr("href"))
43 | })
44 |
45 | c.Visit("https://store.xkcd.com/collections/everything")
46 |
47 | log.Printf("Scraping finished, check file %q for results\n", fName)
48 |
49 | // Display collector's statistics
50 | log.Println(c)
51 | }
52 |
--------------------------------------------------------------------------------
/assets/scrapfly.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gocolly/colly/3a490c99cf2a7493271f151949590baae6a72538/assets/scrapfly.png
--------------------------------------------------------------------------------
/cmd/colly/colly.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package main
16 |
17 | import (
18 | "bytes"
19 | "fmt"
20 | "log"
21 | "os"
22 | "strings"
23 |
24 | "github.com/jawher/mow.cli"
25 | )
26 |
27 | var scraperHeadTemplate = `package main
28 |
29 | import (
30 | "log"
31 |
32 | "github.com/gocolly/colly/v2"
33 | )
34 |
35 | func main() {
36 | c := colly.NewCollector()
37 | `
38 |
39 | var scraperEndTemplate = `
40 | c.Visit("https://yourdomain.com/")
41 | }
42 | `
43 |
44 | var htmlCallbackTemplate = `
45 | c.OnHTML("element-selector", func(e *colly.HTMLElement) {
46 | log.Println(e.Text)
47 | })
48 | `
49 |
50 | var requestCallbackTemplate = `
51 | c.OnRequest(func(r *colly.Request) {
52 | log.Println("Visiting", r.URL)
53 | })
54 | `
55 |
56 | var responseCallbackTemplate = `
57 | c.OnResponse(func(r *colly.Response) {
58 | log.Println("Visited", r.Request.URL, r.StatusCode)
59 | })
60 | `
61 |
62 | var errorCallbackTemplate = `
63 | c.OnError(func(r *colly.Response, err error) {
64 | log.Printf("Error on %s: %s", r.Request.URL, err)
65 | })
66 | `
67 |
68 | func main() {
69 | app := cli.App("colly", "Scraping Framework for Gophers")
70 |
71 | app.Command("new", "Create new scraper", func(cmd *cli.Cmd) {
72 | var (
73 | callbacks = cmd.StringOpt("callbacks", "", "Add callbacks to the template. (E.g. '--callbacks=html,response,error')")
74 | hosts = cmd.StringOpt("hosts", "", "Specify scraper's allowed hosts. (e.g. '--hosts=xy.com,abcd.com')")
75 | path = cmd.StringArg("PATH", "", "Path of the new scraper")
76 | )
77 |
78 | cmd.Spec = "[--callbacks] [--hosts] [PATH]"
79 |
80 | cmd.Action = func() {
81 | scraper := bytes.NewBufferString(scraperHeadTemplate)
82 | outfile := os.Stdout
83 | if *path != "" {
84 | var err error
85 | outfile, err = os.Create(*path)
86 | if err != nil {
87 | log.Fatal(err)
88 | }
89 | defer outfile.Close()
90 | }
91 | if *hosts != "" {
92 | scraper.WriteString("\n c.AllowedDomains = []string{")
93 | for i, h := range strings.Split(*hosts, ",") {
94 | if i > 0 {
95 | scraper.WriteString(", ")
96 | }
97 | scraper.WriteString(fmt.Sprintf("%q", h))
98 | }
99 | scraper.WriteString("}\n")
100 | }
101 | if len(*callbacks) > 0 {
102 | for _, c := range strings.Split(*callbacks, ",") {
103 | switch c {
104 | case "html":
105 | scraper.WriteString(htmlCallbackTemplate)
106 | case "request":
107 | scraper.WriteString(requestCallbackTemplate)
108 | case "response":
109 | scraper.WriteString(responseCallbackTemplate)
110 | case "error":
111 | scraper.WriteString(errorCallbackTemplate)
112 | }
113 | }
114 | }
115 | scraper.WriteString(scraperEndTemplate)
116 | outfile.Write(scraper.Bytes())
117 | }
118 | })
119 |
120 | app.Run(os.Args)
121 | }
122 |
--------------------------------------------------------------------------------
/context.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package colly
16 |
17 | import (
18 | "sync"
19 | )
20 |
21 | // Context provides a tiny layer for passing data between callbacks
22 | type Context struct {
23 | contextMap map[string]interface{}
24 | lock *sync.RWMutex
25 | }
26 |
27 | // NewContext initializes a new Context instance
28 | func NewContext() *Context {
29 | return &Context{
30 | contextMap: make(map[string]interface{}),
31 | lock: &sync.RWMutex{},
32 | }
33 | }
34 |
35 | // UnmarshalBinary decodes Context value to nil
36 | // This function is used by request caching
37 | func (c *Context) UnmarshalBinary(_ []byte) error {
38 | return nil
39 | }
40 |
41 | // MarshalBinary encodes Context value
42 | // This function is used by request caching
43 | func (c *Context) MarshalBinary() (_ []byte, _ error) {
44 | return nil, nil
45 | }
46 |
47 | // Put stores a value of any type in Context
48 | func (c *Context) Put(key string, value interface{}) {
49 | c.lock.Lock()
50 | c.contextMap[key] = value
51 | c.lock.Unlock()
52 | }
53 |
54 | // Get retrieves a string value from Context.
55 | // Get returns an empty string if key not found
56 | func (c *Context) Get(key string) string {
57 | c.lock.RLock()
58 | defer c.lock.RUnlock()
59 | if v, ok := c.contextMap[key]; ok {
60 | return v.(string)
61 | }
62 | return ""
63 | }
64 |
65 | // GetAny retrieves a value from Context.
66 | // GetAny returns nil if key not found
67 | func (c *Context) GetAny(key string) interface{} {
68 | c.lock.RLock()
69 | defer c.lock.RUnlock()
70 | if v, ok := c.contextMap[key]; ok {
71 | return v
72 | }
73 | return nil
74 | }
75 |
76 | // ForEach iterate context
77 | func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{} {
78 | c.lock.RLock()
79 | defer c.lock.RUnlock()
80 |
81 | ret := make([]interface{}, 0, len(c.contextMap))
82 | for k, v := range c.contextMap {
83 | ret = append(ret, fn(k, v))
84 | }
85 |
86 | return ret
87 | }
88 |
--------------------------------------------------------------------------------
/context_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package colly
16 |
17 | import (
18 | "strconv"
19 | "testing"
20 | )
21 |
22 | func TestContextIteration(t *testing.T) {
23 | ctx := NewContext()
24 | for i := 0; i < 10; i++ {
25 | ctx.Put(strconv.Itoa(i), i)
26 | }
27 | values := ctx.ForEach(func(k string, v interface{}) interface{} {
28 | return v.(int)
29 | })
30 | if len(values) != 10 {
31 | t.Fatal("fail to iterate context")
32 | }
33 | for _, i := range values {
34 | v := i.(int)
35 | if v != ctx.GetAny(strconv.Itoa(v)).(int) {
36 | t.Fatal("value not equal")
37 | }
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/debug/debug.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package debug
16 |
17 | // Event represents an action inside a collector
18 | type Event struct {
19 | // Type is the type of the event
20 | Type string
21 | // RequestID identifies the HTTP request of the Event
22 | RequestID uint32
23 | // CollectorID identifies the collector of the Event
24 | CollectorID uint32
25 | // Values contains the event's key-value pairs. Different type of events
26 | // can return different key-value pairs
27 | Values map[string]string
28 | }
29 |
30 | // Debugger is an interface for different type of debugging backends
31 | type Debugger interface {
32 | // Init initializes the backend
33 | Init() error
34 | // Event receives a new collector event.
35 | Event(e *Event)
36 | }
37 |
--------------------------------------------------------------------------------
/debug/logdebugger.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package debug
16 |
17 | import (
18 | "io"
19 | "log"
20 | "os"
21 | "sync/atomic"
22 | "time"
23 | )
24 |
25 | // LogDebugger is the simplest debugger which prints log messages to the STDERR
26 | type LogDebugger struct {
27 | // Output is the log destination, anything can be used which implements them
28 | // io.Writer interface. Leave it blank to use STDERR
29 | Output io.Writer
30 | // Prefix appears at the beginning of each generated log line
31 | Prefix string
32 | // Flag defines the logging properties.
33 | Flag int
34 | logger *log.Logger
35 | counter int32
36 | start time.Time
37 | }
38 |
39 | // Init initializes the LogDebugger
40 | func (l *LogDebugger) Init() error {
41 | l.counter = 0
42 | l.start = time.Now()
43 | if l.Output == nil {
44 | l.Output = os.Stderr
45 | }
46 | l.logger = log.New(l.Output, l.Prefix, l.Flag)
47 | return nil
48 | }
49 |
50 | // Event receives Collector events and prints them to STDERR
51 | func (l *LogDebugger) Event(e *Event) {
52 | i := atomic.AddInt32(&l.counter, 1)
53 | l.logger.Printf("[%06d] %d [%6d - %s] %q (%s)\n", i, e.CollectorID, e.RequestID, e.Type, e.Values, time.Since(l.start))
54 | }
55 |
--------------------------------------------------------------------------------
/debug/webdebugger.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package debug
16 |
17 | import (
18 | "encoding/json"
19 | "log"
20 | "net/http"
21 | "sync"
22 | "time"
23 | )
24 |
25 | // WebDebugger is a web based debuging frontend for colly
26 | type WebDebugger struct {
27 | // Address is the address of the web server. It is 127.0.0.1:7676 by default.
28 | Address string
29 | initialized bool
30 | CurrentRequests map[uint32]requestInfo
31 | RequestLog []requestInfo
32 | sync.Mutex
33 | }
34 |
35 | type requestInfo struct {
36 | URL string
37 | Started time.Time
38 | Duration time.Duration
39 | ResponseStatus string
40 | ID uint32
41 | CollectorID uint32
42 | }
43 |
44 | // Init initializes the WebDebugger
45 | func (w *WebDebugger) Init() error {
46 | if w.initialized {
47 | return nil
48 | }
49 | defer func() {
50 | w.initialized = true
51 | }()
52 | if w.Address == "" {
53 | w.Address = "127.0.0.1:7676"
54 | }
55 | w.RequestLog = make([]requestInfo, 0)
56 | w.CurrentRequests = make(map[uint32]requestInfo)
57 | http.HandleFunc("/", w.indexHandler)
58 | http.HandleFunc("/status", w.statusHandler)
59 | log.Println("Starting debug webserver on", w.Address)
60 | go http.ListenAndServe(w.Address, nil)
61 | return nil
62 | }
63 |
64 | // Event updates the debugger's status
65 | func (w *WebDebugger) Event(e *Event) {
66 | w.Lock()
67 | defer w.Unlock()
68 |
69 | switch e.Type {
70 | case "request":
71 | w.CurrentRequests[e.RequestID] = requestInfo{
72 | URL: e.Values["url"],
73 | Started: time.Now(),
74 | ID: e.RequestID,
75 | CollectorID: e.CollectorID,
76 | }
77 | case "response", "error":
78 | r := w.CurrentRequests[e.RequestID]
79 | r.Duration = time.Since(r.Started)
80 | r.ResponseStatus = e.Values["status"]
81 | w.RequestLog = append(w.RequestLog, r)
82 | delete(w.CurrentRequests, e.RequestID)
83 | }
84 | }
85 |
86 | func (w *WebDebugger) indexHandler(wr http.ResponseWriter, r *http.Request) {
87 | wr.Write([]byte(`
88 |
89 |
90 | Colly Debugger WebUI
91 |
92 |
93 |
94 |
95 |
100 |
101 |
102 |
103 |
Current Requests
104 |
105 |
106 |
107 |
Finished Requests
108 |
109 |
110 |
111 |
- Information: Info 1
- Information: Info 2
`)
27 |
28 | func TestBasicUnmarshal(t *testing.T) {
29 | doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(basicTestData))
30 | e := &HTMLElement{
31 | DOM: doc.First(),
32 | }
33 | s := struct {
34 | String string `selector:"li:first-child" attr:"class"`
35 | Items []string `selector:"li"`
36 | Struct struct {
37 | String string `selector:"li:last-child"`
38 | }
39 | }{}
40 | if err := e.Unmarshal(&s); err != nil {
41 | t.Error("Cannot unmarshal struct: " + err.Error())
42 | }
43 | if s.String != "x" {
44 | t.Errorf(`Invalid data for String: %q, expected "x"`, s.String)
45 | }
46 | if s.Struct.String != "3" {
47 | t.Errorf(`Invalid data for Struct.String: %q, expected "3"`, s.Struct.String)
48 | }
49 | }
50 |
51 | func TestNestedUnmarshalMap(t *testing.T) {
52 | doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(nestedTestData))
53 | e := &HTMLElement{
54 | DOM: doc.First(),
55 | }
56 | doc2, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(basicTestData))
57 | e2 := &HTMLElement{
58 | DOM: doc2.First(),
59 | }
60 | type nested struct {
61 | String string
62 | }
63 | mapSelector := make(map[string]string)
64 | mapSelector["String"] = "div > p"
65 |
66 | mapSelector2 := make(map[string]string)
67 | mapSelector2["String"] = "span"
68 |
69 | s := nested{}
70 | s2 := nested{}
71 | if err := e.UnmarshalWithMap(&s, mapSelector); err != nil {
72 | t.Error("Cannot unmarshal struct: " + err.Error())
73 | }
74 | if err := e2.UnmarshalWithMap(&s2, mapSelector2); err != nil {
75 | t.Error("Cannot unmarshal struct: " + err.Error())
76 | }
77 | if s.String != "a" {
78 | t.Errorf(`Invalid data for String: %q, expected "a"`, s.String)
79 | }
80 | if s2.String != "item" {
81 | t.Errorf(`Invalid data for String: %q, expected "a"`, s.String)
82 | }
83 | }
84 |
85 | func TestNestedUnmarshal(t *testing.T) {
86 | doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(nestedTestData))
87 | e := &HTMLElement{
88 | DOM: doc.First(),
89 | }
90 | type nested struct {
91 | String string `selector:"div > p"`
92 | Struct *nested `selector:"div > div"`
93 | }
94 | s := nested{}
95 | if err := e.Unmarshal(&s); err != nil {
96 | t.Error("Cannot unmarshal struct: " + err.Error())
97 | }
98 | if s.String != "a" {
99 | t.Errorf(`Invalid data for String: %q, expected "a"`, s.String)
100 | }
101 | if s.Struct.String != "b" {
102 | t.Errorf(`Invalid data for Struct.String: %q, expected "b"`, s.Struct.String)
103 | }
104 | if s.Struct.Struct.String != "c" {
105 | t.Errorf(`Invalid data for Struct.Struct.String: %q, expected "c"`, s.Struct.Struct.String)
106 | }
107 | }
108 |
109 | func TestPointerSliceUnmarshall(t *testing.T) {
110 | type info struct {
111 | Text string `selector:"span"`
112 | }
113 | type object struct {
114 | Info []*info `selector:"li.info"`
115 | }
116 |
117 | doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(pointerSliceTestData))
118 | e := HTMLElement{DOM: doc.First()}
119 | o := object{}
120 | err := e.Unmarshal(&o)
121 | if err != nil {
122 | t.Fatalf("Failed to unmarshal page: %s\n", err.Error())
123 | }
124 |
125 | if len(o.Info) != 2 {
126 | t.Errorf("Invalid length for Info: %d, expected 2", len(o.Info))
127 | }
128 | if o.Info[0].Text != "Info 1" {
129 | t.Errorf("Invalid data for Info.[0].Text: %s, expected Info 1", o.Info[0].Text)
130 | }
131 | if o.Info[1].Text != "Info 2" {
132 | t.Errorf("Invalid data for Info.[1].Text: %s, expected Info 2", o.Info[1].Text)
133 | }
134 |
135 | }
136 |
137 | func TestStructSliceUnmarshall(t *testing.T) {
138 | type info struct {
139 | Text string `selector:"span"`
140 | }
141 | type object struct {
142 | Info []info `selector:"li.info"`
143 | }
144 |
145 | doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(pointerSliceTestData))
146 | e := HTMLElement{DOM: doc.First()}
147 | o := object{}
148 | err := e.Unmarshal(&o)
149 | if err != nil {
150 | t.Fatalf("Failed to unmarshal page: %s\n", err.Error())
151 | }
152 |
153 | if len(o.Info) != 2 {
154 | t.Errorf("Invalid length for Info: %d, expected 2", len(o.Info))
155 | }
156 | if o.Info[0].Text != "Info 1" {
157 | t.Errorf("Invalid data for Info.[0].Text: %s, expected Info 1", o.Info[0].Text)
158 | }
159 | if o.Info[1].Text != "Info 2" {
160 | t.Errorf("Invalid data for Info.[1].Text: %s, expected Info 2", o.Info[1].Text)
161 | }
162 |
163 | }
164 |
--------------------------------------------------------------------------------
/xmlelement.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package colly
16 |
17 | import (
18 | "strings"
19 |
20 | "github.com/antchfx/htmlquery"
21 | "github.com/antchfx/xmlquery"
22 | "golang.org/x/net/html"
23 | )
24 |
25 | // XMLElement is the representation of a XML tag.
26 | type XMLElement struct {
27 | // Name is the name of the tag
28 | Name string
29 | Text string
30 | attributes interface{}
31 | // Request is the request object of the element's HTML document
32 | Request *Request
33 | // Response is the Response object of the element's HTML document
34 | Response *Response
35 | // DOM is the DOM object of the page. DOM is relative
36 | // to the current XMLElement and is either a html.Node or xmlquery.Node
37 | // based on how the XMLElement was created.
38 | DOM interface{}
39 | isHTML bool
40 | }
41 |
42 | // NewXMLElementFromHTMLNode creates a XMLElement from a html.Node.
43 | func NewXMLElementFromHTMLNode(resp *Response, s *html.Node) *XMLElement {
44 | return &XMLElement{
45 | Name: s.Data,
46 | Request: resp.Request,
47 | Response: resp,
48 | Text: htmlquery.InnerText(s),
49 | DOM: s,
50 | attributes: s.Attr,
51 | isHTML: true,
52 | }
53 | }
54 |
55 | // NewXMLElementFromXMLNode creates a XMLElement from a xmlquery.Node.
56 | func NewXMLElementFromXMLNode(resp *Response, s *xmlquery.Node) *XMLElement {
57 | return &XMLElement{
58 | Name: s.Data,
59 | Request: resp.Request,
60 | Response: resp,
61 | Text: s.InnerText(),
62 | DOM: s,
63 | attributes: s.Attr,
64 | isHTML: false,
65 | }
66 | }
67 |
68 | // Attr returns the selected attribute of a HTMLElement or empty string
69 | // if no attribute found
70 | func (h *XMLElement) Attr(k string) string {
71 | if h.isHTML {
72 | for _, a := range h.attributes.([]html.Attribute) {
73 | if a.Key == k {
74 | return a.Val
75 | }
76 | }
77 | } else {
78 | for _, a := range h.attributes.([]xmlquery.Attr) {
79 | if a.Name.Local == k {
80 | return a.Value
81 | }
82 | }
83 | }
84 | return ""
85 | }
86 |
87 | // ChildText returns the concatenated and stripped text content of the matching
88 | // elements.
89 | func (h *XMLElement) ChildText(xpathQuery string) string {
90 | if h.isHTML {
91 | child := htmlquery.FindOne(h.DOM.(*html.Node), xpathQuery)
92 | if child == nil {
93 | return ""
94 | }
95 | return strings.TrimSpace(htmlquery.InnerText(child))
96 | }
97 | child := xmlquery.FindOne(h.DOM.(*xmlquery.Node), xpathQuery)
98 | if child == nil {
99 | return ""
100 | }
101 | return strings.TrimSpace(child.InnerText())
102 |
103 | }
104 |
105 | // ChildAttr returns the stripped text content of the first matching
106 | // element's attribute.
107 | func (h *XMLElement) ChildAttr(xpathQuery, attrName string) string {
108 | if h.isHTML {
109 | child := htmlquery.FindOne(h.DOM.(*html.Node), xpathQuery)
110 | if child != nil {
111 | for _, attr := range child.Attr {
112 | if attr.Key == attrName {
113 | return strings.TrimSpace(attr.Val)
114 | }
115 | }
116 | }
117 | } else {
118 | child := xmlquery.FindOne(h.DOM.(*xmlquery.Node), xpathQuery)
119 | if child != nil {
120 | for _, attr := range child.Attr {
121 | if attr.Name.Local == attrName {
122 | return strings.TrimSpace(attr.Value)
123 | }
124 | }
125 | }
126 | }
127 |
128 | return ""
129 | }
130 |
131 | // ChildAttrs returns the stripped text content of all the matching
132 | // element's attributes.
133 | func (h *XMLElement) ChildAttrs(xpathQuery, attrName string) []string {
134 | var res []string
135 | if h.isHTML {
136 | for _, child := range htmlquery.Find(h.DOM.(*html.Node), xpathQuery) {
137 | for _, attr := range child.Attr {
138 | if attr.Key == attrName {
139 | res = append(res, strings.TrimSpace(attr.Val))
140 | }
141 | }
142 | }
143 | } else {
144 | xmlquery.FindEach(h.DOM.(*xmlquery.Node), xpathQuery, func(i int, child *xmlquery.Node) {
145 | for _, attr := range child.Attr {
146 | if attr.Name.Local == attrName {
147 | res = append(res, strings.TrimSpace(attr.Value))
148 | }
149 | }
150 | })
151 | }
152 | return res
153 | }
154 |
155 | // ChildTexts returns an array of strings corresponding to child elements that match the xpath query.
156 | // Each item in the array is the stripped text content of the corresponding matching child element.
157 | func (h *XMLElement) ChildTexts(xpathQuery string) []string {
158 | texts := make([]string, 0)
159 | if h.isHTML {
160 | for _, child := range htmlquery.Find(h.DOM.(*html.Node), xpathQuery) {
161 | texts = append(texts, strings.TrimSpace(htmlquery.InnerText(child)))
162 | }
163 | } else {
164 | xmlquery.FindEach(h.DOM.(*xmlquery.Node), xpathQuery, func(i int, child *xmlquery.Node) {
165 | texts = append(texts, strings.TrimSpace(child.InnerText()))
166 | })
167 | }
168 | return texts
169 | }
170 |
--------------------------------------------------------------------------------
/xmlelement_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018 Adam Tauber
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package colly_test
16 |
17 | import (
18 | "github.com/antchfx/htmlquery"
19 | "github.com/gocolly/colly/v2"
20 | "reflect"
21 | "strings"
22 | "testing"
23 | )
24 |
25 | // Borrowed from http://infohost.nmt.edu/tcc/help/pubs/xhtml/example.html
26 | // Added attributes to the `` tags for testing purposes
27 | const htmlPage = `
28 |
30 |
31 |
32 | Your page title here
33 |
34 |
35 | Your major heading here
36 |
37 | This is a regular text paragraph.
38 |
39 |
40 | -
41 | First bullet of a bullet list.
42 |
43 | -
44 | This is the second bullet.
45 |
46 |
47 |
48 |
49 | `
50 |
51 | func TestAttr(t *testing.T) {
52 | resp := &colly.Response{StatusCode: 200, Body: []byte(htmlPage)}
53 | doc, _ := htmlquery.Parse(strings.NewReader(htmlPage))
54 | xmlNode := htmlquery.FindOne(doc, "/html")
55 | xmlElem := colly.NewXMLElementFromHTMLNode(resp, xmlNode)
56 |
57 | if xmlElem.Attr("xmlns") != "http://www.w3.org/1999/xhtml" {
58 | t.Fatalf("failed xmlns attribute test: %v != http://www.w3.org/1999/xhtml", xmlElem.Attr("xmlns"))
59 | }
60 |
61 | if xmlElem.Attr("xml:lang") != "en" {
62 | t.Fatalf("failed lang attribute test: %v != en", xmlElem.Attr("lang"))
63 | }
64 | }
65 |
66 | func TestChildText(t *testing.T) {
67 | resp := &colly.Response{StatusCode: 200, Body: []byte(htmlPage)}
68 | doc, _ := htmlquery.Parse(strings.NewReader(htmlPage))
69 | xmlNode := htmlquery.FindOne(doc, "/html")
70 | xmlElem := colly.NewXMLElementFromHTMLNode(resp, xmlNode)
71 |
72 | if text := xmlElem.ChildText("//p"); text != "This is a regular text paragraph." {
73 | t.Fatalf("failed child tag test: %v != This is a regular text paragraph.", text)
74 | }
75 | if text := xmlElem.ChildText("//dl"); text != "" {
76 | t.Fatalf("failed child tag test: %v != \"\"", text)
77 | }
78 | }
79 |
80 | func TestChildTexts(t *testing.T) {
81 | resp := &colly.Response{StatusCode: 200, Body: []byte(htmlPage)}
82 | doc, _ := htmlquery.Parse(strings.NewReader(htmlPage))
83 | xmlNode := htmlquery.FindOne(doc, "/html")
84 | xmlElem := colly.NewXMLElementFromHTMLNode(resp, xmlNode)
85 | expected := []string{"First bullet of a bullet list.", "This is the second bullet."}
86 | if texts := xmlElem.ChildTexts("//li"); reflect.DeepEqual(texts, expected) == false {
87 | t.Fatalf("failed child tags test: %v != %v", texts, expected)
88 | }
89 | if texts := xmlElem.ChildTexts("//dl"); reflect.DeepEqual(texts, make([]string, 0)) == false {
90 | t.Fatalf("failed child tag test: %v != \"\"", texts)
91 | }
92 | }
93 | func TestChildAttr(t *testing.T) {
94 | resp := &colly.Response{StatusCode: 200, Body: []byte(htmlPage)}
95 | doc, _ := htmlquery.Parse(strings.NewReader(htmlPage))
96 | xmlNode := htmlquery.FindOne(doc, "/html")
97 | xmlElem := colly.NewXMLElementFromHTMLNode(resp, xmlNode)
98 |
99 | if attr := xmlElem.ChildAttr("/body/ul/li[1]", "class"); attr != "list-item-1" {
100 | t.Fatalf("failed child attribute test: %v != list-item-1", attr)
101 | }
102 | if attr := xmlElem.ChildAttr("/body/ul/li[2]", "class"); attr != "list-item-2" {
103 | t.Fatalf("failed child attribute test: %v != list-item-2", attr)
104 | }
105 | }
106 |
107 | func TestChildAttrs(t *testing.T) {
108 | resp := &colly.Response{StatusCode: 200, Body: []byte(htmlPage)}
109 | doc, _ := htmlquery.Parse(strings.NewReader(htmlPage))
110 | xmlNode := htmlquery.FindOne(doc, "/html")
111 | xmlElem := colly.NewXMLElementFromHTMLNode(resp, xmlNode)
112 |
113 | attrs := xmlElem.ChildAttrs("/body/ul/li", "class")
114 | if len(attrs) != 2 {
115 | t.Fatalf("failed child attributes length test: %d != 2", len(attrs))
116 | }
117 |
118 | for _, attr := range attrs {
119 | if !(attr == "list-item-1" || attr == "list-item-2") {
120 | t.Fatalf("failed child attributes values test: %s != list-item-(1 or 2)", attr)
121 | }
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
50 |
51 |
52 | ### Sponsors
53 |
54 | Thank you to all our sponsors! (please ask your company to also support this open source project by [becoming a sponsor](https://opencollective.com/colly#sponsor))
55 |
56 | 
57 | 
58 | 
59 | 
60 | 
61 | 
62 | 
63 | 
64 | 
65 | 
66 |
67 |
68 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Colly
2 |
3 | Lightning Fast and Elegant Scraping Framework for Gophers
4 |
5 | Colly provides a clean interface to write any kind of crawler/scraper/spider.
6 |
7 | With Colly you can easily extract structured data from websites, which can be used for a wide range of applications, like data mining, data processing or archiving.
8 |
9 | [](https://pkg.go.dev/github.com/gocolly/colly/v2)
10 | [](#backers) [](#sponsors) [](https://github.com/gocolly/colly/actions/workflows/ci.yml)
11 | [](http://goreportcard.com/report/gocolly/colly)
12 | [](https://github.com/gocolly/colly/tree/master/_examples)
13 | [](https://codecov.io/github/gocolly/colly?branch=master)
14 | [](https://app.fossa.io/projects/git%2Bgithub.com%2Fgocolly%2Fcolly?ref=badge_shield)
15 | [](https://twitter.com/gocolly)
16 |
17 |
18 | ------
19 |
20 |
21 |
22 | ## Features
23 |
24 | - Clean API
25 | - Fast (>1k request/sec on a single core)
26 | - Manages request delays and maximum concurrency per domain
27 | - Automatic cookie and session handling
28 | - Sync/async/parallel scraping
29 | - Caching
30 | - Automatic encoding of non-unicode responses
31 | - Robots.txt support
32 | - Distributed scraping
33 | - Configuration via environment variables
34 | - Extensions
35 |
36 | ## Example
37 |
38 | ```go
39 | func main() {
40 | c := colly.NewCollector()
41 |
42 | // Find and visit all links
43 | c.OnHTML("a[href]", func(e *colly.HTMLElement) {
44 | e.Request.Visit(e.Attr("href"))
45 | })
46 |
47 | c.OnRequest(func(r *colly.Request) {
48 | fmt.Println("Visiting", r.URL)
49 | })
50 |
51 | c.Visit("http://go-colly.org/")
52 | }
53 | ```
54 |
55 | See [examples folder](https://github.com/gocolly/colly/tree/master/_examples) for more detailed examples.
56 |
57 | ## Installation
58 |
59 | Add colly to your `go.mod` file:
60 |
61 | ```
62 | module github.com/x/y
63 |
64 | go 1.14
65 |
66 | require (
67 | github.com/gocolly/colly/v2 latest
68 | )
69 | ```
70 |
71 | ## Bugs
72 |
73 | Bugs or suggestions? Visit the [issue tracker](https://github.com/gocolly/colly/issues) or join `#colly` on freenode
74 |
75 | ## Other Projects Using Colly
76 |
77 | Below is a list of public, open source projects that use Colly:
78 |
79 | - [greenpeace/check-my-pages](https://github.com/greenpeace/check-my-pages) Scraping script to test the Spanish Greenpeace web archive.
80 | - [altsab/gowap](https://github.com/altsab/gowap) Wappalyzer implementation in Go.
81 | - [jesuiscamille/goquotes](https://github.com/jesuiscamille/goquotes) A quotes scraper, making your day a little better!
82 | - [jivesearch/jivesearch](https://github.com/jivesearch/jivesearch) A search engine that doesn't track you.
83 | - [Leagify/colly-draft-prospects](https://github.com/Leagify/colly-draft-prospects) A scraper for future NFL Draft prospects.
84 | - [lucasepe/go-ps4](https://github.com/lucasepe/go-ps4) Search playstation store for your favorite PS4 games using the command line.
85 | - [yringler/inside-chassidus-scraper](https://github.com/yringler/inside-chassidus-scraper) Scrapes Rabbi Paltiel's web site for lesson metadata.
86 | - [gamedb/gamedb](https://github.com/gamedb/gamedb) A database of Steam games.
87 | - [lawzava/scrape](https://github.com/lawzava/scrape) CLI for email scraping from any website.
88 | - [eureka101v/WeiboSpiderGo](https://github.com/eureka101v/WeiboSpiderGo) A sina weibo(chinese twitter) scraper
89 | - [Go-phie/gophie](https://github.com/Go-phie/gophie) Search, Download and Stream movies from your terminal
90 | - [imthaghost/goclone](https://github.com/imthaghost/goclone) Clone websites to your computer within seconds.
91 | - [superiss/spidy](https://github.com/superiss/spidy) Crawl the web and collect expired domains.
92 | - [docker-slim/docker-slim](https://github.com/docker-slim/docker-slim) Optimize your Docker containers to make them smaller and better.
93 | - [seversky/gachifinder](https://github.com/seversky/gachifinder) an agent for asynchronous scraping, parsing and writing to some storages(elasticsearch for now)
94 | - [eval-exec/goodreads](https://github.com/eval-exec/goodreads) crawl all tags and all pages of quotes from goodreads.
95 |
96 | If you are using Colly in a project please send a pull request to add it to the list.
97 |
98 | ## Contributors
99 |
100 | This project exists thanks to all the people who contribute. [[Contribute]](CONTRIBUTING.md).
101 |