├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── cloudscraper ├── browsers.go ├── client.go └── resources │ └── browsers.json ├── go.mod ├── go.sum └── readMe.md /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: build_n_test 2 | on: 3 | pull_request: 4 | branches: [ "master" ] 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-24.04 9 | container: 10 | image: golang:1.20.6 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Install dependencies 15 | run: go get ./... 16 | - name: Build 17 | run: go build -v ./... 18 | - name: Test with the Go CLI 19 | run: go test ./... 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2025 RomainMichau 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /cloudscraper/browsers.go: -------------------------------------------------------------------------------- 1 | package cloudscraper 2 | 3 | import ( 4 | _ "embed" 5 | "encoding/json" 6 | "math/rand" 7 | "time" 8 | ) 9 | 10 | type browserDescription struct { 11 | UserAgents userAgents `json:"user_agents"` 12 | Ja3 map[string]string `json:"ja3"` 13 | Headers map[string]map[string]string `json:"headers"` 14 | } 15 | 16 | type userAgents struct { 17 | // os -> browser - [user-agents] 18 | Desktop map[string]map[string][]string `json:"desktop"` 19 | Mobile map[string]map[string][]string `json:"mobile"` 20 | } 21 | 22 | type BrowserConf struct { 23 | UserAgent string 24 | Ja3 string 25 | Headers map[string]string 26 | } 27 | 28 | //go:embed resources/browsers.json 29 | var browsersJson string 30 | 31 | func readJsonFile() (browserDescription, error) { 32 | // Open our jsonFile 33 | var browsers browserDescription 34 | err := json.Unmarshal([]byte(browsersJson), &browsers) 35 | // defer the closing of our jsonFile so that we can parse it later on 36 | return browsers, err 37 | } 38 | 39 | func getUserAgents(mobile bool) (BrowserConf, error) { 40 | rand.Seed(time.Now().UnixNano()) 41 | var userAgents map[string]map[string][]string 42 | browsersDescription, err := readJsonFile() 43 | if err != nil { 44 | return BrowserConf{}, err 45 | } 46 | if mobile { 47 | userAgents = browsersDescription.UserAgents.Mobile 48 | } else { 49 | userAgents = browsersDescription.UserAgents.Desktop 50 | } 51 | var osList []string 52 | for k := range userAgents { 53 | osList = append(osList, k) 54 | } 55 | rnd := rand.Intn(len(osList)) 56 | pickedOs := userAgents[osList[rnd]] 57 | var browserList []string 58 | for k := range pickedOs { 59 | browserList = append(browserList, k) 60 | } 61 | rnd = rand.Intn(len(browserList)) 62 | browserName := browserList[rnd] 63 | pickedBrowser := pickedOs[browserName] 64 | rnd = rand.Intn(len(pickedBrowser)) 65 | pickedUserAgent := pickedBrowser[rnd] 66 | ja3 := browsersDescription.Ja3[browserName] 67 | return BrowserConf{UserAgent: pickedUserAgent, Ja3: ja3, Headers: browsersDescription.Headers[browserName]}, nil 68 | } 69 | -------------------------------------------------------------------------------- /cloudscraper/client.go: -------------------------------------------------------------------------------- 1 | package cloudscraper 2 | 3 | import ( 4 | "github.com/Danny-Dasilva/CycleTLS/cycletls" 5 | ) 6 | 7 | type CloudScrapper struct { 8 | client cycletls.CycleTLS 9 | respChan chan cycletls.Response 10 | defaultHeader map[string]string 11 | ja3 string 12 | userAgent string 13 | } 14 | 15 | func (cs CloudScrapper) Do(url string, options cycletls.Options, method string) (cycletls.Response, error) { 16 | for k, v := range cs.defaultHeader { 17 | options.Headers[k] = v 18 | } 19 | if options.UserAgent == "" { 20 | options.UserAgent = cs.userAgent 21 | } 22 | if options.Ja3 == "" { 23 | options.Ja3 = cs.ja3 24 | } 25 | return cs.client.Do(url, options, method) 26 | } 27 | 28 | // Queue Push a request in the Queue 29 | // Read the result with RespChan() 30 | func (cs CloudScrapper) Queue(url string, options cycletls.Options, method string) { 31 | for k, v := range cs.defaultHeader { 32 | options.Headers[k] = v 33 | } 34 | if options.UserAgent == "" { 35 | options.UserAgent = cs.userAgent 36 | } 37 | if options.Ja3 == "" { 38 | options.Ja3 = cs.ja3 39 | } 40 | 41 | cs.client.Queue(url, options, method) 42 | } 43 | 44 | func (cs CloudScrapper) Get(url string, headers map[string]string, body string) (cycletls.Response, error) { 45 | options := cycletls.Options{ 46 | Ja3: cs.ja3, 47 | Body: body, 48 | UserAgent: cs.userAgent, 49 | Headers: headers} 50 | return cs.Do(url, options, "GET") 51 | } 52 | 53 | func (cs CloudScrapper) Post(url string, headers map[string]string, body string) (cycletls.Response, error) { 54 | options := cycletls.Options{ 55 | Ja3: cs.ja3, 56 | Body: body, 57 | UserAgent: cs.userAgent, 58 | Headers: headers} 59 | return cs.Do(url, options, "POST") 60 | } 61 | 62 | func (cs CloudScrapper) RespChan() chan cycletls.Response { 63 | return cs.respChan 64 | } 65 | 66 | func Init(mobile bool, workers bool) (*CloudScrapper, error) { 67 | browserConf, err := getUserAgents(mobile) 68 | if err != nil { 69 | return nil, err 70 | } 71 | cycleTstClient := cycletls.Init(workers) 72 | p := CloudScrapper{client: cycleTstClient, defaultHeader: browserConf.Headers, ja3: browserConf.Ja3, userAgent: browserConf.UserAgent, 73 | respChan: cycleTstClient.RespChan} 74 | return &p, nil 75 | } 76 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/RomainMichau/cloudscraper_go 2 | 3 | go 1.19 4 | 5 | require github.com/Danny-Dasilva/CycleTLS/cycletls v0.0.0-20220620102923-c84d740b4757 6 | 7 | require ( 8 | github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 // indirect 9 | github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e // indirect 10 | github.com/andybalholm/brotli v1.0.4 // indirect 11 | github.com/dsnet/compress v0.0.1 // indirect 12 | github.com/gorilla/websocket v1.5.0 // indirect 13 | golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect 14 | golang.org/x/net v0.0.0-20220615171555-694bf12d69de // indirect 15 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect 16 | golang.org/x/text v0.3.7 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Danny-Dasilva/CycleTLS/cycletls v0.0.0-20220620102923-c84d740b4757 h1:QH22vGS2DO07khPzKe3/CgFevznQkip5WNGEsQX7mFI= 2 | github.com/Danny-Dasilva/CycleTLS/cycletls v0.0.0-20220620102923-c84d740b4757/go.mod h1:R4Hj85bdRH8zqymQ/oZUCmEsODgP3NpUvTEJtaVai7Y= 3 | github.com/Danny-Dasilva/fhttp v0.0.0-20220418170016-5ea1c560e6a8/go.mod h1:t534vrahRNn9ax1tRiYSUvwJSa9jWaYYgETlfodBPm4= 4 | github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 h1:Wzbitazy0HugGNRACX7ZB1En21LT/TiVF6YbxoTTqN8= 5 | github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6/go.mod h1:2IT2IFG+d+zzFuj3+ksGtVytcCBsF402zMNWHsWhD2U= 6 | github.com/Danny-Dasilva/utls v0.0.0-20220418055514-7c61e0dbb504/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU= 7 | github.com/Danny-Dasilva/utls v0.0.0-20220418175931-f38e470e04f2/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU= 8 | github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e h1:tqiguW0yAcIwQBQtD+d2rjBnboqB7CwG1OZ12F8avX8= 9 | github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e/go.mod h1:ssfbVNUfWJVRfW41RTpedOUlGXSq3J6aLmirUVkDgJk= 10 | github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= 11 | github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= 12 | github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 13 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 14 | github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= 15 | github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= 16 | github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= 17 | github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= 18 | github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 19 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 20 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 21 | github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= 22 | gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ= 23 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 24 | golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 25 | golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 26 | golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 27 | golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= 28 | golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 29 | golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 30 | golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 31 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 32 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 33 | golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 34 | golang.org/x/net v0.0.0-20220615171555-694bf12d69de h1:ogOG2+P6LjO2j55AkRScrkB2BFpd+Z8TY2wcM0Z3MGo= 35 | golang.org/x/net v0.0.0-20220615171555-694bf12d69de/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 36 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 37 | golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 38 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 39 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 40 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 41 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 42 | golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 43 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 44 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= 45 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 46 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 47 | golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 48 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 49 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 50 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 51 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 52 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 53 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 54 | -------------------------------------------------------------------------------- /readMe.md: -------------------------------------------------------------------------------- 1 | # CloudScraper Go 2 | 3 | Simple go module to bypass CloudFlare securities 4 | 5 | For now only passive checks based on TLS footprint (JA3) and user agents are handled 6 | Inspired of python project https://github.com/venomous/cloudscraper 7 | 8 | Rely on [CycleTls](https://github.com/Danny-Dasilva/CycleTLS) to customize JA3 9 | 10 | JA3 list: 11 | https://raw.githubusercontent.com/trisulnsm/trisul-scripts/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json 12 | 13 | Get my JA3: https://kawayiyi.com/tls 14 | # Example of usage 15 | ## Simple Post 16 | ```go 17 | package main 18 | 19 | import ( 20 | "github.com/RomainMichau/cloudscraper_go/cloudscraper" 21 | ) 22 | 23 | func main() { 24 | 25 | client, _ := cloudscraper.Init(false, false) 26 | res, _ := client.Post("https://www.facebook.com/anything/", make(map[string]string), "") 27 | res2, _ := client.Get("https://www.facebook.com/anything/", make(map[string]string), "") 28 | print(res.Body) 29 | print(res2.Body) 30 | } 31 | ``` 32 | ## Customize request 33 | ```go 34 | package main 35 | 36 | import ( 37 | "github.com/Danny-Dasilva/CycleTLS/cycletls" 38 | "github.com/RomainMichau/cloudscraper_go/cloudscraper" 39 | ) 40 | 41 | func main() { 42 | 43 | client, _ := cloudscraper.Init(false, false) 44 | options := cycletls.Options{ 45 | Headers: map[string]string{"my_custom_header": "header_value"}, 46 | Body: "", 47 | Proxy: "proxy.company.com", 48 | Timeout: 10, 49 | DisableRedirect: true, 50 | } 51 | res, _ := client.Do("https://www.facebook.com/anything", options, "PUT") 52 | print(res.Body) 53 | } 54 | ``` 55 | ## Async Request 56 | ```go 57 | package main 58 | 59 | import ( 60 | "github.com/Danny-Dasilva/CycleTLS/cycletls" 61 | "github.com/RomainMichau/cloudscraper_go/cloudscraper" 62 | ) 63 | 64 | func main() { 65 | 66 | client, _ := cloudscraper.Init(false, true) 67 | options := cycletls.Options{ 68 | Headers: map[string]string{"my_custom_header": "header_value"}, 69 | Body: "", 70 | Timeout: 10, 71 | DisableRedirect: true, 72 | } 73 | client.Queue("https://www.facebook.com/anything", options, "PUT") 74 | respChannel := client.RespChan() 75 | res := <-respChannel 76 | print(res.Body) 77 | } 78 | 79 | ``` 80 | 81 | # Todo 82 | - Bypass cloudflare active counter-measure (JS challenge) as done on python [cloudscraper](https://github.com/venomous/cloudscraper) 83 | --------------------------------------------------------------------------------