├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── shape.go ├── structs.go └── utils.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Johnw7789 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Note 2 | This repository is now deprecated. Success rate is unknown. Feel free to fork and mofidy if desired. 3 | 4 | # Shape Harvester 5 | Hijacks the requests from a headless browser in order to harvest required headers used for protected endpoints. After the headers are harvested the request is blocked so it cannot be completed. 6 | 7 | This package could also be modified to allow multiple harvesters through the use of multiple pages (or incognito pages?) to simultaneously generate headers (potentially tens of thousands per minute). 8 | 9 | ## Installation 10 | ``go get github.com/Johnw7789/shape`` 11 | 12 | ## Usage 13 | ##### Target 14 | This harvester has only been tested on Target.com. Success on other sites may vary wildly. 15 | 16 | 17 | ```Go 18 | // * The keys of the headers we want the values for 19 | headerNames := []string{ 20 | "X-GyJwza5Z-a", 21 | "X-GyJwza5Z-b", 22 | "X-GyJwza5Z-c", 23 | "X-GyJwza5Z-d", 24 | "X-GyJwza5Z-f", 25 | "X-GyJwza5Z-z", 26 | } 27 | 28 | opts := shape.ShapeOpts{ 29 | HeaderNames: headerNames, 30 | Proxy: "", 31 | Url: "https://www.target.com", 32 | ShapeUrl: "https://carts.target.com/web_checkouts/v1/cart_items?field_groups=CART,CART_ITEMS,SUMMARY&key=9f36aeafbe60771e321a7cc95a78140772ab3e96", 33 | Identifier: "cart_items", 34 | Method: "POST", 35 | Body: "{}", 36 | BlockResources: true, // * Will block extra unnecessary resources such as images and css 37 | } 38 | 39 | // * Create a ShapeHarvester, which immediately starts hijacking requests 40 | harvester := shape.NewShapeHarvester(opts) 41 | 42 | // * Harvest headers every 2 seconds, the Headers map will be directly updated on the struct 43 | for { 44 | harvester.HarvestHeaders() 45 | log.Println(harvester.Headers["XGyJwza5Za"]) 46 | time.Sleep(time.Second * time.Duration(2)) 47 | } 48 | ``` 49 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Johnw7789/shape 2 | go 1.19 3 | 4 | require ( 5 | github.com/go-rod/rod v0.112.2 6 | github.com/go-rod/stealth v0.4.8 7 | ) 8 | 9 | require ( 10 | github.com/ysmood/goob v0.4.0 // indirect 11 | github.com/ysmood/gson v0.7.3 // indirect 12 | github.com/ysmood/leakless v0.8.0 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-rod/rod v0.112.0/go.mod h1:GZDtmEs6RpF6kBRYpGCZXxXlKNneKVPiKOjaMbmVVjE= 2 | github.com/go-rod/rod v0.112.2 h1:dwauKYC/H2em8/BcGk3gC0LTzZHf5MIDKf2DVM4z9gU= 3 | github.com/go-rod/rod v0.112.2/go.mod h1:ElViL9ABbcshNQw93+11FrYRH92RRhMKleuILo6+5V0= 4 | github.com/go-rod/stealth v0.4.8 h1:jlZJWncLPixDaRWpEEauqHPmjdacgFAqBbB1jh7s4P8= 5 | github.com/go-rod/stealth v0.4.8/go.mod h1:O1V1megmCu1xH165Mydzhb35m+KUDOgiUv6DtKV/a08= 6 | github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= 7 | github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= 8 | github.com/ysmood/got v0.31.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= 9 | github.com/ysmood/got v0.32.0 h1:aAHdQgfgMb/lo4v+OekM+SSqEJYFI035h5YYvLXsVyU= 10 | github.com/ysmood/got v0.32.0/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= 11 | github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= 12 | github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= 13 | github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= 14 | github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= 15 | github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= 16 | github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= 17 | github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= 18 | -------------------------------------------------------------------------------- /shape.go: -------------------------------------------------------------------------------- 1 | package shape 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/go-rod/rod" 8 | "github.com/go-rod/rod/lib/launcher" 9 | "github.com/go-rod/rod/lib/proto" 10 | ) 11 | 12 | // * Returns a new ShapeHarvester with browser, page, and highjacking initialized and ready 13 | func NewShapeHarvester(opts ShapeOpts) *ShapeHarvester { 14 | launcher.NewBrowser().MustGet() 15 | 16 | harvester := ShapeHarvester{ 17 | Headers: make(map[string]string), 18 | opts: opts, 19 | } 20 | 21 | harvester.Browser = newBrowser("") 22 | harvester.Page = newPage(harvester.Browser) 23 | 24 | harvester.Page.MustNavigate(harvester.opts.Url).MustWaitLoad() 25 | 26 | harvester.initializeHijacking() 27 | harvester.HarvestHeaders() 28 | 29 | return &harvester 30 | } 31 | 32 | // * Fires a http request, which will then be intercepted and blocked by the hijacker 33 | func (harvester *ShapeHarvester) HarvestHeaders() { 34 | harvester.Page.MustEval(fmt.Sprintf(`function shape() { 35 | try { 36 | fetch("%s", { 37 | "method" : "%s", 38 | "referrerPolicy": "no-referrer-when-downgrade", 39 | "credentials": "include", 40 | "body": "%s", 41 | "headers": { 42 | "accept": "application/json", 43 | "accept-language": "en-US,en;q=0.9", 44 | "content-type": "application/json", 45 | "sec-ch-ua": "\"Google Chrome\";v=\"107\", \"Chromium\";v=\"107\", \"Not=A?Brand\";v=\"24\"", 46 | "sec-ch-ua-mobile": "?0", 47 | "sec-ch-ua-platform": "\"Windows\"", 48 | "sec-fetch-dest": "empty", 49 | "sec-fetch-mode": "cors", 50 | "sec-fetch-site": "same-site", 51 | "x-application-name": "web", 52 | }, 53 | }) 54 | } catch {} 55 | }`, harvester.opts.ShapeUrl, harvester.opts.Method, harvester.opts.Body)) 56 | } 57 | 58 | // * Initializes the constant hijacking of requests, all but the specified url will be allowed and continued 59 | func (harvester *ShapeHarvester) initializeHijacking() { 60 | router := harvester.Page.HijackRequests() 61 | 62 | router.MustAdd("*", func(ctx *rod.Hijack) { 63 | if harvester.opts.BlockResources { 64 | if ctx.Request.Method() == "GET" { 65 | if ctx.Request.Type() == proto.NetworkResourceTypeImage || ctx.Request.Type() == proto.NetworkResourceTypeStylesheet { 66 | ctx.Response.Fail(proto.NetworkErrorReasonBlockedByClient) 67 | } 68 | 69 | ctx.ContinueRequest(&proto.FetchContinueRequest{}) 70 | } 71 | } 72 | 73 | if strings.Contains(ctx.Request.URL().Path, harvester.opts.Identifier) { 74 | if ctx.Request.Method() == "OPTIONS" { 75 | ctx.ContinueRequest(&proto.FetchContinueRequest{}) 76 | } else if ctx.Request.Method() == "POST" { 77 | ctx.Response.Fail(proto.NetworkErrorReasonBlockedByClient) 78 | 79 | for _, header := range harvester.opts.HeaderNames { 80 | harvester.Headers[header] = ctx.Request.Header(header) 81 | } 82 | } 83 | } 84 | 85 | }) 86 | 87 | go router.Run() 88 | } 89 | -------------------------------------------------------------------------------- /structs.go: -------------------------------------------------------------------------------- 1 | package shape 2 | 3 | import "github.com/go-rod/rod" 4 | 5 | type ShapeHarvester struct { 6 | Page *rod.Page 7 | Browser *rod.Browser 8 | 9 | Headers map[string]string 10 | 11 | opts ShapeOpts 12 | } 13 | 14 | type ShapeOpts struct { 15 | HeaderNames []string 16 | Proxy string 17 | Url string 18 | ShapeUrl string 19 | Identifier string 20 | Method string 21 | Body string 22 | BlockResources bool 23 | } 24 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package shape 2 | 3 | import ( 4 | "github.com/go-rod/rod" 5 | "github.com/go-rod/rod/lib/launcher" 6 | "github.com/go-rod/rod/lib/launcher/flags" 7 | "github.com/go-rod/stealth" 8 | ) 9 | 10 | func newBrowser(proxy string) *rod.Browser { 11 | var browser *rod.Browser 12 | 13 | if proxy != "" { 14 | // incomplete code, proxy won't work yet 15 | 16 | l := launcher.New() 17 | l = l.Set(flags.ProxyServer, proxy) 18 | 19 | controlURL, _ := l.Launch() 20 | browser := rod.New().ControlURL(controlURL).MustConnect() 21 | 22 | go browser.MustHandleAuth("user", "password")() 23 | 24 | browser.MustIgnoreCertErrors(true) 25 | } 26 | 27 | browser = rod.New().MustConnect() 28 | 29 | return browser 30 | } 31 | 32 | func newPage(browser *rod.Browser) *rod.Page { 33 | page := stealth.MustPage(browser) 34 | 35 | return page 36 | } 37 | --------------------------------------------------------------------------------