├── .DS_Store ├── .gitattributes ├── .github └── workflows │ └── build.yml ├── .gitignore ├── LICENSE.txt ├── README.MD ├── chromeRequests ├── __init__.py ├── cookies.py ├── headers.py ├── response.py ├── sessions.py └── utils.py ├── examples.py ├── go ├── chromeRequests.h ├── chromeRequests.so ├── chromeRequests.so~ ├── compiled │ ├── library-Darwin-64.h │ ├── library32-windows.dll │ ├── library32-windows.so │ └── library64-windows.so ├── go.mod ├── go.sum ├── library.so ├── main.go ├── models │ ├── requestParameters.go │ ├── response.go │ ├── session.go │ └── sessionParameters.go └── test.sh ├── pyproject.toml └── setup.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/.DS_Store -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: "Build Binaries" 2 | on: 3 | workflow_dispatch: 4 | # This Action only starts when you go to Actions and manually run the workflow. 5 | 6 | jobs: 7 | package: 8 | strategy: 9 | matrix: 10 | include: 11 | - platform: macos-latest 12 | arch: arm64 13 | - platform: windows-latest 14 | arch: amd64 15 | 16 | runs-on: ${{ matrix.platform }} 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Install Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: 1.18 23 | - name: Build Binaries 24 | working-directory: ./go 25 | run: go build -buildmode=c-shared -o ../compiled/${{ matrix.platform }}-${{ matrix.arch }}.so 26 | env: 27 | GOARCH: ${{ matrix.arch }} 28 | CGO_ENABLED: 1 29 | - name: upload artifacts 30 | uses: actions/upload-artifact@v2 31 | with: 32 | name: ${{ matrix.platform }} 33 | path: ./compiled/${{ matrix.platform }}-${{ matrix.arch }}.so 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.pyc 3 | dist 4 | examples.py 5 | chromeRequests.egg-info/dependency_links.txt 6 | chromeRequests.egg-info/PKG-INFO 7 | chromeRequests.egg-info/requires.txt 8 | chromeRequests.egg-info/SOURCES.txt 9 | chromeRequests.egg-info/top_level.txt 10 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Miro 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # chromeRequests 2 | 3 | A python package that mimics chromes TLS using golang 4 | 5 | ## Installation 6 | 7 | Instll chromeRequests using pip 8 | 9 | ```bash 10 | pip install chromeRequests 11 | ``` 12 | 13 | ## API Reference 14 | 15 | ```py 16 | import chromeRequests 17 | 18 | chromeRequests.load_library("./library.so") # this is necessary, download from go folder. 19 | 20 | # get request 21 | 22 | response = chromeRequests.get( 23 | "example.com", 24 | headers={ 25 | "foo": "bar", 26 | }, 27 | proxy="http://user:password@host:port" # never tested with passwordless. 28 | ) 29 | 30 | # post request 31 | 32 | response = chromeRequests.post( 33 | "example.com", 34 | headers={ 35 | "foo": "bar", 36 | }, 37 | json={"foo": "bar"}, 38 | data={"foo": "bar"} 39 | ) 40 | response.json() 41 | cookies = response.cookies 42 | headers = response.headers 43 | status = response.status_code 44 | text = response.text 45 | 46 | # put request 47 | response = chromeRequests.put( 48 | "example.com", 49 | headers={ 50 | "foo": "bar", 51 | }, 52 | json={"foo": "bar"}, 53 | data={"foo": "bar"} 54 | ) 55 | # session 56 | 57 | session = chromeRequests.session() 58 | 59 | session.set_proxy("http://user:password@host:port") 60 | 61 | session.headers.update({ 62 | "foo": "bar" 63 | }) 64 | 65 | session.headers.get_dict() 66 | 67 | session.cookies.set(name, value) 68 | 69 | session.cookies.get_dict() 70 | 71 | ``` 72 | 73 | ## Acknowledgements 74 | 75 | - [Mimic By Sauce](https://github.com/saucesteals/mimic) 76 | 77 | ## Authors 78 | 79 | - [@Miro](https://github.com/miromiro11) 80 | 81 | ## To do 82 | 83 | - [ ] Speed Improvement 84 | - [x] Same syntax as pythons [Request Library](https://pypi.org/project/requests/) 85 | 86 | ## Contributing 87 | 88 | Contributions are always welcome! 89 | 90 | Please adhere to this project's `code of conduct`. 91 | -------------------------------------------------------------------------------- /chromeRequests/__init__.py: -------------------------------------------------------------------------------- 1 | from . import sessions 2 | from .response import Response 3 | import ctypes 4 | 5 | globalLibrary = None 6 | 7 | 8 | def load_library(path_to_lib): 9 | global globalLibrary 10 | globalLibrary = ctypes.cdll.LoadLibrary(path_to_lib) 11 | print("Library Loaded Successfully") 12 | 13 | 14 | def get(url: str, **kwargs) -> Response: 15 | with sessions.Session(one_time=True) as session: 16 | return session.get(url, **kwargs) 17 | 18 | 19 | def post(url: str, **kwargs) -> Response: 20 | with sessions.Session(one_time=True) as session: 21 | return session.post(url, **kwargs) 22 | 23 | 24 | def put(url: str, **kwargs) -> Response: 25 | with sessions.Session(one_time=True) as session: 26 | return session.put(url, **kwargs) 27 | 28 | 29 | def session(proxy="") -> sessions.Session: 30 | return sessions.Session(proxy) 31 | -------------------------------------------------------------------------------- /chromeRequests/cookies.py: -------------------------------------------------------------------------------- 1 | class Cookies: 2 | def __init__(self): 3 | self.__cookies = {} 4 | 5 | def set(self, name: str, value: str) -> None: 6 | self.__cookies.update({ 7 | name: value, 8 | }) 9 | 10 | def update(self, cookies: dict): 11 | self.__cookies.update(cookies) 12 | 13 | def get_dict(self) -> dict: 14 | return self.__cookies 15 | 16 | def clear(self) -> None: 17 | self.__cookies = {} 18 | 19 | def __str__(self) -> str: 20 | return str(self.__cookies) 21 | -------------------------------------------------------------------------------- /chromeRequests/headers.py: -------------------------------------------------------------------------------- 1 | from typing import OrderedDict 2 | 3 | 4 | class Headers: 5 | def __init__(self): 6 | self.__headers = OrderedDict() 7 | 8 | def update(self, headers: dict) -> None: 9 | self.__headers.update(headers) 10 | 11 | def get_dict(self) -> dict: 12 | ordered = OrderedDict() 13 | for key in self.__headers: 14 | ordered[key.lower()] = self.__headers[key] 15 | return ordered 16 | 17 | def clear(self) -> None: 18 | self.__headers = {} 19 | 20 | def __str__(self) -> str: 21 | return str(self.__headers) 22 | -------------------------------------------------------------------------------- /chromeRequests/response.py: -------------------------------------------------------------------------------- 1 | import json as toJson 2 | 3 | 4 | class Response: 5 | def __init__(self, payload): 6 | self.status_code = payload.get("statusCode", 0) 7 | self.text = payload.get("body", "") 8 | self.cookies = payload.get("cookies", {}) 9 | self.headers = payload.get("headers", {}) 10 | self.url = payload.get("url", "") 11 | 12 | def json(self): 13 | try: 14 | return toJson.loads(self.text) 15 | except Exception as exception: 16 | raise exception 17 | 18 | def __str__(self): 19 | return str(self.status_code) 20 | -------------------------------------------------------------------------------- /chromeRequests/sessions.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import OrderedDict 3 | from .response import Response 4 | from .headers import Headers 5 | from .cookies import Cookies 6 | from .utils import pull_from_mem, check_error 7 | import ctypes 8 | 9 | library = ctypes.cdll.LoadLibrary("./go/chromeRequests.so") 10 | 11 | class Session: 12 | def __init__(self, proxy="", one_time=False): 13 | if library is None: 14 | raise Exception("Library is not loaded") 15 | self.__request = library.request 16 | self.__request.restype = ctypes.c_void_p 17 | self.cookies = Cookies() 18 | self.headers = Headers() 19 | 20 | def request(self, load): 21 | response = self.__request(load) 22 | json_ = json.loads(pull_from_mem(response)) 23 | check_error(json_) 24 | toReturn = Response(json_) 25 | cookies = toReturn.cookies 26 | self.cookies.update(cookies) 27 | return toReturn 28 | 29 | def createPayload(self, request_type: str, url: str, **kwargs) -> dict: 30 | payload = OrderedDict({ 31 | "requestType": request_type, 32 | "url": url, 33 | "parameters": { 34 | "url": url, 35 | } 36 | }) 37 | allowed_get_params = ['proxy', 'headers', "cookies", "allow_redirects"] 38 | allowed_post_params = ['json', 'proxy', 39 | 'headers', 'data', "cookies", "allow_redirects"] 40 | params = allowed_get_params if request_type == "GET" else allowed_post_params 41 | for item in kwargs: 42 | if item not in params: 43 | raise Exception(f"{item} is not an accepted PARAM") 44 | payload['parameters']['headers'] = { 45 | k.lower(): v for k, v in kwargs.get("headers", {}).items()} 46 | payload['parameters']['headers'].update(self.headers.get_dict()) 47 | payload['parameters']['proxy'] = kwargs.get("proxy", "") 48 | payload['parameters']['cookies'] = kwargs.get("cookies", {}) 49 | payload['parameters']['redirects'] = kwargs.get( 50 | "allow_redirects", True) 51 | payload['parameters']['cookies'].update(self.cookies.get_dict()) 52 | payload['parameters']['headerOrder'] = list( 53 | payload['parameters']['headers'].keys()) 54 | if request_type == "POST" or request_type == "PUT": 55 | payload['parameters']['form'] = kwargs.get("data", {}) 56 | payload['parameters']['json'] = json.dumps( 57 | kwargs.get("json", {}), sort_keys=True) 58 | return payload 59 | 60 | def get(self, url: str, **kwargs) -> Response: 61 | payload = self.createPayload("GET", url, **kwargs) 62 | toSend = json.dumps( 63 | payload, sort_keys=True).encode('utf-8') 64 | response = self.request(toSend) 65 | return response 66 | 67 | def post(self, url: str, **kwargs) -> Response: 68 | payload = self.createPayload("POST", url, **kwargs) 69 | print(json.dumps( 70 | payload, sort_keys=True).encode('utf-8')) 71 | response = self.request(json.dumps( 72 | payload, sort_keys=True).encode('utf-8')) 73 | return response 74 | 75 | def put(self, url: str, **kwargs) -> Response: 76 | payload = self.createPayload("PUT", url, **kwargs) 77 | response = self.request(json.dumps( 78 | payload, sort_keys=True).encode('utf-8')) 79 | return response 80 | 81 | def close(self): 82 | self.__closeSession(self.__uuid.encode('utf-8')) 83 | 84 | def __enter__(self): 85 | return self 86 | 87 | def __exit__(self, exc_type, exc_value, traceback): 88 | if self.__uuid != "": 89 | self.close() 90 | return False 91 | -------------------------------------------------------------------------------- /chromeRequests/utils.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | 3 | 4 | def pull_from_mem(bytes_) -> str: 5 | out = ctypes.string_at(bytes_) 6 | return out.decode('utf-8') 7 | 8 | 9 | def check_error(json_): 10 | if json_.get("Error", False): 11 | raise Exception(json_['Error']) 12 | return True 13 | -------------------------------------------------------------------------------- /examples.py: -------------------------------------------------------------------------------- 1 | import chromeRequests 2 | from pprint import pprint 3 | 4 | session = chromeRequests.session() 5 | 6 | response = session.post( 7 | "https://tls.peet.ws/api/all", 8 | json = { 9 | "test":{ 10 | "test":1123 11 | }, 12 | }, 13 | headers = { 14 | "test1":"1", 15 | "test2":"2", 16 | } 17 | ) 18 | 19 | 20 | pprint(response.json()) -------------------------------------------------------------------------------- /go/chromeRequests.h: -------------------------------------------------------------------------------- 1 | /* Code generated by cmd/cgo; DO NOT EDIT. */ 2 | 3 | /* package command-line-arguments */ 4 | 5 | 6 | #line 1 "cgo-builtin-export-prolog" 7 | 8 | #include 9 | 10 | #ifndef GO_CGO_EXPORT_PROLOGUE_H 11 | #define GO_CGO_EXPORT_PROLOGUE_H 12 | 13 | #ifndef GO_CGO_GOSTRING_TYPEDEF 14 | typedef struct { const char *p; ptrdiff_t n; } _GoString_; 15 | #endif 16 | 17 | #endif 18 | 19 | /* Start of preamble from import "C" comments. */ 20 | 21 | 22 | 23 | 24 | /* End of preamble from import "C" comments. */ 25 | 26 | 27 | /* Start of boilerplate cgo prologue. */ 28 | #line 1 "cgo-gcc-export-header-prolog" 29 | 30 | #ifndef GO_CGO_PROLOGUE_H 31 | #define GO_CGO_PROLOGUE_H 32 | 33 | typedef signed char GoInt8; 34 | typedef unsigned char GoUint8; 35 | typedef short GoInt16; 36 | typedef unsigned short GoUint16; 37 | typedef int GoInt32; 38 | typedef unsigned int GoUint32; 39 | typedef long long GoInt64; 40 | typedef unsigned long long GoUint64; 41 | typedef GoInt64 GoInt; 42 | typedef GoUint64 GoUint; 43 | typedef size_t GoUintptr; 44 | typedef float GoFloat32; 45 | typedef double GoFloat64; 46 | #ifdef _MSC_VER 47 | #include 48 | typedef _Fcomplex GoComplex64; 49 | typedef _Dcomplex GoComplex128; 50 | #else 51 | typedef float _Complex GoComplex64; 52 | typedef double _Complex GoComplex128; 53 | #endif 54 | 55 | /* 56 | static assertion to make sure the file is being used on architecture 57 | at least with matching size of GoInt. 58 | */ 59 | typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; 60 | 61 | #ifndef GO_CGO_GOSTRING_TYPEDEF 62 | typedef _GoString_ GoString; 63 | #endif 64 | typedef void *GoMap; 65 | typedef void *GoChan; 66 | typedef struct { void *t; void *v; } GoInterface; 67 | typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; 68 | 69 | #endif 70 | 71 | /* End of boilerplate cgo prologue. */ 72 | 73 | #ifdef __cplusplus 74 | extern "C" { 75 | #endif 76 | 77 | extern __declspec(dllexport) char* request(char* cParams); 78 | 79 | #ifdef __cplusplus 80 | } 81 | #endif 82 | -------------------------------------------------------------------------------- /go/chromeRequests.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/go/chromeRequests.so -------------------------------------------------------------------------------- /go/chromeRequests.so~: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/go/chromeRequests.so~ -------------------------------------------------------------------------------- /go/compiled/library-Darwin-64.h: -------------------------------------------------------------------------------- 1 | /* Code generated by cmd/cgo; DO NOT EDIT. */ 2 | 3 | /* package chromeRequests */ 4 | 5 | 6 | #line 1 "cgo-builtin-export-prolog" 7 | 8 | #include /* for ptrdiff_t below */ 9 | 10 | #ifndef GO_CGO_EXPORT_PROLOGUE_H 11 | #define GO_CGO_EXPORT_PROLOGUE_H 12 | 13 | #ifndef GO_CGO_GOSTRING_TYPEDEF 14 | typedef struct { const char *p; ptrdiff_t n; } _GoString_; 15 | #endif 16 | 17 | #endif 18 | 19 | /* Start of preamble from import "C" comments. */ 20 | 21 | 22 | 23 | 24 | /* End of preamble from import "C" comments. */ 25 | 26 | 27 | /* Start of boilerplate cgo prologue. */ 28 | #line 1 "cgo-gcc-export-header-prolog" 29 | 30 | #ifndef GO_CGO_PROLOGUE_H 31 | #define GO_CGO_PROLOGUE_H 32 | 33 | typedef signed char GoInt8; 34 | typedef unsigned char GoUint8; 35 | typedef short GoInt16; 36 | typedef unsigned short GoUint16; 37 | typedef int GoInt32; 38 | typedef unsigned int GoUint32; 39 | typedef long long GoInt64; 40 | typedef unsigned long long GoUint64; 41 | typedef GoInt64 GoInt; 42 | typedef GoUint64 GoUint; 43 | typedef __SIZE_TYPE__ GoUintptr; 44 | typedef float GoFloat32; 45 | typedef double GoFloat64; 46 | typedef float _Complex GoComplex64; 47 | typedef double _Complex GoComplex128; 48 | 49 | /* 50 | static assertion to make sure the file is being used on architecture 51 | at least with matching size of GoInt. 52 | */ 53 | typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; 54 | 55 | #ifndef GO_CGO_GOSTRING_TYPEDEF 56 | typedef _GoString_ GoString; 57 | #endif 58 | typedef void *GoMap; 59 | typedef void *GoChan; 60 | typedef struct { void *t; void *v; } GoInterface; 61 | typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; 62 | 63 | #endif 64 | 65 | /* End of boilerplate cgo prologue. */ 66 | 67 | #ifdef __cplusplus 68 | extern "C" { 69 | #endif 70 | 71 | extern char* changeProxy(char* params); 72 | extern char* createSession(char* cProxy); 73 | extern char* closeSession(char* uuid); 74 | extern char* request(char* cParams); 75 | 76 | #ifdef __cplusplus 77 | } 78 | #endif 79 | -------------------------------------------------------------------------------- /go/compiled/library32-windows.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/go/compiled/library32-windows.dll -------------------------------------------------------------------------------- /go/compiled/library32-windows.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/go/compiled/library32-windows.so -------------------------------------------------------------------------------- /go/compiled/library64-windows.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/go/compiled/library64-windows.so -------------------------------------------------------------------------------- /go/go.mod: -------------------------------------------------------------------------------- 1 | module chromeRequests 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/saucesteals/fhttp v0.0.0-20220713050157-e3eea48d7785 // direct 7 | github.com/saucesteals/mimic v0.0.0-20220714184236-0f4567d7f8ed // direct 8 | ) 9 | 10 | require ( 11 | github.com/andybalholm/brotli v1.0.4 // indirect 12 | github.com/google/uuid v1.3.0 // direct 13 | github.com/klauspost/compress v1.15.4 // indirect 14 | github.com/saucesteals/utls v1.1.2-0.20220526033452-2e6d7376b70a // indirect 15 | golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect 16 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 17 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 18 | golang.org/x/text v0.3.7 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= 2 | github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 3 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 4 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 5 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 6 | github.com/klauspost/compress v1.15.4 h1:1kn4/7MepF/CHmYub99/nNX8az0IJjfSOU/jbnTVfqQ= 7 | github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 8 | github.com/saucesteals/fhttp v0.0.0-20220713050157-e3eea48d7785 h1:0MRKtgK0/wR6sup5ynRymChlfbpbEmZkLKI4nU34zlo= 9 | github.com/saucesteals/fhttp v0.0.0-20220713050157-e3eea48d7785/go.mod h1:tEKifE6IjbxcvVSmEOiHH0e/3dvSv6sN4nailAD7tPE= 10 | github.com/saucesteals/mimic v0.0.0-20220714184236-0f4567d7f8ed h1:Ucn/LZIz8pqIm39FXtmrKd6mELUHmaHQ0rge9K76tEU= 11 | github.com/saucesteals/mimic v0.0.0-20220714184236-0f4567d7f8ed/go.mod h1:m285Jf6WgFhqHO38LL55da2AzG26QSYIEbLC9KEPmLE= 12 | github.com/saucesteals/utls v1.1.2-0.20220526033452-2e6d7376b70a h1:jiEP6CVElK5wDp/e+eoiNopskFQKIQbI9ZvqzGh40cA= 13 | github.com/saucesteals/utls v1.1.2-0.20220526033452-2e6d7376b70a/go.mod h1:I0obaa60NZ0gTwX8rpbeH6fb18E90q8YlhLdOF0ot0k= 14 | golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 15 | golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0= 16 | golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 17 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 18 | golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 19 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 20 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= 21 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 22 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 23 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 24 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 25 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= 26 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 27 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 28 | golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 29 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 30 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 31 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 32 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 33 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 34 | -------------------------------------------------------------------------------- /go/library.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miromiro11/chromeRequests/75cc1a14c4d7d6923fcf0dbeea302882021d384a/go/library.so -------------------------------------------------------------------------------- /go/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "C" 5 | "bytes" 6 | "chromeRequests/models" 7 | "encoding/json" 8 | "net/url" 9 | 10 | http "github.com/saucesteals/fhttp" 11 | "github.com/saucesteals/mimic" 12 | ) 13 | import ( 14 | "fmt" 15 | "io" 16 | "sync" 17 | ) 18 | 19 | var Sessions = make(map[string]models.Session) 20 | var latestVersion = mimic.MustGetLatestVersion(mimic.PlatformWindows) 21 | var m, _ = mimic.Chromium(mimic.BrandChrome, latestVersion) 22 | var cachedTransport = map[string]*LockableTransport{} 23 | 24 | func createTransport(proxy string) (*LockableTransport, error) { 25 | var LockableTransport = &LockableTransport{} 26 | 27 | if trans, ok := cachedTransport[proxy]; ok { 28 | return trans, nil 29 | } 30 | 31 | if proxy != "" { 32 | proxyURL, err := url.Parse(proxy) 33 | 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | transport := m.ConfigureTransport(&http.Transport{ 39 | Proxy: http.ProxyURL(proxyURL), 40 | }) 41 | 42 | LockableTransport.Transport = transport 43 | LockableTransport.Lock = &sync.Mutex{} 44 | 45 | cachedTransport[proxy] = LockableTransport 46 | 47 | return cachedTransport[proxy], nil 48 | } 49 | 50 | transport := m.ConfigureTransport(&http.Transport{}) 51 | LockableTransport.Transport = transport 52 | LockableTransport.Lock = &sync.Mutex{} 53 | 54 | return LockableTransport, nil 55 | } 56 | 57 | //export request 58 | func request(cParams *C.char) *C.char { 59 | var client http.Client = http.Client{} 60 | 61 | var req *http.Request 62 | 63 | params := C.GoString(cParams) 64 | 65 | data := models.SessionParameters{} 66 | 67 | err := json.Unmarshal([]byte(params), &data) 68 | 69 | if err != nil { 70 | return createCResponse(&models.Response{Error: err.Error()}) 71 | } 72 | 73 | req, err = http.NewRequest(data.RequestType, data.Parameters.URL, nil) 74 | 75 | if err != nil { 76 | return createCResponse(&models.Response{Error: err.Error()}) 77 | } 78 | 79 | if data.Parameters.Json != "" { 80 | req.Body = io.NopCloser(bytes.NewBuffer([]byte(data.Parameters.Json))) 81 | } 82 | 83 | if data.Parameters.Form != nil { 84 | formData := url.Values{} 85 | for key, value := range data.Parameters.Form { 86 | formData.Add(key, value) 87 | } 88 | req.Body = io.NopCloser(bytes.NewBufferString(formData.Encode())) 89 | } 90 | 91 | transport, err := createTransport(data.Parameters.Proxy) 92 | 93 | if err != nil { 94 | return createCResponse(&models.Response{Error: err.Error()}) 95 | } 96 | 97 | transport.Lock.Lock() 98 | defer transport.Lock.Unlock() 99 | client.Transport = transport.Transport 100 | 101 | if data.Parameters.Headers != nil { 102 | for k, v := range data.Parameters.Headers { 103 | req.Header[k] = []string{v} 104 | } 105 | req.Header[http.HeaderOrderKey] = data.Parameters.HeaderOrder 106 | req.Header[http.PHeaderOrderKey] = m.PseudoHeaderOrder() 107 | } 108 | 109 | for k, v := range data.Parameters.Cookies { 110 | req.AddCookie(&http.Cookie{ 111 | Name: k, 112 | Value: v, 113 | }) 114 | } 115 | 116 | if !data.Parameters.Redirects { 117 | client.CheckRedirect = func(req *http.Request, via []*http.Request) error { 118 | return http.ErrUseLastResponse 119 | } 120 | } 121 | 122 | resp, err := client.Do(req) 123 | 124 | if err != nil { 125 | return createCResponse(&models.Response{Error: err.Error()}) 126 | } 127 | 128 | defer resp.Body.Close() 129 | 130 | body, err := io.ReadAll(resp.Body) 131 | 132 | if err != nil { 133 | return createCResponse(&models.Response{Error: err.Error()}) 134 | } 135 | 136 | headersMap := map[string]string{} 137 | 138 | cookieMap := map[string]string{} 139 | 140 | for _, cookie := range resp.Cookies() { 141 | cookieMap[cookie.Name] = cookie.Value 142 | } 143 | 144 | client.CheckRedirect = nil 145 | 146 | return createCResponse(&models.Response{ 147 | StatusCode: resp.StatusCode, 148 | Body: string(body), 149 | Cookies: cookieMap, 150 | Headers: headersMap, 151 | Url: resp.Request.URL.String(), 152 | }) 153 | } 154 | 155 | func createCResponse(resp *models.Response) *C.char { 156 | errorJson, _ := json.Marshal(resp) 157 | return C.CString(string(errorJson)) 158 | } 159 | 160 | func main() { 161 | // RequestStructure := models.SessionParameters{ 162 | // Parameters: &models.RequestParameters{ 163 | // URL: "https://tls.peet.ws/api/all", 164 | // Headers: map[string]string{ 165 | // "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0", 166 | // "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 167 | // "accept-language": "en-US,en;q=0.5", 168 | // "accept-encoding": "gzip, deflate, br", 169 | // }, 170 | // HeaderOrder: []string{ 171 | // "user-agent", 172 | // "accept", 173 | // "accept-language", 174 | // "accept-encoding", 175 | // }, 176 | // Cookies: map[string]string{}, 177 | // }, 178 | // RequestType: "GET", 179 | // } 180 | // seshJson, _ := json.Marshal(RequestStructure) 181 | // log.Println(string(seshJson)) 182 | resp := request(C.CString(string(`{"parameters": {"cookies": {}, "form": {}, "headerOrder": [], "headers": {}, "json": "{}", "proxy": "", "redirects": true, "url": "https://httpbin.org/post"}, "requestType": "POST", "url": "https://httpbin.org/post"}`))) 183 | fmt.Println(C.GoString(resp)) 184 | 185 | } 186 | 187 | type LockableTransport struct { 188 | Transport *http.Transport 189 | Lock *sync.Mutex 190 | } 191 | 192 | //make a command to build a c-shared library 193 | //go:generate go build -buildmode=c-shared -o chromeRequests.so main.go 194 | -------------------------------------------------------------------------------- /go/models/requestParameters.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type RequestParameters struct { 4 | URL string `json:"url"` 5 | Proxy string `json:"proxy"` 6 | Headers map[string]string `json:"headers"` 7 | Form map[string]string `json:"form"` 8 | Json string `json:"json"` 9 | Cookies map[string]string `json:"cookies"` 10 | Redirects bool `json:"redirects"` 11 | HeaderOrder []string `json:"headerOrder"` 12 | } 13 | -------------------------------------------------------------------------------- /go/models/response.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Response struct { 4 | StatusCode int `json:"statusCode,omitempty"` 5 | Body string `json:"body,omitempty"` 6 | Cookies map[string]string `json:"cookies,omitempty"` 7 | Headers map[string]string `json:"headers,omitempty"` 8 | Url string `json:"url,omitempty"` 9 | Error string `json:"error,omitempty"` 10 | SessionId string `json:"sessionId,omitempty"` 11 | } 12 | -------------------------------------------------------------------------------- /go/models/session.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import http "github.com/saucesteals/fhttp" 4 | 5 | type Session struct { 6 | Client *http.Client 7 | Headers map[string]string 8 | Cookies map[string]string 9 | Randomize bool 10 | } 11 | -------------------------------------------------------------------------------- /go/models/sessionParameters.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type SessionParameters struct { 4 | SessionId string `json:"session"` 5 | RequestType string `json:"requestType"` 6 | Parameters *RequestParameters `json:"parameters"` 7 | Proxy string `json:"proxy"` 8 | } 9 | -------------------------------------------------------------------------------- /go/test.sh: -------------------------------------------------------------------------------- 1 | # 32-bit 2 | GOOS=windows CGO_ENABLED=1 GOARCH=386 go build -buildmode=c-shared -o ./compiled/library32-windows.so 3 | 4 | # 64-bit 5 | GOOS=windows CGO_ENABLED=1 GOARCH=amd64 go build -buildmode=c-shared -o ./compiled/library64-windows.so 6 | 7 | darwin 8 | GOOS=darwin CGO_ENABLED=1 GOARCH=arm64 go build -buildmode=c-shared -o ./compiled/library-Darwin-64.so -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | author = "Miro" 2 | module = "chromeRequests" 3 | dist-name = "chromeRequests" 4 | description-file = "README.md" 5 | requires-python = ">=3.6" -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | setup( 5 | name='chromeRequests', 6 | version='0.1.9', 7 | author="Miro", 8 | license='MIT', 9 | packages=find_packages(), 10 | install_requires=[ 11 | 'scikit-learn', 12 | ], 13 | 14 | ) 15 | --------------------------------------------------------------------------------