84 |
85 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Feature request
3 | about: Suggest an idea for this project
4 | title: ''
5 | labels: 'Feature request'
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 |
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 |
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 |
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | updates:
3 | - package-ecosystem: "pip"
4 | directory: "/"
5 | schedule:
6 | interval: "daily"
7 | open-pull-requests-limit: 10
8 | - package-ecosystem: "github-actions"
9 | directory: "/"
10 | schedule:
11 | interval: "daily"
12 |
--------------------------------------------------------------------------------
/.github/pull_request_template.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Fixes #
5 |
6 | Changes proposed in this pull request:
7 |
8 | -
9 |
10 | ## Before submitting
11 |
12 |
13 | - [ ] I've read and followed all steps in the [Making a pull request](https://github.com/allenai/python-package-template/blob/main/CONTRIBUTING.md#making-a-pull-request)
14 | section of the `CONTRIBUTING` docs.
15 | - [ ] I've updated or added any relevant docstrings following the syntax described in the
16 | [Writing docstrings](https://github.com/allenai/python-package-template/blob/main/CONTRIBUTING.md#writing-docstrings) section of the `CONTRIBUTING` docs.
17 | - [ ] If this PR fixes a bug, I've added a test that will fail without my fix.
18 | - [ ] If this PR adds a new feature, I've added tests that sufficiently cover my new functionality.
19 |
20 | ## After submitting
21 |
22 |
23 | - [ ] All GitHub Actions jobs for my pull request have passed.
24 |
--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
1 | name: Main
2 |
3 | concurrency:
4 | group: ${{ github.workflow }}-${{ github.ref }}
5 | cancel-in-progress: true
6 |
7 | on:
8 | pull_request:
9 | branches:
10 | - main
11 | push:
12 | branches:
13 | - main
14 | tags:
15 | - 'v*.*.*'
16 |
17 | env:
18 | # Change this to invalidate existing cache.
19 | CACHE_PREFIX: v0
20 | PYTHON_PATH: ./
21 |
22 | jobs:
23 | checks:
24 | name: Python ${{ matrix.python }} - ${{ matrix.task.name }}
25 | runs-on: [ubuntu-latest]
26 | timeout-minutes: 30
27 | strategy:
28 | fail-fast: false
29 | matrix:
30 | python: ['3.7', '3.8', '3.9', '3.10']
31 | task:
32 | - name: Test
33 | run: |
34 | pytest -v --color=yes tests/
35 |
36 | include:
37 | - python: '3.10'
38 | task:
39 | name: Lint
40 | run: flake8 .
41 |
42 | - python: '3.10'
43 | task:
44 | name: Type check
45 | run: mypy .
46 |
47 | - python: '3.10'
48 | task:
49 | name: Build
50 | run: |
51 | python setup.py check
52 | python setup.py bdist_wheel sdist
53 |
54 | - python: '3.10'
55 | task:
56 | name: Style
57 | run: black --check .
58 |
59 | - python: '3.10'
60 | task:
61 | name: Docs
62 | run: cd docs && make html
63 |
64 | steps:
65 | - uses: actions/checkout@v3
66 |
67 | - name: Setup Python
68 | uses: actions/setup-python@v4
69 | with:
70 | python-version: ${{ matrix.python }}
71 |
72 | - name: Install prerequisites
73 | run: |
74 | pip install --upgrade pip setuptools wheel virtualenv
75 |
76 | - name: Set build variables
77 | shell: bash
78 | run: |
79 | # Get the exact Python version to use in the cache key.
80 | echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
81 | echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
82 | # Use week number in cache key so we can refresh the cache weekly.
83 | echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
84 |
85 | - uses: actions/cache@v3
86 | id: virtualenv-cache
87 | with:
88 | path: .venv
89 | key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}
90 | restore-keys: |
91 | ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-
92 |
93 | - name: Setup virtual environment (no cache hit)
94 | if: steps.virtualenv-cache.outputs.cache-hit != 'true'
95 | run: |
96 | test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
97 | . .venv/bin/activate
98 | pip install -e .[dev]
99 |
100 | - name: Install editable (cache hit)
101 | if: steps.virtualenv-cache.outputs.cache-hit == 'true'
102 | run: |
103 | . .venv/bin/activate
104 | pip install --no-deps -e .[dev]
105 |
106 | - name: Show environment info
107 | run: |
108 | . .venv/bin/activate
109 | which python
110 | python --version
111 | pip freeze
112 |
113 | - name: ${{ matrix.task.name }}
114 | run: |
115 | . .venv/bin/activate
116 | ${{ matrix.task.run }}
117 |
118 | - name: Upload package distribution files
119 | if: matrix.task.name == 'Build'
120 | uses: actions/upload-artifact@v3
121 | with:
122 | name: package
123 | path: dist
124 |
125 | - name: Clean up
126 | if: always()
127 | run: |
128 | . .venv/bin/activate
129 | pip uninstall -y my-package
130 |
131 | release:
132 | name: Release
133 | runs-on: ubuntu-latest
134 | needs: [checks]
135 | if: startsWith(github.ref, 'refs/tags/')
136 | steps:
137 | - uses: actions/checkout@v1
138 |
139 | - name: Setup Python
140 | uses: actions/setup-python@v4
141 | with:
142 | python-version: '3.10'
143 |
144 | - name: Install requirements
145 | run: |
146 | pip install --upgrade pip setuptools wheel twine
147 |
148 | - name: Prepare environment
149 | run: |
150 | echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
151 | echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
152 |
153 | - name: Download package distribution files
154 | uses: actions/download-artifact@v3
155 | with:
156 | name: package
157 | path: dist
158 |
159 | - name: Generate release notes
160 | run: |
161 | python scripts/release_notes.py > ${{ github.workspace }}-RELEASE_NOTES.md
162 |
163 | - name: Publish package to PyPI
164 | run: |
165 | twine upload -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_PASSWORD }} dist/*
166 |
167 | - name: Publish GitHub release
168 | uses: softprops/action-gh-release@v1
169 | env:
170 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
171 | with:
172 | body_path: ${{ github.workspace }}-RELEASE_NOTES.md
173 | prerelease: ${{ contains(env.TAG, 'rc') }}
174 | files: |
175 | dist/*
176 |
--------------------------------------------------------------------------------
/.github/workflows/pr_checks.yml:
--------------------------------------------------------------------------------
1 | name: PR Checks
2 |
3 | concurrency:
4 | group: ${{ github.workflow }}-${{ github.ref }}
5 | cancel-in-progress: true
6 |
7 | on:
8 | pull_request:
9 | branches:
10 | - main
11 | paths:
12 | - 'my_package/**'
13 |
14 | jobs:
15 | changelog:
16 | name: CHANGELOG
17 | runs-on: ubuntu-latest
18 | if: github.event_name == 'pull_request'
19 |
20 | steps:
21 | - uses: actions/checkout@v1
22 |
23 | - name: Check that CHANGELOG has been updated
24 | run: |
25 | # If this step fails, this means you haven't updated the CHANGELOG.md
26 | # file with notes on your contribution.
27 | git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!"
28 |
--------------------------------------------------------------------------------
/.github/workflows/setup.yml:
--------------------------------------------------------------------------------
1 | name: Setup
2 |
3 | concurrency:
4 | group: ${{ github.workflow }}-${{ github.ref }}
5 | cancel-in-progress: true
6 |
7 | on:
8 | pull_request:
9 | branches:
10 | - main
11 | push:
12 | branches:
13 | - main
14 |
15 | jobs:
16 | test_personalize:
17 | name: Personalize
18 | runs-on: [ubuntu-latest]
19 | timeout-minutes: 10
20 | steps:
21 | - uses: actions/checkout@v3
22 |
23 | - name: Setup Python
24 | uses: actions/setup-python@v4
25 | with:
26 | python-version: '3.7'
27 |
28 | - name: Install prerequisites
29 | run: |
30 | pip install -r setup-requirements.txt
31 |
32 | - name: Run personalize script
33 | run: |
34 | python scripts/personalize.py --github-org epwalsh --github-repo new-repo --package-name new-package --yes
35 |
36 | - name: Verify changes
37 | shell: bash
38 | run: |
39 | set -eo pipefail
40 | # Check that 'new-package' replaced 'my-package' in some files.
41 | grep -q 'new-package' setup.py .github/workflows/main.yml CONTRIBUTING.md
42 | # Check that the new repo URL replaced the old one in some files.
43 | grep -q 'https://github.com/epwalsh/new-repo' setup.py CONTRIBUTING.md
44 | # Double check that there are no lingering mentions of old names.
45 | for pattern in 'my[-_]package' 'https://github.com/allenai/python-package-template'; do
46 | if find . -type f -not -path './.git/*' | xargs grep "$pattern"; then
47 | echo "Found ${pattern} where it shouldn't be!"
48 | exit 1
49 | fi
50 | done
51 | echo "All good!"
52 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # build artifacts
2 |
3 | .eggs/
4 | .mypy_cache
5 | *.egg-info/
6 | build/
7 | dist/
8 | pip-wheel-metadata/
9 |
10 |
11 | # dev tools
12 |
13 | .envrc
14 | .python-version
15 | .idea
16 | .venv/
17 | .vscode/
18 | /*.iml
19 |
20 |
21 | # jupyter notebooks
22 |
23 | .ipynb_checkpoints
24 |
25 |
26 | # miscellaneous
27 |
28 | .cache/
29 | doc/_build/
30 | *.swp
31 | .DS_Store
32 |
33 |
34 | # python
35 |
36 | *.pyc
37 | *.pyo
38 | __pycache__
39 |
40 |
41 | # testing and continuous integration
42 |
43 | .coverage
44 | .pytest_cache/
45 | .benchmarks
46 |
47 | # documentation build artifacts
48 |
49 | docs/build
50 | site/
51 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Danny-Dasilva/cycletls_python/43d8db8aa5e31ead0244397eac0014e7ad7b582f/MANIFEST.in
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY : docs
2 | docs :
3 | rm -rf docs/build/
4 | sphinx-autobuild -b html --watch my_package/ docs/source/ docs/build/
5 |
6 | .PHONY : run-checks
7 | run-checks :
8 | isort --check .
9 | black --check .
10 | flake8 .
11 | mypy .
12 | CUDA_VISIBLE_DEVICES='' pytest -v --color=yes --doctest-modules tests/ my_package/
13 |
--------------------------------------------------------------------------------
/Pipfile:
--------------------------------------------------------------------------------
1 | [[source]]
2 | url = "https://pypi.python.org/simple"
3 | verify_ssl = true
4 | name = "pypi"
5 |
6 | [packages]
7 | websockets = "*"
8 | pydantic = "*"
9 | websocket-client = "*"
10 |
11 | [dev-packages]
12 | mypy = "*"
13 |
14 | [requires]
15 | python_version = "3.9"
16 |
--------------------------------------------------------------------------------
/Pipfile.lock:
--------------------------------------------------------------------------------
1 | {
2 | "_meta": {
3 | "hash": {
4 | "sha256": "0f101a973e8cdc8e59c0fd9c48f20833f1f7fb0697035dcd8533895281c4a7af"
5 | },
6 | "pipfile-spec": 6,
7 | "requires": {
8 | "python_version": "3.9"
9 | },
10 | "sources": [
11 | {
12 | "name": "pypi",
13 | "url": "https://pypi.python.org/simple",
14 | "verify_ssl": true
15 | }
16 | ]
17 | },
18 | "default": {
19 | "pydantic": {
20 | "hashes": [
21 | "sha256:02eefd7087268b711a3ff4db528e9916ac9aa18616da7bca69c1871d0b7a091f",
22 | "sha256:059b6c1795170809103a1538255883e1983e5b831faea6558ef873d4955b4a74",
23 | "sha256:0bf07cab5b279859c253d26a9194a8906e6f4a210063b84b433cf90a569de0c1",
24 | "sha256:1542636a39c4892c4f4fa6270696902acb186a9aaeac6f6cf92ce6ae2e88564b",
25 | "sha256:177071dfc0df6248fd22b43036f936cfe2508077a72af0933d0c1fa269b18537",
26 | "sha256:18f3e912f9ad1bdec27fb06b8198a2ccc32f201e24174cec1b3424dda605a310",
27 | "sha256:1dd8fecbad028cd89d04a46688d2fcc14423e8a196d5b0a5c65105664901f810",
28 | "sha256:1ed987c3ff29fff7fd8c3ea3a3ea877ad310aae2ef9889a119e22d3f2db0691a",
29 | "sha256:447d5521575f18e18240906beadc58551e97ec98142266e521c34968c76c8761",
30 | "sha256:494f7c8537f0c02b740c229af4cb47c0d39840b829ecdcfc93d91dcbb0779892",
31 | "sha256:4988c0f13c42bfa9ddd2fe2f569c9d54646ce84adc5de84228cfe83396f3bd58",
32 | "sha256:4ce9ae9e91f46c344bec3b03d6ee9612802682c1551aaf627ad24045ce090761",
33 | "sha256:5d93d4e95eacd313d2c765ebe40d49ca9dd2ed90e5b37d0d421c597af830c195",
34 | "sha256:61b6760b08b7c395975d893e0b814a11cf011ebb24f7d869e7118f5a339a82e1",
35 | "sha256:72ccb318bf0c9ab97fc04c10c37683d9eea952ed526707fabf9ac5ae59b701fd",
36 | "sha256:79b485767c13788ee314669008d01f9ef3bc05db9ea3298f6a50d3ef596a154b",
37 | "sha256:7eb57ba90929bac0b6cc2af2373893d80ac559adda6933e562dcfb375029acee",
38 | "sha256:8bc541a405423ce0e51c19f637050acdbdf8feca34150e0d17f675e72d119580",
39 | "sha256:969dd06110cb780da01336b281f53e2e7eb3a482831df441fb65dd30403f4608",
40 | "sha256:985ceb5d0a86fcaa61e45781e567a59baa0da292d5ed2e490d612d0de5796918",
41 | "sha256:9bcf8b6e011be08fb729d110f3e22e654a50f8a826b0575c7196616780683380",
42 | "sha256:9ce157d979f742a915b75f792dbd6aa63b8eccaf46a1005ba03aa8a986bde34a",
43 | "sha256:9f659a5ee95c8baa2436d392267988fd0f43eb774e5eb8739252e5a7e9cf07e0",
44 | "sha256:a4a88dcd6ff8fd47c18b3a3709a89adb39a6373f4482e04c1b765045c7e282fd",
45 | "sha256:a955260d47f03df08acf45689bd163ed9df82c0e0124beb4251b1290fa7ae728",
46 | "sha256:a9af62e9b5b9bc67b2a195ebc2c2662fdf498a822d62f902bf27cccb52dbbf49",
47 | "sha256:ae72f8098acb368d877b210ebe02ba12585e77bd0db78ac04a1ee9b9f5dd2166",
48 | "sha256:b83ba3825bc91dfa989d4eed76865e71aea3a6ca1388b59fc801ee04c4d8d0d6",
49 | "sha256:c11951b404e08b01b151222a1cb1a9f0a860a8153ce8334149ab9199cd198131",
50 | "sha256:c320c64dd876e45254bdd350f0179da737463eea41c43bacbee9d8c9d1021f11",
51 | "sha256:c8098a724c2784bf03e8070993f6d46aa2eeca031f8d8a048dff277703e6e193",
52 | "sha256:d12f96b5b64bec3f43c8e82b4aab7599d0157f11c798c9f9c528a72b9e0b339a",
53 | "sha256:e565a785233c2d03724c4dc55464559639b1ba9ecf091288dd47ad9c629433bd",
54 | "sha256:f0f047e11febe5c3198ed346b507e1d010330d56ad615a7e0a89fae604065a0e",
55 | "sha256:fe4670cb32ea98ffbf5a1262f14c3e102cccd92b1869df3bb09538158ba90fe6"
56 | ],
57 | "index": "pypi",
58 | "version": "==1.9.1"
59 | },
60 | "typing-extensions": {
61 | "hashes": [
62 | "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
63 | "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
64 | ],
65 | "markers": "python_version >= '3.7'",
66 | "version": "==4.2.0"
67 | },
68 | "websocket-client": {
69 | "hashes": [
70 | "sha256:50b21db0058f7a953d67cc0445be4b948d7fc196ecbeb8083d68d94628e4abf6",
71 | "sha256:722b171be00f2b90e1d4fb2f2b53146a536ca38db1da8ff49c972a4e1365d0ef"
72 | ],
73 | "index": "pypi",
74 | "version": "==1.3.2"
75 | },
76 | "websockets": {
77 | "hashes": [
78 | "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af",
79 | "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c",
80 | "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76",
81 | "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47",
82 | "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69",
83 | "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079",
84 | "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c",
85 | "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55",
86 | "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02",
87 | "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559",
88 | "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3",
89 | "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e",
90 | "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978",
91 | "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98",
92 | "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae",
93 | "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755",
94 | "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d",
95 | "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991",
96 | "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1",
97 | "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680",
98 | "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247",
99 | "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f",
100 | "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2",
101 | "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7",
102 | "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4",
103 | "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667",
104 | "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb",
105 | "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094",
106 | "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36",
107 | "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79",
108 | "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500",
109 | "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e",
110 | "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582",
111 | "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442",
112 | "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd",
113 | "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6",
114 | "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731",
115 | "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4",
116 | "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d",
117 | "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8",
118 | "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f",
119 | "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677",
120 | "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8",
121 | "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9",
122 | "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e",
123 | "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b",
124 | "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916",
125 | "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"
126 | ],
127 | "index": "pypi",
128 | "version": "==10.3"
129 | }
130 | },
131 | "develop": {
132 | "mypy": {
133 | "hashes": [
134 | "sha256:006be38474216b833eca29ff6b73e143386f352e10e9c2fbe76aa8549e5554f5",
135 | "sha256:03c6cc893e7563e7b2949b969e63f02c000b32502a1b4d1314cabe391aa87d66",
136 | "sha256:0e9f70df36405c25cc530a86eeda1e0867863d9471fe76d1273c783df3d35c2e",
137 | "sha256:1ece702f29270ec6af25db8cf6185c04c02311c6bb21a69f423d40e527b75c56",
138 | "sha256:3e09f1f983a71d0672bbc97ae33ee3709d10c779beb613febc36805a6e28bb4e",
139 | "sha256:439c726a3b3da7ca84a0199a8ab444cd8896d95012c4a6c4a0d808e3147abf5d",
140 | "sha256:5a0b53747f713f490affdceef835d8f0cb7285187a6a44c33821b6d1f46ed813",
141 | "sha256:5f1332964963d4832a94bebc10f13d3279be3ce8f6c64da563d6ee6e2eeda932",
142 | "sha256:63e85a03770ebf403291ec50097954cc5caf2a9205c888ce3a61bd3f82e17569",
143 | "sha256:64759a273d590040a592e0f4186539858c948302c653c2eac840c7a3cd29e51b",
144 | "sha256:697540876638ce349b01b6786bc6094ccdaba88af446a9abb967293ce6eaa2b0",
145 | "sha256:9940e6916ed9371809b35b2154baf1f684acba935cd09928952310fbddaba648",
146 | "sha256:9f5f5a74085d9a81a1f9c78081d60a0040c3efb3f28e5c9912b900adf59a16e6",
147 | "sha256:a5ea0875a049de1b63b972456542f04643daf320d27dc592d7c3d9cd5d9bf950",
148 | "sha256:b117650592e1782819829605a193360a08aa99f1fc23d1d71e1a75a142dc7e15",
149 | "sha256:b24be97351084b11582fef18d79004b3e4db572219deee0212078f7cf6352723",
150 | "sha256:b88f784e9e35dcaa075519096dc947a388319cb86811b6af621e3523980f1c8a",
151 | "sha256:bdd5ca340beffb8c44cb9dc26697628d1b88c6bddf5c2f6eb308c46f269bb6f3",
152 | "sha256:d5aaf1edaa7692490f72bdb9fbd941fbf2e201713523bdb3f4038be0af8846c6",
153 | "sha256:e999229b9f3198c0c880d5e269f9f8129c8862451ce53a011326cad38b9ccd24",
154 | "sha256:f4a21d01fc0ba4e31d82f0fff195682e29f9401a8bdb7173891070eb260aeb3b",
155 | "sha256:f4b794db44168a4fc886e3450201365c9526a522c46ba089b55e1f11c163750d",
156 | "sha256:f730d56cb924d371c26b8eaddeea3cc07d78ff51c521c6d04899ac6904b75492"
157 | ],
158 | "index": "pypi",
159 | "version": "==0.961"
160 | },
161 | "mypy-extensions": {
162 | "hashes": [
163 | "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
164 | "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
165 | ],
166 | "version": "==0.4.3"
167 | },
168 | "tomli": {
169 | "hashes": [
170 | "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
171 | "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
172 | ],
173 | "markers": "python_version < '3.11'",
174 | "version": "==2.0.1"
175 | },
176 | "typing-extensions": {
177 | "hashes": [
178 | "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
179 | "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
180 | ],
181 | "markers": "python_version >= '3.7'",
182 | "version": "==4.2.0"
183 | }
184 | }
185 | }
186 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # CycleTLS Python
2 |
3 |
4 |
5 | Currently a WIP and in Active development
6 |
7 | ✔️ Initial Golang compilation/websocket connection
8 |
9 | ✖️ Full Pydantic models for all request types
10 |
11 | ✖️ Fix connection retry for WS server
12 |
13 | ✖️ Unit and integration tests
14 |
15 | ✖️ Documentation
16 |
--------------------------------------------------------------------------------
/RELEASE_PROCESS.md:
--------------------------------------------------------------------------------
1 | # GitHub Release Process
2 |
3 | ## Steps
4 |
5 | 1. Update the version in `my_package/version.py`.
6 |
7 | 3. Run the release script:
8 |
9 | ```bash
10 | ./scripts/release.sh
11 | ```
12 |
13 | This will commit the changes to the CHANGELOG and `version.py` files and then create a new tag in git
14 | which will trigger a workflow on GitHub Actions that handles the rest.
15 |
16 | ## Fixing a failed release
17 |
18 | If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete both the tag and corresponding release from GitHub. After you've pushed a fix, delete the tag from your local clone with
19 |
20 | ```bash
21 | git tag -l | xargs git tag -d && git fetch -t
22 | ```
23 |
24 | Then repeat the steps above.
25 |
--------------------------------------------------------------------------------
/build.sh:
--------------------------------------------------------------------------------
1 | go build -o ./dist/cycletls ./golang && chmod +x ./dist/cycletls
2 |
--------------------------------------------------------------------------------
/build/lib/cycletls/__init__.py:
--------------------------------------------------------------------------------
1 | from .api import CycleTLS
2 | from .schema import *
3 |
--------------------------------------------------------------------------------
/build/lib/cycletls/__version__.py:
--------------------------------------------------------------------------------
1 |
2 | __title__ = 'requests'
3 | __description__ = 'Python HTTP for Humans.'
4 | __url__ = 'https://requests.readthedocs.io'
5 | __version__ = '2.27.1'
6 | __build__ = 0x022701
7 | __author__ = 'Kenneth Reitz'
8 | __author_email__ = 'me@kennethreitz.org'
9 | __license__ = 'Apache 2.0'
10 | __copyright__ = 'Copyright 2022 Kenneth Reitz'
11 | __cake__ = u'\u2728 \U0001f370 \u2728'
--------------------------------------------------------------------------------
/build/lib/cycletls/api.py:
--------------------------------------------------------------------------------
1 | import json
2 | from websocket import create_connection
3 | from .schema import Response, Request
4 | import subprocess
5 | from time import sleep
6 | import psutil
7 |
8 |
9 | def kill(proc_pid):
10 | if proc_pid:
11 | process = psutil.Process(proc_pid.pid)
12 | for proc in process.children(recursive=True):
13 | proc.kill()
14 | process.kill()
15 | else:
16 | for proc in psutil.process_iter():
17 | # check whether the process name matches
18 | if proc.name() == "cycletls":
19 | proc.kill()
20 |
21 |
22 | class CycleTLS:
23 | def __init__(self):
24 | try:
25 | self.ws = create_connection("ws://localhost:8080")
26 | self.proc = None
27 | except:
28 |
29 | self.proc = subprocess.Popen(["./dist/cycletls"], shell=True)
30 | # TODO remove this
31 | sleep(0.1)
32 |
33 | self.ws = create_connection("ws://localhost:8080")
34 |
35 | def request(self, method, url, **kwargs):
36 | request = Request(method=method, url=url, **kwargs)
37 | request = {
38 | "requestId": "requestId",
39 | "options": request.dict(by_alias=True, exclude_none=True),
40 | }
41 | self.ws.send(json.dumps(request))
42 | response = json.loads(self.ws.recv())
43 |
44 | return Response(**response)
45 |
46 | def get(self, url, params=None, **kwargs) -> Response:
47 | """Sends an GET request.
48 | Args:
49 | url (str): URL for the new :class:`Request` object.
50 | params (dict): (optional) Dictionary, list of tuples or bytes to send
51 | in the query string for the :class:`Request`.
52 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
53 | object to send in the body of the :class:`Request`.
54 | url (str): URL for the new :class:`Request` object.
55 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
56 | ja3 (str): JA3 token to send with :class:`Request`.
57 | user_agent (str): User Agent to send with :class:`Request`.
58 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
59 | cookies (dict): Dict object to send with the :class:`Request`.
60 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
61 | header_order (Optional[list]): Optional list setting request header order.
62 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
63 | Returns:
64 | Response: Response object with (request_id, status_code, headers, body)
65 | properties.
66 | """
67 | return self.request("get", url, params=params, **kwargs)
68 |
69 | def options(self, url, params=None, **kwargs) -> Response:
70 | """Sends an OPTIONS request.
71 | Args:
72 | url (str): URL for the new :class:`Request` object.
73 | params (dict): (optional) Dictionary, list of tuples or bytes to send
74 | in the query string for the :class:`Request`.
75 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
76 | object to send in the body of the :class:`Request`.
77 | url (str): URL for the new :class:`Request` object.
78 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
79 | ja3 (str): JA3 token to send with :class:`Request`.
80 | user_agent (str): User Agent to send with :class:`Request`.
81 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
82 | cookies (dict): Dict object to send with the :class:`Request`.
83 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
84 | header_order (Optional[list]): Optional list setting request header order.
85 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
86 | Returns:
87 | Response: Response object with (request_id, status_code, headers, body)
88 | properties.
89 | """
90 | return self.request("options", url, params=params, **kwargs)
91 |
92 | def head(self, url, params=None, **kwargs) -> Response:
93 | """Sends an HEAD request.
94 | Args:
95 | url (str): URL for the new :class:`Request` object.
96 | params (dict): (optional) Dictionary, list of tuples or bytes to send
97 | in the query string for the :class:`Request`.
98 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
99 | object to send in the body of the :class:`Request`.
100 | url (str): URL for the new :class:`Request` object.
101 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
102 | ja3 (str): JA3 token to send with :class:`Request`.
103 | user_agent (str): User Agent to send with :class:`Request`.
104 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
105 | cookies (dict): Dict object to send with the :class:`Request`.
106 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
107 | header_order (Optional[list]): Optional list setting request header order.
108 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
109 | Returns:
110 | Response: Response object with (request_id, status_code, headers, body)
111 | properties.
112 | """
113 | return self.request("head", url, params=params, **kwargs)
114 |
115 | def post(self, url, params=None, **kwargs) -> Response:
116 | """Sends an POST request.
117 | Args:
118 | url (str): URL for the new :class:`Request` object.
119 | params (dict): (optional) Dictionary, list of tuples or bytes to send
120 | in the query string for the :class:`Request`.
121 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
122 | object to send in the body of the :class:`Request`.
123 | url (str): URL for the new :class:`Request` object.
124 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
125 | ja3 (str): JA3 token to send with :class:`Request`.
126 | user_agent (str): User Agent to send with :class:`Request`.
127 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
128 | cookies (dict): Dict object to send with the :class:`Request`.
129 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
130 | header_order (Optional[list]): Optional list setting request header order.
131 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
132 | Returns:
133 | Response: Response object with (request_id, status_code, headers, body)
134 | properties.
135 | """
136 | return self.request("post", url, params=params, **kwargs)
137 |
138 | def put(self, url, params=None, **kwargs) -> Response:
139 | """Sends an PUT request.
140 | Args:
141 | url (str): URL for the new :class:`Request` object.
142 | params (dict): (optional) Dictionary, list of tuples or bytes to send
143 | in the query string for the :class:`Request`.
144 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
145 | object to send in the body of the :class:`Request`.
146 | url (str): URL for the new :class:`Request` object.
147 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
148 | ja3 (str): JA3 token to send with :class:`Request`.
149 | user_agent (str): User Agent to send with :class:`Request`.
150 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
151 | cookies (dict): Dict object to send with the :class:`Request`.
152 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
153 | header_order (Optional[list]): Optional list setting request header order.
154 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
155 | Returns:
156 | Response: Response object with (request_id, status_code, headers, body)
157 | properties.
158 | """
159 | return self.request("put", url, params=params, **kwargs)
160 |
161 | def patch(self, url, params=None, **kwargs) -> Response:
162 | """Sends an PATCH request.
163 | Args:
164 | url (str): URL for the new :class:`Request` object.
165 | params (dict): (optional) Dictionary, list of tuples or bytes to send
166 | in the query string for the :class:`Request`.
167 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
168 | object to send in the body of the :class:`Request`.
169 | url (str): URL for the new :class:`Request` object.
170 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
171 | ja3 (str): JA3 token to send with :class:`Request`.
172 | user_agent (str): User Agent to send with :class:`Request`.
173 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
174 | cookies (dict): Dict object to send with the :class:`Request`.
175 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
176 | header_order (Optional[list]): Optional list setting request header order.
177 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
178 | Returns:
179 | Response: Response object with (request_id, status_code, headers, body)
180 | properties.
181 | """
182 | return self.request("patch", url, params=params, **kwargs)
183 |
184 | def delete(self, url, params=None, **kwargs) -> Response:
185 | """Sends an DELETE request.
186 | Args:
187 | url (str): URL for the new :class:`Request` object.
188 | params (dict): (optional) Dictionary, list of tuples or bytes to send
189 | in the query string for the :class:`Request`.
190 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
191 | object to send in the body of the :class:`Request`.
192 | url (str): URL for the new :class:`Request` object.
193 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
194 | ja3 (str): JA3 token to send with :class:`Request`.
195 | user_agent (str): User Agent to send with :class:`Request`.
196 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
197 | cookies (dict): Dict object to send with the :class:`Request`.
198 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
199 | header_order (Optional[list]): Optional list setting request header order.
200 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
201 | Returns:
202 | Response: Response object with (request_id, status_code, headers, body)
203 | properties.
204 | """
205 | return self.request("delete", url, params=params, **kwargs)
206 |
207 | def close(self):
208 | self.ws.close()
209 |
210 | kill(self.proc)
211 |
--------------------------------------------------------------------------------
/build/lib/cycletls/schema.py:
--------------------------------------------------------------------------------
1 | from pydantic import BaseModel
2 | import json
3 | from typing import Optional
4 |
5 | class Cookie(BaseModel):
6 | test: int
7 |
8 |
9 | class Request(BaseModel):
10 | url: str
11 | method: str
12 | body: str = ""
13 | headers: dict = {}
14 | ja3: str = "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0"
15 | user_agent: str = (
16 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0"
17 | )
18 | proxy: str = ""
19 | cookies: Optional[list[Cookie]] = None
20 | timeout: int = 6
21 | disable_redirect: bool = False
22 | header_order: Optional[list] = None
23 | order_headers_as_provided: Optional[bool] = None
24 |
25 | class Config:
26 | fields = {
27 | "user_agent": "userAgent",
28 | "disable_redirect": "disableRedirect",
29 | }
30 |
31 |
32 | class Response(BaseModel):
33 | request_id: str
34 | status_code: int
35 | headers: dict
36 | body: str
37 |
38 | class Config:
39 | fields = {
40 | "request_id": "RequestID",
41 | "status_code": "Status",
42 | "headers": "Headers",
43 | "body": "Body",
44 | }
45 |
46 | def json(self) -> dict:
47 | return json.loads(self.body)
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/client.py:
--------------------------------------------------------------------------------
1 | from cycletls import CycleTLS
2 |
3 |
4 |
5 | cycle = CycleTLS()
6 | result = cycle.get("https://ja3er.com/json")
7 | print(result)
8 | cycle.close()
9 |
10 |
--------------------------------------------------------------------------------
/cycletls.egg-info/PKG-INFO:
--------------------------------------------------------------------------------
1 | Metadata-Version: 2.1
2 | Name: cycletls
3 | Version: 0.0.1
4 | Summary: A python package for spoofing TLS
5 | Home-page: https://github.com/Danny-Dasilva/cycletls_python
6 | Author: Danny-Dasilva
7 | Author-email: dannydasilva.solutions@gmail.com
8 | License: none
9 | Description: # CycleTLS Python
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | Currently a WIP and in Active development. See the  Tab for more info
18 |
19 |
20 |
21 |
22 | 
23 | [](http://godoc.org/github.com/Danny-Dasilva/CycleTLS/cycletls)
24 | [](https://github.com/Danny-Dasilva/CycleTLS/blob/main/LICENSE)
25 | [](https://goreportcard.com/report/github.com/Danny-Dasilva/CycleTLS/cycletls)
26 | [](https://www.npmjs.org/package/cycletls)
27 |
28 |
29 | If you have a API change or feature request feel free to open an Issue
30 |
31 |
32 |
33 | # 🚀 Features
34 |
35 | - [High-performance](#-performance) Built-in goroutine pool used for handling asynchronous requests
36 | - Custom header ordering via fhttp
37 | - Proxy support
38 | - Ja3 Token configuration
39 | - Request Redirection toggle
40 |
41 |
42 | Table of contents
43 | =================
44 |
45 |
46 | * [Table of contents](#table-of-contents)
47 | * [Installation](#installation)
48 | * [Usage](#usage)
49 | * [QuickStart JS](#example-cycletls-request-for-typescript-and-javascript)
50 | * [Quickstart Golang](#example-cycletls-request-for-golang)
51 | * [Initializing CycleTLS](#creating-an-instance)
52 | * [API/Methods](#cycletls-alias-methods)
53 | * [Request Config](#cycletls-request-config)
54 | * [Response Schema](#cycletls-response-schema)
55 | * [Multiple Requests Example](#multiple-requests-example-for-typescript-and-javascript)
56 | * [Dev Setup](#dev-setup)
57 | * [LICENSE](#license)
58 |
59 |
60 |
61 | For any feature requests or API change requests, please feel free to open an issue.
62 |
63 |
64 | ## Dependencies
65 |
66 | ```
67 | node ^v14.0
68 | golang ^v1.16x
69 | ```
70 |
71 | ## Installation
72 |
73 | ```bash
74 | $ npm install cycletls
75 | ```
76 |
77 | # Usage
78 |
79 | ## Example CycleTLS Request for Typescript and Javascript
80 |
81 | You can run this test in `tests/simple.test.ts`
82 |
83 | ```js
84 |
85 | const initCycleTLS = require('cycletls');
86 | // Typescript: import initCycleTLS from 'cycletls';
87 |
88 | (async () => {
89 | // Initiate CycleTLS
90 | const cycleTLS = await initCycleTLS();
91 |
92 | // Send request
93 | const response = await cycleTLS('https://ja3er.com/json', {
94 | body: '',
95 | ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
96 | userAgent: 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0',
97 | proxy: 'http://username:password@hostname.com:443'
98 | }, 'get');
99 |
100 | console.log(response);
101 |
102 | // Cleanly exit CycleTLS
103 | cycleTLS.exit();
104 |
105 | })();
106 |
107 | ```
108 |
109 | ## Example CycleTLS Request for Golang
110 |
111 | ```go
112 | package main
113 |
114 | import (
115 | "log"
116 | "github.com/Danny-Dasilva/CycleTLS/cycletls"
117 | )
118 |
119 | func main() {
120 |
121 | client := cycletls.Init()
122 |
123 | response, err := client.Do("https://ja3er.com/json", cycletls.Options{
124 | Body : "",
125 | Ja3: "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0",
126 | UserAgent: "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
127 | }, "GET");
128 | if err != nil {
129 | log.Print("Request Failed: " + err.Error())
130 | }
131 | log.Println(response)
132 | }
133 |
134 | ```
135 |
136 | ## Creating an instance
137 |
138 | In order to create a `cycleTLS` instance, you can run the following:
139 |
140 | #### JavaScript
141 |
142 | ```js
143 | // The initCycleTLS function spawns a Golang process that handles all requests concurrently via goroutine loops.
144 | const initCycleTLS = require('cycletls');
145 | // import initCycleTLS from 'cycletls';
146 |
147 | // Async/Await method
148 | const cycleTLS = await initCycleTLS();
149 | // .then method
150 | initCycleTLS().then((cycleTLS) => {});
151 |
152 | ```
153 | #### Golang
154 |
155 | ```go
156 | import (
157 | "github.com/Danny-Dasilva/CycleTLS/cycletls"
158 | )
159 |
160 | //The `Init` function initializes golang channels to process requests.
161 | client := cycletls.Init()
162 | ```
163 |
164 |
165 | ## CycleTLS Alias Methods
166 |
167 | The following methods exist in CycleTLS
168 |
169 | **cycleTLS([url], config)**
170 |
171 | **cycleTLS.get([url], config)**
172 |
173 | **cycleTLS.delete([url], config)**
174 |
175 | **cycleTLS.head([url], config)**
176 |
177 | **cycleTLS.options([url], config)**
178 |
179 | **cycleTLS.post([url], config)**
180 |
181 | **cycleTLS.put([url], config)**
182 |
183 | **cycleTLS.patch([url], config)**
184 |
185 | If URL is not passed, one must be specified in the config.
186 |
187 | ## CycleTLS Request Config
188 |
189 | ```js
190 | {
191 | // URL for the request (required if not specified as an argument)
192 | url: "https://example.com"
193 | // Method for the request ("head" | "get" | "post" | "put" | "delete" | "trace" | "options" | "connect" | "patch")
194 | method: "get" // Default method
195 | // Custom headers to send
196 | headers: { "Authorization": "Bearer someexampletoken" }
197 | // Custom cookies to send
198 | Cookies: [{
199 | "name": "key",
200 | "value": "val",
201 | "path": "/docs",
202 | "domain": "google.com",
203 | "expires": "Mon, 02-Jan-2022 15:04:05 EST"
204 | "maxAge": 90,
205 | "secure": false,
206 | "httpOnly": true,
207 | "sameSite": "Lax"
208 | }],
209 | // Body to send with request (must be a string - cannot pass an object)
210 | body: '',
211 | // JA3 token to send with request
212 | ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
213 | // User agent for request
214 | userAgent: 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0',
215 | // Proxy to send request through (must be in the same format)
216 | proxy: 'http://username:password@hostname.com:443',
217 | // Amount of seconds before request timeout (default: 7)
218 | timeout: 2,
219 | // Toggle if CycleTLS should follow redirects
220 | disableRedirect: true
221 | // Custom header order to send with request (This value will overwrite default header order)
222 | headerOrder: ["cache-control", "connection", "host"]
223 | }
224 | );
225 |
226 | ```
227 |
228 | ## CycleTLS Response Schema
229 |
230 | ```js
231 | {
232 | // Status code returned from server (Number)
233 | status: 200,
234 | // Body returned from the server (String)
235 | body: "",
236 | // Headers returned from the server (Object)
237 | headers: {
238 | "some": "header",
239 | ...
240 | }
241 | }
242 | );
243 |
244 | ```
245 |
246 |
247 |
248 | ## Multiple Requests Example for Typescript and Javascript
249 |
250 | If CycleTLS is being used by in a JavaScript environment, CycleTLS will spawn a Golang process to handle requests. This Golang process handles requests `concurrently` in a worker pool. Due to this, CycleTLS returns response objects as soon as they are made available
251 | (in other terms, CycleTLS processes requests as they are received, but responses are returned asynchronously so they will NOT be returned in the order requested)
252 |
253 | If you are using CycleTLS in JavaScript, it is necessary to exit out of the instance to prevent zombie processes. The example below shows one way to approach cleanly exiting CycleTLS if you need to process multiple requests (note: keep in mind that calling the `exit()` function will kill any requests in progress). If your workflow requires requests running the entire time the process runs, modules such as [exit-hook](https://www.npmjs.com/package/exit-hook) could serve as an alternative solution to cleanly exiting CycleTLS.
254 |
255 | ```js
256 | const initCycleTLS = require("cycletls");
257 | // Typescript: import initCycleTLS from 'cycletls';
258 |
259 | // Defining custom JA3 token and user agenton multiple requests,
260 | "https://httpbin.org/user-agent": {
261 | ja3: ja3,
262 | userAgent: userAgent,
263 | },
264 | "http://httpbin.org/post": {
265 | body: '{"field":"POST-VAL"}',
266 | method: "POST",
267 | },
268 | "http://httpbin.org/cookies": {
269 | cookies: [
270 | {
271 | name: "example1",
272 | value: "aaaaaaa",
273 | expires: "Mon, 02-Jan-2022 15:04:05 EST",
274 | },
275 | ],
276 | },
277 | };
278 |
279 | // Promises array of requests
280 | const promises = [];
281 |
282 | // Anonymous async function
283 | (async () => {
284 | // Initiate CycleTLS
285 | const cycleTLS = await initCycleTLS();
286 |
287 | // Loop through requestDict (Object) defined above
288 | for (const url in requestDict) {
289 | // Fetch configs from requestDict (Object)
290 | const params = requestDict[url];
291 |
292 | // Send request (note: no waiting)
293 | const response = cycleTLS(
294 | url, {
295 | body: params.body ?? "", //?? is just setting defaults in this case
296 | ja3: params.ja3 ?? ja3,
297 | userAgent: params.userAgent ?? userAgent,
298 | headers: params.headers,
299 | cookies: params.cookies,
300 | }, params.method ?? "GET");
301 |
302 | // console.log the response object
303 | response.then((out) => {
304 | console.log(url, out);
305 | });
306 |
307 | // Push request to promise array
308 | promises.push(response);
309 | }
310 |
311 | // Wait for all requests to execute successfully
312 | Promise.all(promises).then(() => {
313 | // Cleanly exit CycleTLS one all requests have been received
314 | cycleTLS.exit();
315 | });
316 | })();
317 | ```
318 |
319 |
320 |
321 | # Dev Setup
322 |
323 | If you would like to compile CycleTLS on your own, use the following commands:
324 |
325 | Set module-aware mode
326 |
327 | `go env -w GO111MODULE=auto`
328 |
329 | Install golang dependencies
330 |
331 | `go get github.com/Danny-Dasilva/CycleTLS/cycletls`
332 |
333 | install npm packages
334 |
335 | `npm install`
336 |
337 | ### To recompile index.ts in the src folder
338 |
339 | `npm run build`
340 |
341 | ### To recompile Golang files in the golang folder
342 | Windows
343 |
344 | `npm run build:windows`
345 |
346 | Linux
347 |
348 | `npm run build:linux`
349 |
350 | Mac
351 |
352 | `npm run build:mac:`
353 |
354 |
355 | ## LICENSE
356 | ### GPL3 LICENSE SYNOPSIS
357 |
358 | **_TL;DR_*** Here's what the GPL3 license entails:
359 |
360 | ```markdown
361 | 1. Anyone can copy, modify and distribute this software.
362 | 2. You have to include the license and copyright notice with each and every distribution.
363 | 3. You can use this software privately.
364 | 4. You can use this software for commercial purposes.
365 | 5. Source code MUST be made available when the software is distributed.
366 | 6. Any modifications of this code base MUST be distributed with the same license, GPLv3.
367 | 7. This software is provided without warranty.
368 | 8. The software author or license can not be held liable for any damages inflicted by the software.
369 | ```
370 |
371 | More information on about the [LICENSE can be found here](http://choosealicense.com/licenses/gpl-3.0/)
372 | Platform: UNKNOWN
373 | Description-Content-Type: text/markdown
374 |
--------------------------------------------------------------------------------
/cycletls.egg-info/SOURCES.txt:
--------------------------------------------------------------------------------
1 | MANIFEST.in
2 | README.md
3 | setup.py
4 | cycletls/__init__.py
5 | cycletls/__version__.py
6 | cycletls/api.py
7 | cycletls/schema.py
8 | cycletls.egg-info/PKG-INFO
9 | cycletls.egg-info/SOURCES.txt
10 | cycletls.egg-info/dependency_links.txt
11 | cycletls.egg-info/top_level.txt
--------------------------------------------------------------------------------
/cycletls.egg-info/dependency_links.txt:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/cycletls.egg-info/top_level.txt:
--------------------------------------------------------------------------------
1 | cycletls
2 |
--------------------------------------------------------------------------------
/cycletls/__init__.py:
--------------------------------------------------------------------------------
1 | from .api import CycleTLS
2 | from .schema import *
3 |
--------------------------------------------------------------------------------
/cycletls/__version__.py:
--------------------------------------------------------------------------------
1 |
2 | __title__ = 'requests'
3 | __description__ = 'Python HTTP for Humans.'
4 | __url__ = 'https://requests.readthedocs.io'
5 | __version__ = '0.0.1'
6 | __build__ = 0x022701
7 | __author__ = 'Danny Dasilva'
8 | __author_email__ = 'dannydasilva.solutions@gmail.com'
9 | __license__ = ' GNU General Public License v3 (GPLv3)'
10 | __copyright__ = 'Copyright 2022 Danny Dasilva'
11 | __cake__ = u'\u2728 \U0001f370 \u2728'
12 |
--------------------------------------------------------------------------------
/cycletls/api.py:
--------------------------------------------------------------------------------
1 | import json
2 | from websocket import create_connection
3 | from .schema import Response, Request
4 | import subprocess
5 | from time import sleep
6 | import psutil
7 |
8 |
9 | def kill(proc_pid):
10 | if proc_pid:
11 | process = psutil.Process(proc_pid.pid)
12 | for proc in process.children(recursive=True):
13 | proc.kill()
14 | process.kill()
15 | else:
16 | for proc in psutil.process_iter():
17 | # check whether the process name matches
18 | if proc.name() == "cycletls":
19 | proc.kill()
20 |
21 |
22 | class CycleTLS:
23 | def __init__(self):
24 | try:
25 | self.ws = create_connection("ws://localhost:8080")
26 | self.proc = None
27 | except:
28 |
29 | self.proc = subprocess.Popen(["./dist/cycletls"], shell=True)
30 | # TODO remove this
31 | sleep(0.1)
32 |
33 | self.ws = create_connection("ws://localhost:8080")
34 |
35 | def request(self, method, url, **kwargs):
36 | request = Request(method=method, url=url, **kwargs)
37 | request = {
38 | "requestId": "requestId",
39 | "options": request.dict(by_alias=True, exclude_none=True),
40 | }
41 | self.ws.send(json.dumps(request))
42 | response = json.loads(self.ws.recv())
43 |
44 | return Response(**response)
45 |
46 | def get(self, url, params=None, **kwargs) -> Response:
47 | """Sends an GET request.
48 | Args:
49 | url (str): URL for the new :class:`Request` object.
50 | params (dict): (optional) Dictionary, list of tuples or bytes to send
51 | in the query string for the :class:`Request`.
52 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
53 | object to send in the body of the :class:`Request`.
54 | url (str): URL for the new :class:`Request` object.
55 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
56 | ja3 (str): JA3 token to send with :class:`Request`.
57 | user_agent (str): User Agent to send with :class:`Request`.
58 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
59 | cookies (dict): Dict object to send with the :class:`Request`.
60 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
61 | header_order (Optional[list]): Optional list setting request header order.
62 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
63 | Returns:
64 | Response: Response object with (request_id, status_code, headers, body)
65 | properties.
66 | """
67 | return self.request("get", url, params=params, **kwargs)
68 |
69 | def options(self, url, params=None, **kwargs) -> Response:
70 | """Sends an OPTIONS request.
71 | Args:
72 | url (str): URL for the new :class:`Request` object.
73 | params (dict): (optional) Dictionary, list of tuples or bytes to send
74 | in the query string for the :class:`Request`.
75 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
76 | object to send in the body of the :class:`Request`.
77 | url (str): URL for the new :class:`Request` object.
78 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
79 | ja3 (str): JA3 token to send with :class:`Request`.
80 | user_agent (str): User Agent to send with :class:`Request`.
81 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
82 | cookies (dict): Dict object to send with the :class:`Request`.
83 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
84 | header_order (Optional[list]): Optional list setting request header order.
85 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
86 | Returns:
87 | Response: Response object with (request_id, status_code, headers, body)
88 | properties.
89 | """
90 | return self.request("options", url, params=params, **kwargs)
91 |
92 | def head(self, url, params=None, **kwargs) -> Response:
93 | """Sends an HEAD request.
94 | Args:
95 | url (str): URL for the new :class:`Request` object.
96 | params (dict): (optional) Dictionary, list of tuples or bytes to send
97 | in the query string for the :class:`Request`.
98 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
99 | object to send in the body of the :class:`Request`.
100 | url (str): URL for the new :class:`Request` object.
101 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
102 | ja3 (str): JA3 token to send with :class:`Request`.
103 | user_agent (str): User Agent to send with :class:`Request`.
104 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
105 | cookies (dict): Dict object to send with the :class:`Request`.
106 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
107 | header_order (Optional[list]): Optional list setting request header order.
108 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
109 | Returns:
110 | Response: Response object with (request_id, status_code, headers, body)
111 | properties.
112 | """
113 | return self.request("head", url, params=params, **kwargs)
114 |
115 | def post(self, url, params=None, **kwargs) -> Response:
116 | """Sends an POST request.
117 | Args:
118 | url (str): URL for the new :class:`Request` object.
119 | params (dict): (optional) Dictionary, list of tuples or bytes to send
120 | in the query string for the :class:`Request`.
121 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
122 | object to send in the body of the :class:`Request`.
123 | url (str): URL for the new :class:`Request` object.
124 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
125 | ja3 (str): JA3 token to send with :class:`Request`.
126 | user_agent (str): User Agent to send with :class:`Request`.
127 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
128 | cookies (dict): Dict object to send with the :class:`Request`.
129 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
130 | header_order (Optional[list]): Optional list setting request header order.
131 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
132 | Returns:
133 | Response: Response object with (request_id, status_code, headers, body)
134 | properties.
135 | """
136 | return self.request("post", url, params=params, **kwargs)
137 |
138 | def put(self, url, params=None, **kwargs) -> Response:
139 | """Sends an PUT request.
140 | Args:
141 | url (str): URL for the new :class:`Request` object.
142 | params (dict): (optional) Dictionary, list of tuples or bytes to send
143 | in the query string for the :class:`Request`.
144 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
145 | object to send in the body of the :class:`Request`.
146 | url (str): URL for the new :class:`Request` object.
147 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
148 | ja3 (str): JA3 token to send with :class:`Request`.
149 | user_agent (str): User Agent to send with :class:`Request`.
150 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
151 | cookies (dict): Dict object to send with the :class:`Request`.
152 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
153 | header_order (Optional[list]): Optional list setting request header order.
154 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
155 | Returns:
156 | Response: Response object with (request_id, status_code, headers, body)
157 | properties.
158 | """
159 | return self.request("put", url, params=params, **kwargs)
160 |
161 | def patch(self, url, params=None, **kwargs) -> Response:
162 | """Sends an PATCH request.
163 | Args:
164 | url (str): URL for the new :class:`Request` object.
165 | params (dict): (optional) Dictionary, list of tuples or bytes to send
166 | in the query string for the :class:`Request`.
167 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
168 | object to send in the body of the :class:`Request`.
169 | url (str): URL for the new :class:`Request` object.
170 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
171 | ja3 (str): JA3 token to send with :class:`Request`.
172 | user_agent (str): User Agent to send with :class:`Request`.
173 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
174 | cookies (dict): Dict object to send with the :class:`Request`.
175 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
176 | header_order (Optional[list]): Optional list setting request header order.
177 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
178 | Returns:
179 | Response: Response object with (request_id, status_code, headers, body)
180 | properties.
181 | """
182 | return self.request("patch", url, params=params, **kwargs)
183 |
184 | def delete(self, url, params=None, **kwargs) -> Response:
185 | """Sends an DELETE request.
186 | Args:
187 | url (str): URL for the new :class:`Request` object.
188 | params (dict): (optional) Dictionary, list of tuples or bytes to send
189 | in the query string for the :class:`Request`.
190 | body (Optional[str]): Dictionary, list of tuples, bytes, or file-like
191 | object to send in the body of the :class:`Request`.
192 | url (str): URL for the new :class:`Request` object.
193 | headers (dict): Dictionary of HTTP Headers to send with the :class:`Request`.
194 | ja3 (str): JA3 token to send with :class:`Request`.
195 | user_agent (str): User Agent to send with :class:`Request`.
196 | proxy (str): Proxy to send request through must be in format `http://username:password@hostname.com:443`
197 | cookies (dict): Dict object to send with the :class:`Request`.
198 | disable_redirect (bool): Disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``False``.
199 | header_order (Optional[list]): Optional list setting request header order.
200 | order_headers_as_provided (Optional[bool]): Set header_order based on provided headers
201 | Returns:
202 | Response: Response object with (request_id, status_code, headers, body)
203 | properties.
204 | """
205 | return self.request("delete", url, params=params, **kwargs)
206 |
207 | def close(self):
208 | self.ws.close()
209 |
210 | kill(self.proc)
211 |
--------------------------------------------------------------------------------
/cycletls/schema.py:
--------------------------------------------------------------------------------
1 | from pydantic import BaseModel
2 | import json
3 | from typing import Optional, List
4 |
5 | class Cookie(BaseModel):
6 | #TODO
7 | test: int
8 |
9 |
10 | class Request(BaseModel):
11 | url: str
12 | method: str
13 | body: str = ""
14 | headers: dict = {}
15 | ja3: str = "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0"
16 | user_agent: str = (
17 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0"
18 | )
19 | proxy: str = ""
20 | cookies: Optional[List] = None
21 | timeout: int = 6
22 | disable_redirect: bool = False
23 | header_order: Optional[List] = None
24 | order_headers_as_provided: Optional[bool] = None
25 |
26 | class Config:
27 | fields = {
28 | "user_agent": "userAgent",
29 | "disable_redirect": "disableRedirect",
30 | }
31 |
32 |
33 | class Response(BaseModel):
34 | request_id: str
35 | status_code: int
36 | headers: dict
37 | body: str
38 |
39 | class Config:
40 | fields = {
41 | "request_id": "RequestID",
42 | "status_code": "Status",
43 | "headers": "Headers",
44 | "body": "Body",
45 | }
46 |
47 | def json(self) -> dict:
48 | return json.loads(self.body)
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/dist/cycletls-0.0.1-py3-none-any.whl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Danny-Dasilva/cycletls_python/43d8db8aa5e31ead0244397eac0014e7ad7b582f/dist/cycletls-0.0.1-py3-none-any.whl
--------------------------------------------------------------------------------
/dist/cycletls-0.0.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Danny-Dasilva/cycletls_python/43d8db8aa5e31ead0244397eac0014e7ad7b582f/dist/cycletls-0.0.1.tar.gz
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Minimal makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line, and also
5 | # from the environment for the first two.
6 | SPHINXOPTS ?=
7 | SPHINXBUILD ?= sphinx-build
8 | SOURCEDIR = source
9 | BUILDDIR = build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 |
--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | pushd %~dp0
4 |
5 | REM Command file for Sphinx documentation
6 |
7 | if "%SPHINXBUILD%" == "" (
8 | set SPHINXBUILD=sphinx-build
9 | )
10 | set SOURCEDIR=source
11 | set BUILDDIR=build
12 |
13 | if "%1" == "" goto help
14 |
15 | %SPHINXBUILD% >NUL 2>NUL
16 | if errorlevel 9009 (
17 | echo.
18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
19 | echo.installed, then set the SPHINXBUILD environment variable to point
20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you
21 | echo.may add the Sphinx directory to PATH.
22 | echo.
23 | echo.If you don't have Sphinx installed, grab it from
24 | echo.https://www.sphinx-doc.org/
25 | exit /b 1
26 | )
27 |
28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
29 | goto end
30 |
31 | :help
32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
33 |
34 | :end
35 | popd
36 |
--------------------------------------------------------------------------------
/docs/source/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | ../../CHANGELOG.md
--------------------------------------------------------------------------------
/docs/source/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | ../../CONTRIBUTING.md
--------------------------------------------------------------------------------
/docs/source/_static/css/custom.css:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Danny-Dasilva/cycletls_python/43d8db8aa5e31ead0244397eac0014e7ad7b582f/docs/source/_static/css/custom.css
--------------------------------------------------------------------------------
/docs/source/_static/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Danny-Dasilva/cycletls_python/43d8db8aa5e31ead0244397eac0014e7ad7b582f/docs/source/_static/favicon.ico
--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
1 | # Configuration file for the Sphinx documentation builder.
2 | #
3 | # This file only contains a selection of the most common options. For a full
4 | # list see the documentation:
5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html
6 |
7 | import os
8 | import sys
9 | from datetime import datetime
10 |
11 | # -- Path setup --------------------------------------------------------------
12 |
13 | # If extensions (or modules to document with autodoc) are in another directory,
14 | # add these directories to sys.path here. If the directory is relative to the
15 | # documentation root, use os.path.abspath to make it absolute, like shown here.
16 | #
17 |
18 | sys.path.insert(0, os.path.abspath("../../"))
19 |
20 | from my_package.version import VERSION, VERSION_SHORT # noqa: E402
21 |
22 | # -- Project information -----------------------------------------------------
23 |
24 | project = "my-package"
25 | copyright = f"{datetime.today().year}, Allen Institute for Artificial Intelligence"
26 | author = "Allen Institute for Artificial Intelligence"
27 | version = VERSION_SHORT
28 | release = VERSION
29 |
30 |
31 | # -- General configuration ---------------------------------------------------
32 |
33 | # Add any Sphinx extension module names here, as strings. They can be
34 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
35 | # ones.
36 | extensions = [
37 | "sphinx.ext.autodoc",
38 | "sphinx.ext.napoleon",
39 | "myst_parser",
40 | "sphinx.ext.intersphinx",
41 | "sphinx.ext.viewcode",
42 | "sphinx.ext.doctest",
43 | "sphinx_copybutton",
44 | "sphinx_autodoc_typehints",
45 | ]
46 |
47 | # Tell myst-parser to assign header anchors for h1-h3.
48 | myst_heading_anchors = 3
49 |
50 | suppress_warnings = ["myst.header"]
51 |
52 | # Add any paths that contain templates here, relative to this directory.
53 | templates_path = ["_templates"]
54 |
55 | # List of patterns, relative to source directory, that match files and
56 | # directories to ignore when looking for source files.
57 | # This pattern also affects html_static_path and html_extra_path.
58 | exclude_patterns = ["_build"]
59 |
60 | source_suffix = [".rst", ".md"]
61 |
62 | intersphinx_mapping = {
63 | "python": ("https://docs.python.org/3", None),
64 | # Uncomment these if you use them in your codebase:
65 | # "torch": ("https://pytorch.org/docs/stable", None),
66 | # "datasets": ("https://huggingface.co/docs/datasets/master/en", None),
67 | # "transformers": ("https://huggingface.co/docs/transformers/master/en", None),
68 | }
69 |
70 | # By default, sort documented members by type within classes and modules.
71 | autodoc_member_order = "groupwise"
72 |
73 | # Include default values when documenting parameter types.
74 | typehints_defaults = "comma"
75 |
76 |
77 | # -- Options for HTML output -------------------------------------------------
78 |
79 | # The theme to use for HTML and HTML Help pages. See the documentation for
80 | # a list of builtin themes.
81 | #
82 | html_theme = "furo"
83 |
84 | html_title = f"my-package v{VERSION}"
85 |
86 | # Add any paths that contain custom static files (such as style sheets) here,
87 | # relative to this directory. They are copied after the builtin static files,
88 | # so a file named "default.css" will overwrite the builtin "default.css".
89 | html_static_path = ["_static"]
90 |
91 | html_css_files = ["css/custom.css"]
92 |
93 | html_favicon = "_static/favicon.ico"
94 |
95 | html_theme_options = {
96 | "footer_icons": [
97 | {
98 | "name": "GitHub",
99 | "url": "https://github.com/allenai/python-package-template",
100 | "html": """
101 |
104 | """, # noqa: E501
105 | "class": "",
106 | },
107 | ],
108 | }
109 |
--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
1 | .. my_package documentation master file, created by
2 | sphinx-quickstart on Tue Sep 21 08:07:48 2021.
3 | You can adapt this file completely to your liking, but it should at least
4 | contain the root `toctree` directive.
5 |
6 | **my-package**
7 | ===============
8 |
9 | .. automodule:: my_package
10 |
11 | Contents
12 | --------
13 |
14 | .. toctree::
15 | :maxdepth: 2
16 | :caption: Getting started:
17 |
18 | installation
19 | overview
20 | CHANGELOG
21 |
22 | .. toctree::
23 | :hidden:
24 | :caption: Development
25 |
26 | License
27 | CONTRIBUTING
28 | GitHub Repository
29 |
30 | Team
31 | ----
32 |
33 | **my-package** is developed and maintained by the AllenNLP team, backed by
34 | `the Allen Institute for Artificial Intelligence (AI2) `_.
35 | AI2 is a non-profit institute with the mission to contribute to humanity through high-impact AI research and engineering.
36 | To learn more about who specifically contributed to this codebase, see
37 | `our contributors `_ page.
38 |
39 | License
40 | -------
41 |
42 | **my-package** is licensed under `Apache 2.0 `_.
43 | A full copy of the license can be found `on GitHub `_.
44 |
45 | Indices and tables
46 | ------------------
47 |
48 | * :ref:`genindex`
49 | * :ref:`modindex`
50 |
--------------------------------------------------------------------------------
/docs/source/installation.md:
--------------------------------------------------------------------------------
1 | Installation
2 | ============
3 |
4 | **my-package** supports Python >= 3.7.
5 |
6 | ## Installing with `pip`
7 |
8 | **my-package** is available [on PyPI](https://pypi.org/project/my-package/). Just run
9 |
10 | ```bash
11 | pip install my-package
12 | ```
13 |
14 | ## Installing from source
15 |
16 | To install **my-package** from source, first clone [the repository](https://github.com/allenai/python-package-template):
17 |
18 | ```bash
19 | git clone https://github.com/allenai/python-package-template.git
20 | cd python-package-template
21 | ```
22 |
23 | Then run
24 |
25 | ```bash
26 | pip install -e .
27 | ```
28 |
--------------------------------------------------------------------------------
/docs/source/overview.md:
--------------------------------------------------------------------------------
1 | Overview
2 | ========
3 |
4 |
--------------------------------------------------------------------------------
/golang/client.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | http "github.com/Danny-Dasilva/fhttp"
5 |
6 | "time"
7 |
8 | "golang.org/x/net/proxy"
9 | )
10 |
11 | type browser struct {
12 | // Return a greeting that embeds the name in a message.
13 | JA3 string
14 | UserAgent string
15 | Cookies []Cookie
16 | }
17 |
18 | var disabledRedirect = func(req *http.Request, via []*http.Request) error {
19 | return http.ErrUseLastResponse
20 | }
21 |
22 | func clientBuilder(browser browser, dialer proxy.ContextDialer, timeout int, disableRedirect bool) http.Client {
23 | //if timeout is not set in call default to 7
24 | if timeout == 0 {
25 | timeout = 7
26 | }
27 | client := http.Client{
28 | Transport: newRoundTripper(browser, dialer),
29 | Timeout: time.Duration(timeout) * time.Second,
30 | }
31 | //if disableRedirect is set to true httpclient will not redirect
32 | if disableRedirect {
33 | client.CheckRedirect = disabledRedirect
34 | }
35 | return client
36 | }
37 |
38 | // newClient creates a new http client
39 | func newClient(browser browser, timeout int, disableRedirect bool, UserAgent string, proxyURL ...string) (http.Client, error) {
40 | //fix check PR
41 | if len(proxyURL) > 0 && len(proxyURL[0]) > 0 {
42 | dialer, err := newConnectDialer(proxyURL[0], UserAgent)
43 | if err != nil {
44 | return http.Client{
45 | Timeout: time.Duration(timeout) * time.Second,
46 | CheckRedirect: disabledRedirect, //fix this fallthrough issue (test for incorrect proxy)
47 | }, err
48 | }
49 | return clientBuilder(
50 | browser,
51 | dialer,
52 | timeout,
53 | disableRedirect,
54 | ), nil
55 | }
56 |
57 | return clientBuilder(
58 | browser,
59 | proxy.Direct,
60 | timeout,
61 | disableRedirect,
62 | ), nil
63 |
64 | }
65 |
--------------------------------------------------------------------------------
/golang/connect.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | // borrowed from from https://github.com/caddyserver/forwardproxy/blob/master/httpclient/httpclient.go
4 | import (
5 | "bufio"
6 | "context"
7 | "crypto/tls"
8 | "encoding/base64"
9 | "errors"
10 | "golang.org/x/net/proxy"
11 | "io"
12 | "net"
13 | "net/http"
14 | "net/url"
15 | "strconv"
16 | "sync"
17 |
18 | "golang.org/x/net/http2"
19 | )
20 |
21 | // connectDialer allows to configure one-time use HTTP CONNECT client
22 | type connectDialer struct {
23 | ProxyURL url.URL
24 | DefaultHeader http.Header
25 |
26 | Dialer net.Dialer // overridden dialer allow to control establishment of TCP connection
27 |
28 | // overridden DialTLS allows user to control establishment of TLS connection
29 | // MUST return connection with completed Handshake, and NegotiatedProtocol
30 | DialTLS func(network string, address string) (net.Conn, string, error)
31 |
32 | EnableH2ConnReuse bool
33 | cacheH2Mu sync.Mutex
34 | cachedH2ClientConn *http2.ClientConn
35 | cachedH2RawConn net.Conn
36 | }
37 |
38 | // newConnectDialer creates a dialer to issue CONNECT requests and tunnel traffic via HTTP/S proxy.
39 | // proxyUrlStr must provide Scheme and Host, may provide credentials and port.
40 | // Example: https://username:password@golang.org:443
41 | func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer, error) {
42 | proxyURL, err := url.Parse(proxyURLStr)
43 | if err != nil {
44 | return nil, err
45 | }
46 |
47 | if proxyURL.Host == "" || proxyURL.Host == "undefined" {
48 | return nil, errors.New("invalid url `" + proxyURLStr +
49 | "`, make sure to specify full url like https://username:password@hostname.com:443/")
50 | }
51 |
52 | switch proxyURL.Scheme {
53 | case "http":
54 | if proxyURL.Port() == "" {
55 | proxyURL.Host = net.JoinHostPort(proxyURL.Host, "80")
56 | }
57 | case "https":
58 | if proxyURL.Port() == "" {
59 | proxyURL.Host = net.JoinHostPort(proxyURL.Host, "443")
60 | }
61 | case "":
62 | return nil, errors.New("specify scheme explicitly (https://)")
63 | default:
64 | return nil, errors.New("scheme " + proxyURL.Scheme + " is not supported")
65 | }
66 |
67 | client := &connectDialer{
68 | ProxyURL: *proxyURL,
69 | DefaultHeader: make(http.Header),
70 | EnableH2ConnReuse: true,
71 | }
72 |
73 | if proxyURL.User != nil {
74 | if proxyURL.User.Username() != "" {
75 | // password, _ := proxyUrl.User.Password()
76 | // client.DefaultHeader.Set("Proxy-Authorization", "Basic "+
77 | // base64.StdEncoding.EncodeToString([]byte(proxyUrl.User.Username()+":"+password)))
78 |
79 | username := proxyURL.User.Username()
80 | password, _ := proxyURL.User.Password()
81 |
82 | // client.DefaultHeader.SetBasicAuth(username, password)
83 | auth := username + ":" + password
84 | basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
85 | client.DefaultHeader.Add("Proxy-Authorization", basicAuth)
86 | }
87 | }
88 | client.DefaultHeader.Set("User-Agent", UserAgent)
89 | return client, nil
90 | }
91 |
92 | func (c *connectDialer) Dial(network, address string) (net.Conn, error) {
93 | return c.DialContext(context.Background(), network, address)
94 | }
95 |
96 | // ContextKeyHeader Users of context.WithValue should define their own types for keys
97 | type ContextKeyHeader struct{}
98 |
99 | // ctx.Value will be inspected for optional ContextKeyHeader{} key, with `http.Header` value,
100 | // which will be added to outgoing request headers, overriding any colliding c.DefaultHeader
101 | func (c *connectDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
102 | req := (&http.Request{
103 | Method: "CONNECT",
104 | URL: &url.URL{Host: address},
105 | Header: make(http.Header),
106 | Host: address,
107 | }).WithContext(ctx)
108 | for k, v := range c.DefaultHeader {
109 | req.Header[k] = v
110 | }
111 | if ctxHeader, ctxHasHeader := ctx.Value(ContextKeyHeader{}).(http.Header); ctxHasHeader {
112 | for k, v := range ctxHeader {
113 | req.Header[k] = v
114 | }
115 | }
116 | connectHTTP2 := func(rawConn net.Conn, h2clientConn *http2.ClientConn) (net.Conn, error) {
117 | req.Proto = "HTTP/2.0"
118 | req.ProtoMajor = 2
119 | req.ProtoMinor = 0
120 | pr, pw := io.Pipe()
121 | req.Body = pr
122 |
123 | resp, err := h2clientConn.RoundTrip(req)
124 | if err != nil {
125 | _ = rawConn.Close()
126 | return nil, err
127 | }
128 |
129 | if resp.StatusCode != http.StatusOK {
130 | _ = rawConn.Close()
131 | return nil, errors.New("Proxy responded with non 200 code: " + resp.Status + "StatusCode:" + strconv.Itoa(resp.StatusCode))
132 | }
133 | return newHTTP2Conn(rawConn, pw, resp.Body), nil
134 | }
135 |
136 | connectHTTP1 := func(rawConn net.Conn) (net.Conn, error) {
137 | req.Proto = "HTTP/1.1"
138 | req.ProtoMajor = 1
139 | req.ProtoMinor = 1
140 |
141 | err := req.Write(rawConn)
142 | if err != nil {
143 | _ = rawConn.Close()
144 | return nil, err
145 | }
146 |
147 | resp, err := http.ReadResponse(bufio.NewReader(rawConn), req)
148 | if err != nil {
149 | _ = rawConn.Close()
150 | return nil, err
151 | }
152 |
153 | if resp.StatusCode != http.StatusOK {
154 | _ = rawConn.Close()
155 | return nil, errors.New("Proxy responded with non 200 code: " + resp.Status + " StatusCode:" + strconv.Itoa(resp.StatusCode))
156 | }
157 | return rawConn, nil
158 | }
159 |
160 | if c.EnableH2ConnReuse {
161 | c.cacheH2Mu.Lock()
162 | unlocked := false
163 | if c.cachedH2ClientConn != nil && c.cachedH2RawConn != nil {
164 | if c.cachedH2ClientConn.CanTakeNewRequest() {
165 | rc := c.cachedH2RawConn
166 | cc := c.cachedH2ClientConn
167 | c.cacheH2Mu.Unlock()
168 | unlocked = true
169 | proxyConn, err := connectHTTP2(rc, cc)
170 | if err == nil {
171 | return proxyConn, err
172 | }
173 | // else: carry on and try again
174 | }
175 | }
176 | if !unlocked {
177 | c.cacheH2Mu.Unlock()
178 | }
179 | }
180 |
181 | var err error
182 | var rawConn net.Conn
183 | negotiatedProtocol := ""
184 | switch c.ProxyURL.Scheme {
185 | case "http":
186 | rawConn, err = c.Dialer.DialContext(ctx, network, c.ProxyURL.Host)
187 | if err != nil {
188 | return nil, err
189 | }
190 | case "https":
191 | if c.DialTLS != nil {
192 | rawConn, negotiatedProtocol, err = c.DialTLS(network, c.ProxyURL.Host)
193 | if err != nil {
194 | return nil, err
195 | }
196 | } else {
197 | tlsConf := tls.Config{
198 | NextProtos: []string{"h2", "http/1.1"},
199 | ServerName: c.ProxyURL.Hostname(),
200 | }
201 | tlsConn, err := tls.Dial(network, c.ProxyURL.Host, &tlsConf)
202 | if err != nil {
203 | return nil, err
204 | }
205 | err = tlsConn.Handshake()
206 | if err != nil {
207 | return nil, err
208 | }
209 | negotiatedProtocol = tlsConn.ConnectionState().NegotiatedProtocol
210 | rawConn = tlsConn
211 | }
212 | default:
213 | return nil, errors.New("scheme " + c.ProxyURL.Scheme + " is not supported")
214 | }
215 |
216 | switch negotiatedProtocol {
217 | case "":
218 | fallthrough
219 | case "http/1.1":
220 | return connectHTTP1(rawConn)
221 | case "h2":
222 | t := http2.Transport{}
223 | h2clientConn, err := t.NewClientConn(rawConn)
224 | if err != nil {
225 | _ = rawConn.Close()
226 | return nil, err
227 | }
228 |
229 | proxyConn, err := connectHTTP2(rawConn, h2clientConn)
230 | if err != nil {
231 | _ = rawConn.Close()
232 | return nil, err
233 | }
234 | if c.EnableH2ConnReuse {
235 | c.cacheH2Mu.Lock()
236 | c.cachedH2ClientConn = h2clientConn
237 | c.cachedH2RawConn = rawConn
238 | c.cacheH2Mu.Unlock()
239 | }
240 | return proxyConn, err
241 | default:
242 | _ = rawConn.Close()
243 | return nil, errors.New("negotiated unsupported application layer protocol: " +
244 | negotiatedProtocol)
245 | }
246 | }
247 |
248 | func newHTTP2Conn(c net.Conn, pipedReqBody *io.PipeWriter, respBody io.ReadCloser) net.Conn {
249 | return &http2Conn{Conn: c, in: pipedReqBody, out: respBody}
250 | }
251 |
252 | type http2Conn struct {
253 | net.Conn
254 | in *io.PipeWriter
255 | out io.ReadCloser
256 | }
257 |
258 | func (h *http2Conn) Read(p []byte) (n int, err error) {
259 | return h.out.Read(p)
260 | }
261 |
262 | func (h *http2Conn) Write(p []byte) (n int, err error) {
263 | return h.in.Write(p)
264 | }
265 |
266 | func (h *http2Conn) Close() error {
267 | var retErr error = nil
268 | if err := h.in.Close(); err != nil {
269 | retErr = err
270 | }
271 | if err := h.out.Close(); err != nil {
272 | retErr = err
273 | }
274 | return retErr
275 | }
276 |
277 | func (h *http2Conn) CloseConn() error {
278 | return h.Conn.Close()
279 | }
280 |
281 | func (h *http2Conn) CloseWrite() error {
282 | return h.in.Close()
283 | }
284 |
285 | func (h *http2Conn) CloseRead() error {
286 | return h.out.Close()
287 | }
288 |
--------------------------------------------------------------------------------
/golang/cookie.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "net/http"
5 | "strconv"
6 | "strings"
7 | "time"
8 | )
9 |
10 | // Time wraps time.Time overriddin the json marshal/unmarshal to pass
11 | // timestamp as integer
12 | type Time struct {
13 | time.Time
14 | }
15 |
16 | type data struct {
17 | Time Time `json:"time"`
18 | }
19 |
20 | // A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an
21 | // HTTP response or the Cookie header of an HTTP request.
22 | //
23 | // See https://tools.ietf.org/html/rfc6265 for details.
24 | //Stolen from Net/http/cookies
25 | type Cookie struct {
26 | Name string `json:"name"`
27 | Value string `json:"value"`
28 |
29 | Path string `json:"path"` // optional
30 | Domain string `json:"domain"` // optional
31 | Expires time.Time
32 | JSONExpires Time `json:"expires"` // optional
33 | RawExpires string `json:"rawExpires"` // for reading cookies only
34 |
35 | // MaxAge=0 means no 'Max-Age' attribute specified.
36 | // MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
37 | // MaxAge>0 means Max-Age attribute present and given in seconds
38 | MaxAge int `json:"maxAge"`
39 | Secure bool `json:"secure"`
40 | HTTPOnly bool `json:"httpOnly"`
41 | SameSite http.SameSite `json:"sameSite"`
42 | Raw string
43 | Unparsed []string `json:"unparsed"` // Raw text of unparsed attribute-value pairs
44 | }
45 |
46 | // UnmarshalJSON implements json.Unmarshaler inferface.
47 | func (t *Time) UnmarshalJSON(buf []byte) error {
48 | // Try to parse the timestamp integer
49 | ts, err := strconv.ParseInt(string(buf), 10, 64)
50 | if err == nil {
51 | if len(buf) == 19 {
52 | t.Time = time.Unix(ts/1e9, ts%1e9)
53 | } else {
54 | t.Time = time.Unix(ts, 0)
55 | }
56 | return nil
57 | }
58 | str := strings.Trim(string(buf), `"`)
59 | if str == "null" || str == "" {
60 | return nil
61 | }
62 | // Try to manually parse the data
63 | tt, err := ParseDateString(str)
64 | if err != nil {
65 | return err
66 | }
67 | t.Time = tt
68 | return nil
69 | }
70 |
71 | // ParseDateString takes a string and passes it through Approxidate
72 | // Parses into a time.Time
73 | func ParseDateString(dt string) (time.Time, error) {
74 | const layout = "Mon, 02-Jan-2006 15:04:05 MST"
75 |
76 | return time.Parse(layout, dt)
77 | }
78 |
--------------------------------------------------------------------------------
/golang/errors.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net"
6 | "net/url"
7 | "os"
8 | "strconv"
9 | "strings"
10 | )
11 |
12 | type errorMessage struct {
13 | StatusCode int
14 | debugger string
15 | ErrorMsg string
16 | Op string
17 | }
18 |
19 | func lastString(ss []string) string {
20 | return ss[len(ss)-1]
21 | }
22 |
23 | // func createErrorString(err: string) (msg, debugger string) {
24 | func createErrorString(err error) (msg, debugger string) {
25 | msg = fmt.Sprintf("Proxy returned a Syscall Error: %s", err)
26 | debugger = fmt.Sprintf("%#v\n", err)
27 | return
28 | }
29 |
30 | func createErrorMessage(StatusCode int, err error, op string) errorMessage {
31 | msg := fmt.Sprintf("Proxy returned a Syscall Error: %s", err)
32 | debugger := fmt.Sprintf("%#v\n", err)
33 | return errorMessage{StatusCode: StatusCode, debugger: debugger, ErrorMsg: msg, Op: op}
34 | }
35 |
36 | func parseError(err error) (errormessage errorMessage) {
37 | var op string
38 |
39 | httpError := string(err.Error())
40 | status := lastString(strings.Split(httpError, "StatusCode:"))
41 | StatusCode, _ := strconv.Atoi(status)
42 | if StatusCode != 0 {
43 | msg, debugger := createErrorString(err)
44 | return errorMessage{StatusCode: StatusCode, debugger: debugger, ErrorMsg: msg}
45 | }
46 | if uerr, ok := err.(*url.Error); ok {
47 | if noerr, ok := uerr.Err.(*net.OpError); ok {
48 | op = noerr.Op
49 | if SyscallError, ok := noerr.Err.(*os.SyscallError); ok {
50 | if noerr.Timeout() {
51 | return createErrorMessage(408, SyscallError, op)
52 | }
53 | return createErrorMessage(401, SyscallError, op)
54 | } else if AddrError, ok := noerr.Err.(*net.AddrError); ok {
55 | return createErrorMessage(405, AddrError, op)
56 | } else if DNSError, ok := noerr.Err.(*net.DNSError); ok {
57 | return createErrorMessage(421, DNSError, op)
58 | } else {
59 | return createErrorMessage(421, noerr, op)
60 | }
61 | }
62 | if uerr.Timeout() {
63 | return createErrorMessage(408, uerr, op)
64 | }
65 | }
66 | return
67 | }
68 |
69 | type errExtensionNotExist struct {
70 | Context string
71 | }
72 |
73 | func (w *errExtensionNotExist) Error() string {
74 | return fmt.Sprintf("Extension {{ %s }} is not Supported by CycleTLS please raise an issue", w.Context)
75 | }
76 |
77 | func raiseExtensionError(info string) *errExtensionNotExist {
78 | return &errExtensionNotExist{
79 | Context: info,
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/golang/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/Danny-Dasilva/CycleTLS/cycletls
2 |
3 | go 1.14
4 |
5 | require (
6 | github.com/Danny-Dasilva/fhttp v0.0.0-20211010093114-56fde831fe2f
7 | github.com/andybalholm/brotli v1.0.3
8 | github.com/gorilla/websocket v1.4.2
9 | gitlab.com/yawning/utls.git v0.0.12-1
10 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
11 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
12 | golang.org/x/text v0.3.7 // indirect
13 | )
14 |
--------------------------------------------------------------------------------
/golang/go.sum:
--------------------------------------------------------------------------------
1 | github.com/Danny-Dasilva/fhttp v0.0.0-20211010093114-56fde831fe2f h1:Hlra6eNmVL0lopvAnL8u/WxibvlsxAfxo4utzMKlKRk=
2 | github.com/Danny-Dasilva/fhttp v0.0.0-20211010093114-56fde831fe2f/go.mod h1:lN6LBYb4KCkMZzo1qlSvQHLboot650pJfOwg4pKBKDc=
3 | github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
4 | github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
5 | github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
6 | github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
7 | github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
8 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
9 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
10 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
11 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
12 | github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
13 | gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec h1:FpfFs4EhNehiVfzQttTuxanPIT43FtkkCFypIod8LHo=
14 | gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ=
15 | gitlab.com/yawning/utls.git v0.0.12-1 h1:RL6O0MP2YI0KghuEU/uGN6+8b4183eqNWoYgx7CXD0U=
16 | gitlab.com/yawning/utls.git v0.0.12-1/go.mod h1:3ONKiSFR9Im/c3t5RKmMJTVdmZN496FNyk3mjrY1dyo=
17 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
18 | golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
19 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
20 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
21 | golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
22 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
23 | golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
24 | golang.org/x/net v0.0.0-20211005215030-d2e5035098b3 h1:G64nFNerDErBd2KdvHvIn3Ee6ccUQBTfhDZEO0DccfU=
25 | golang.org/x/net v0.0.0-20211005215030-d2e5035098b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
26 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
27 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
28 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
29 | golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
30 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
31 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
32 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
33 | golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
34 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
35 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
36 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
37 | golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
38 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
39 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
40 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
41 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
42 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
43 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
44 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
45 |
--------------------------------------------------------------------------------
/golang/index.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "flag"
6 | "io/ioutil"
7 | "log"
8 | "net/url"
9 | "os"
10 | "runtime"
11 | "strings"
12 | "time"
13 |
14 | rhttp "net/http"
15 |
16 | http "github.com/Danny-Dasilva/fhttp"
17 | "github.com/gorilla/websocket"
18 | )
19 |
20 | // Options sets CycleTLS client options
21 | type Options struct {
22 | URL string `json:"url"`
23 | Method string `json:"method"`
24 | Headers map[string]string `json:"headers"`
25 | Body string `json:"body"`
26 | Ja3 string `json:"ja3"`
27 | UserAgent string `json:"userAgent"`
28 | Proxy string `json:"proxy"`
29 | Cookies []Cookie `json:"cookies"`
30 | Timeout int `json:"timeout"`
31 | DisableRedirect bool `json:"disableRedirect"`
32 | HeaderOrder []string `json:"headerOrder"`
33 | OrderAsProvided bool `json:"orderAsProvided"` //TODO
34 | }
35 |
36 | type cycleTLSRequest struct {
37 | RequestID string `json:"requestId"`
38 | Options Options `json:"options"`
39 | }
40 |
41 | //rename to request+client+options
42 | type fullRequest struct {
43 | req *http.Request
44 | client http.Client
45 | options cycleTLSRequest
46 | }
47 |
48 | //Response contains Cycletls response data
49 | type Response struct {
50 | RequestID string
51 | Status int
52 | Body string
53 | Headers map[string]string
54 | }
55 |
56 | //JSONBody converts response body to json
57 | func (re Response) JSONBody() map[string]interface{} {
58 | var data map[string]interface{}
59 | err := json.Unmarshal([]byte(re.Body), &data)
60 | if err != nil {
61 | log.Print("Json Conversion failed " + err.Error() + re.Body)
62 | }
63 | return data
64 | }
65 |
66 | //CycleTLS creates full request and response
67 | type CycleTLS struct {
68 | ReqChan chan fullRequest
69 | RespChan chan Response
70 | }
71 |
72 | func getWebsocketAddr() string {
73 | port, exists := os.LookupEnv("WS_PORT")
74 |
75 | var addr *string
76 |
77 | if exists {
78 | addr = flag.String("addr", "localhost:"+port, "http service address")
79 | } else {
80 | addr = flag.String("addr", "localhost:9112", "http service address")
81 | }
82 | u := url.URL{Scheme: "ws", Host: *addr, Path: "/"}
83 |
84 | return u.String()
85 | }
86 |
87 | // ready Request
88 | func processRequest(request cycleTLSRequest) (result fullRequest) {
89 |
90 | var browser = browser{
91 | JA3: request.Options.Ja3,
92 | UserAgent: request.Options.UserAgent,
93 | Cookies: request.Options.Cookies,
94 | }
95 |
96 | client, err := newClient(
97 | browser,
98 | request.Options.Timeout,
99 | request.Options.DisableRedirect,
100 | request.Options.UserAgent,
101 | request.Options.Proxy,
102 | )
103 | if err != nil {
104 | log.Fatal(err)
105 | }
106 |
107 | req, err := http.NewRequest(strings.ToUpper(request.Options.Method), request.Options.URL, strings.NewReader(request.Options.Body))
108 | if err != nil {
109 | log.Fatal(err)
110 | }
111 | headerorder := []string{}
112 | //master header order, all your headers will be ordered based on this list and anything extra will be appended to the end
113 | //if your site has any custom headers, see the header order chrome uses and then add those headers to this list
114 | if len(request.Options.HeaderOrder) > 0 {
115 | //lowercase headers
116 | for _, v := range request.Options.HeaderOrder {
117 | lowercasekey := strings.ToLower(v)
118 | headerorder = append(headerorder, lowercasekey)
119 | }
120 | } else {
121 | headerorder = append(headerorder,
122 | "host",
123 | "connection",
124 | "cache-control",
125 | "device-memory",
126 | "viewport-width",
127 | "rtt",
128 | "downlink",
129 | "ect",
130 | "sec-ch-ua",
131 | "sec-ch-ua-mobile",
132 | "sec-ch-ua-full-version",
133 | "sec-ch-ua-arch",
134 | "sec-ch-ua-platform",
135 | "sec-ch-ua-platform-version",
136 | "sec-ch-ua-model",
137 | "upgrade-insecure-requests",
138 | "user-agent",
139 | "accept",
140 | "sec-fetch-site",
141 | "sec-fetch-mode",
142 | "sec-fetch-user",
143 | "sec-fetch-dest",
144 | "referer",
145 | "accept-encoding",
146 | "accept-language",
147 | "cookie",
148 | )
149 | }
150 |
151 | headermap := make(map[string]string)
152 | //TODO: Shorten this
153 | headerorderkey := []string{}
154 | for _, key := range headerorder {
155 | for k, v := range request.Options.Headers {
156 | lowercasekey := strings.ToLower(k)
157 | if key == lowercasekey {
158 | headermap[k] = v
159 | headerorderkey = append(headerorderkey, lowercasekey)
160 | }
161 | }
162 |
163 | }
164 |
165 | //ordering the pseudo headers and our normal headers
166 | req.Header = http.Header{
167 | http.HeaderOrderKey: headerorderkey,
168 | http.PHeaderOrderKey: {":method", ":authority", ":scheme", ":path"},
169 | }
170 | //set our Host header
171 | u, err := url.Parse(request.Options.URL)
172 | if err != nil {
173 | panic(err)
174 | }
175 | req.Header.Set("Host", u.Host)
176 |
177 | //append our normal headers
178 | for k, v := range headermap {
179 | req.Header.Set(k, v)
180 | }
181 |
182 | return fullRequest{req: req, client: client, options: request}
183 |
184 | }
185 |
186 | func dispatcher(res fullRequest) (response Response, err error) {
187 | resp, err := res.client.Do(res.req)
188 | if err != nil {
189 |
190 | parsedError := parseError(err)
191 |
192 | headers := make(map[string]string)
193 | return Response{res.options.RequestID, parsedError.StatusCode, parsedError.ErrorMsg + "-> \n" + string(err.Error()), headers}, nil //normally return error here
194 |
195 | }
196 | defer resp.Body.Close()
197 |
198 | encoding := resp.Header["Content-Encoding"]
199 |
200 | bodyBytes, err := ioutil.ReadAll(resp.Body)
201 | if err != nil {
202 | log.Print("Parse Bytes" + err.Error())
203 | return response, err
204 | }
205 | Body := DecompressBody(bodyBytes, encoding)
206 | headers := make(map[string]string)
207 |
208 | for name, values := range resp.Header {
209 | if name == "Set-Cookie" {
210 | headers[name] = strings.Join(values, "/,/")
211 | } else {
212 | for _, value := range values {
213 | headers[name] = value
214 | }
215 | }
216 | }
217 | return Response{res.options.RequestID, resp.StatusCode, Body, headers}, nil
218 |
219 | }
220 |
221 | // Queue queues request in worker pool
222 | func (client CycleTLS) Queue(URL string, options Options, Method string) {
223 |
224 | options.URL = URL
225 | options.Method = Method
226 | //TODO add timestamp to request
227 | opt := cycleTLSRequest{"Queued Request", options}
228 | response := processRequest(opt)
229 | client.ReqChan <- response
230 | }
231 |
232 | // Do creates a single request
233 | func (client CycleTLS) Do(URL string, options Options, Method string) (response Response, err error) {
234 |
235 | options.URL = URL
236 | options.Method = Method
237 | opt := cycleTLSRequest{"cycleTLSRequest", options}
238 |
239 | res := processRequest(opt)
240 | response, err = dispatcher(res)
241 | if err != nil {
242 | log.Print("Request Failed: " + err.Error())
243 | return response, err
244 | }
245 |
246 | return response, nil
247 | }
248 |
249 | //TODO rename this
250 |
251 | // Init starts the worker pool or returns a empty cycletls struct
252 | func Init(workers ...bool) CycleTLS {
253 | if len(workers) > 0 && workers[0] {
254 | reqChan := make(chan fullRequest)
255 | respChan := make(chan Response)
256 | go workerPool(reqChan, respChan)
257 | log.Println("Worker Pool Started")
258 |
259 | return CycleTLS{ReqChan: reqChan, RespChan: respChan}
260 | }
261 | return CycleTLS{}
262 |
263 | }
264 |
265 | // Close closes channels
266 | func (client CycleTLS) Close() {
267 | close(client.ReqChan)
268 | close(client.RespChan)
269 |
270 | }
271 |
272 | // Worker Pool
273 | func workerPool(reqChan chan fullRequest, respChan chan Response) {
274 | //MAX
275 | for i := 0; i < 100; i++ {
276 | go worker(reqChan, respChan)
277 | }
278 | }
279 |
280 | // Worker
281 | func worker(reqChan chan fullRequest, respChan chan Response) {
282 | for res := range reqChan {
283 | response, err := dispatcher(res)
284 | if err != nil {
285 | log.Print("Request Failed: " + err.Error())
286 | }
287 | respChan <- response
288 | }
289 | }
290 |
291 | func readSocket(reqChan chan fullRequest, c *websocket.Conn) {
292 | for {
293 | _, message, err := c.ReadMessage()
294 | if err != nil {
295 | if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
296 | return
297 | } else {
298 | log.Print("Socket Error", err)
299 | return
300 | }
301 | }
302 | request := new(cycleTLSRequest)
303 |
304 | err = json.Unmarshal(message, &request)
305 | if err != nil {
306 | log.Print("Unmarshal Error", err)
307 | return
308 | }
309 |
310 | reply := processRequest(*request)
311 |
312 | reqChan <- reply
313 | }
314 | }
315 |
316 | func writeSocket(respChan chan Response, c *websocket.Conn) {
317 | for {
318 | select {
319 | case r := <-respChan:
320 | message, err := json.Marshal(r)
321 | if err != nil {
322 | log.Print("Marshal Json Failed" + err.Error())
323 | continue
324 | }
325 | err = c.WriteMessage(websocket.TextMessage, message)
326 | if err != nil {
327 | log.Print("Socket WriteMessage Failed" + err.Error())
328 | continue
329 | }
330 |
331 | }
332 |
333 | }
334 | }
335 |
336 | var upgrader = websocket.Upgrader{
337 | ReadBufferSize: 1024,
338 | WriteBufferSize: 1024,
339 | }
340 |
341 | func wsEndpoint(w rhttp.ResponseWriter, r *rhttp.Request) {
342 | upgrader.CheckOrigin = func(r *rhttp.Request) bool { return true }
343 |
344 | // upgrade this connection to a WebSocket
345 | // connection
346 | ws, err := upgrader.Upgrade(w, r, nil)
347 | if err != nil {
348 | log.Println(err)
349 | }
350 | log.Println("Client Connected")
351 | reqChan := make(chan fullRequest)
352 | respChan := make(chan Response)
353 | go workerPool(reqChan, respChan)
354 |
355 | go readSocket(reqChan, ws)
356 | //run as main thread
357 | writeSocket(respChan, ws)
358 |
359 | }
360 | func homePage(w rhttp.ResponseWriter, r *rhttp.Request) {
361 | log.Println(w, "Home Page")
362 | }
363 | func setupRoutes() {
364 | rhttp.HandleFunc("/", wsEndpoint)
365 | }
366 |
367 | func main() {
368 |
369 | runtime.GOMAXPROCS(runtime.NumCPU())
370 |
371 | setupRoutes()
372 | log.Fatal(rhttp.ListenAndServe(":8080", nil))
373 | }
374 |
--------------------------------------------------------------------------------
/golang/roundtripper.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "crypto/tls"
6 | "errors"
7 | "fmt"
8 | "net"
9 |
10 | "strings"
11 | "sync"
12 |
13 | http "github.com/Danny-Dasilva/fhttp"
14 | http2 "github.com/Danny-Dasilva/fhttp/http2"
15 | "golang.org/x/net/proxy"
16 |
17 | utls "gitlab.com/yawning/utls.git"
18 | )
19 |
20 | var errProtocolNegotiated = errors.New("protocol negotiated")
21 |
22 | type roundTripper struct {
23 | sync.Mutex
24 | // fix typing
25 | JA3 string
26 | UserAgent string
27 |
28 | Cookies []Cookie
29 | cachedConnections map[string]net.Conn
30 | cachedTransports map[string]http.RoundTripper
31 |
32 | dialer proxy.ContextDialer
33 | }
34 |
35 | func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
36 | // Fix this later for proper cookie parsing
37 | for _, properties := range rt.Cookies {
38 | req.AddCookie(&http.Cookie{Name: properties.Name,
39 | Value: properties.Value,
40 | Path: properties.Path,
41 | Domain: properties.Domain,
42 | Expires: properties.JSONExpires.Time, //TODO: scuffed af
43 | RawExpires: properties.RawExpires,
44 | MaxAge: properties.MaxAge,
45 | HttpOnly: properties.HTTPOnly,
46 | Secure: properties.Secure,
47 | Raw: properties.Raw,
48 | Unparsed: properties.Unparsed,
49 | })
50 | }
51 | req.Header.Set("User-Agent", rt.UserAgent)
52 | addr := rt.getDialTLSAddr(req)
53 | if _, ok := rt.cachedTransports[addr]; !ok {
54 | if err := rt.getTransport(req, addr); err != nil {
55 | return nil, err
56 | }
57 | }
58 | return rt.cachedTransports[addr].RoundTrip(req)
59 | }
60 |
61 | func (rt *roundTripper) getTransport(req *http.Request, addr string) error {
62 | switch strings.ToLower(req.URL.Scheme) {
63 | case "http":
64 | rt.cachedTransports[addr] = &http.Transport{DialContext: rt.dialer.DialContext, DisableKeepAlives: true}
65 | return nil
66 | case "https":
67 | default:
68 | return fmt.Errorf("invalid URL scheme: [%v]", req.URL.Scheme)
69 | }
70 |
71 | _, err := rt.dialTLS(context.Background(), "tcp", addr)
72 | switch err {
73 | case errProtocolNegotiated:
74 | case nil:
75 | // Should never happen.
76 | panic("dialTLS returned no error when determining cachedTransports")
77 | default:
78 | return err
79 | }
80 |
81 | return nil
82 | }
83 |
84 | func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net.Conn, error) {
85 | rt.Lock()
86 | defer rt.Unlock()
87 |
88 | // If we have the connection from when we determined the HTTPS
89 | // cachedTransports to use, return that.
90 | if conn := rt.cachedConnections[addr]; conn != nil {
91 | delete(rt.cachedConnections, addr)
92 | return conn, nil
93 | }
94 | rawConn, err := rt.dialer.DialContext(ctx, network, addr)
95 | if err != nil {
96 | return nil, err
97 | }
98 |
99 | var host string
100 | if host, _, err = net.SplitHostPort(addr); err != nil {
101 | host = addr
102 | }
103 | //////////////////
104 |
105 | spec, err := StringToSpec(rt.JA3, rt.UserAgent)
106 | if err != nil {
107 | return nil, err
108 | }
109 |
110 | conn := utls.UClient(rawConn, &utls.Config{ServerName: host}, // MinVersion: tls.VersionTLS10,
111 | // MaxVersion: tls.VersionTLS13,
112 |
113 | utls.HelloCustom)
114 |
115 | if err := conn.ApplyPreset(spec); err != nil {
116 | return nil, err
117 | }
118 |
119 | if err = conn.Handshake(); err != nil {
120 | _ = conn.Close()
121 |
122 | if err.Error() == "tls: CurvePreferences includes unsupported curve" {
123 | //fix this
124 | return nil, fmt.Errorf("conn.Handshake() error for tls 1.3 (please retry request): %+v", err)
125 | }
126 | return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err)
127 | }
128 |
129 | //////////
130 | if rt.cachedTransports[addr] != nil {
131 | return conn, nil
132 | }
133 |
134 | // No http.Transport constructed yet, create one based on the results
135 | // of ALPN.
136 | switch conn.ConnectionState().NegotiatedProtocol {
137 | case http2.NextProtoTLS:
138 | t2 := http2.Transport{DialTLS: rt.dialTLSHTTP2}
139 | t2.Settings = []http2.Setting{
140 | {ID: http2.SettingMaxConcurrentStreams, Val: 1000},
141 | {ID: http2.SettingMaxFrameSize, Val: 16384},
142 | {ID: http2.SettingMaxHeaderListSize, Val: 262144},
143 | }
144 | t2.InitialWindowSize = 6291456
145 | t2.HeaderTableSize = 65536
146 | t2.PushHandler = &http2.DefaultPushHandler{}
147 | rt.cachedTransports[addr] = &t2
148 | default:
149 | // Assume the remote peer is speaking HTTP 1.x + TLS.
150 | rt.cachedTransports[addr] = &http.Transport{DialTLSContext: rt.dialTLS}
151 |
152 | }
153 |
154 | // Stash the connection just established for use servicing the
155 | // actual request (should be near-immediate).
156 | rt.cachedConnections[addr] = conn
157 |
158 | return nil, errProtocolNegotiated
159 | }
160 |
161 | func (rt *roundTripper) dialTLSHTTP2(network, addr string, _ *tls.Config) (net.Conn, error) {
162 | return rt.dialTLS(context.Background(), network, addr)
163 | }
164 |
165 | func (rt *roundTripper) getDialTLSAddr(req *http.Request) string {
166 | host, port, err := net.SplitHostPort(req.URL.Host)
167 | if err == nil {
168 | return net.JoinHostPort(host, port)
169 | }
170 | return net.JoinHostPort(req.URL.Host, "443") // we can assume port is 443 at this point
171 | }
172 |
173 | func newRoundTripper(browser browser, dialer ...proxy.ContextDialer) http.RoundTripper {
174 | if len(dialer) > 0 {
175 |
176 | return &roundTripper{
177 | dialer: dialer[0],
178 |
179 | JA3: browser.JA3,
180 | UserAgent: browser.UserAgent,
181 | Cookies: browser.Cookies,
182 | cachedTransports: make(map[string]http.RoundTripper),
183 | cachedConnections: make(map[string]net.Conn),
184 | }
185 | }
186 |
187 | return &roundTripper{
188 | dialer: proxy.Direct,
189 |
190 | JA3: browser.JA3,
191 | UserAgent: browser.UserAgent,
192 | Cookies: browser.Cookies,
193 | cachedTransports: make(map[string]http.RoundTripper),
194 | cachedConnections: make(map[string]net.Conn),
195 | }
196 | }
197 |
--------------------------------------------------------------------------------
/golang/utils.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "compress/gzip"
6 | "compress/zlib"
7 | "crypto/sha256"
8 | "github.com/andybalholm/brotli"
9 | utls "gitlab.com/yawning/utls.git"
10 | "io/ioutil"
11 | "log"
12 | "strconv"
13 | "strings"
14 | )
15 |
16 | const (
17 | chrome = "chrome" //chrome User agent enum
18 | firefox = "firefox" //firefox User agent enum
19 | )
20 |
21 | func parseUserAgent(userAgent string) string {
22 | switch {
23 | case strings.Contains(strings.ToLower(userAgent), "chrome"):
24 | return chrome
25 | case strings.Contains(strings.ToLower(userAgent), "firefox"):
26 | return firefox
27 | default:
28 | return chrome
29 | }
30 |
31 | }
32 |
33 | // DecompressBody unzips compressed data
34 | func DecompressBody(Body []byte, encoding []string) (parsedBody string) {
35 | if len(encoding) > 0 {
36 | if encoding[0] == "gzip" {
37 | unz, err := gUnzipData(Body)
38 | if err != nil {
39 | return string(Body)
40 | }
41 | parsedBody = string(unz)
42 | } else if encoding[0] == "deflate" {
43 | unz, err := enflateData(Body)
44 | if err != nil {
45 | return string(Body)
46 | }
47 | parsedBody = string(unz)
48 | } else if encoding[0] == "br" {
49 | unz, err := unBrotliData(Body)
50 | if err != nil {
51 | return string(Body)
52 | }
53 | parsedBody = string(unz)
54 | } else {
55 | log.Println("Unknown Encoding" + encoding[0])
56 | parsedBody = string(Body)
57 | }
58 | } else {
59 | parsedBody = string(Body)
60 | }
61 | return parsedBody
62 |
63 | }
64 |
65 | func gUnzipData(data []byte) (resData []byte, err error) {
66 | gz, err := gzip.NewReader(bytes.NewReader(data))
67 | if err != nil {
68 | return []byte{}, err
69 | }
70 | defer gz.Close()
71 | respBody, err := ioutil.ReadAll(gz)
72 | return respBody, err
73 | }
74 | func enflateData(data []byte) (resData []byte, err error) {
75 | zr, err := zlib.NewReader(bytes.NewReader(data))
76 | if err != nil {
77 | return []byte{}, err
78 | }
79 | defer zr.Close()
80 | enflated, err := ioutil.ReadAll(zr)
81 | return enflated, err
82 | }
83 | func unBrotliData(data []byte) (resData []byte, err error) {
84 | br := brotli.NewReader(bytes.NewReader(data))
85 | respBody, err := ioutil.ReadAll(br)
86 | return respBody, err
87 | }
88 |
89 | // StringToSpec creates a ClientHelloSpec based on a JA3 string
90 | func StringToSpec(ja3 string, userAgent string) (*utls.ClientHelloSpec, error) {
91 | parsedUserAgent := parseUserAgent("chrome")
92 | extMap := genMap()
93 | tokens := strings.Split(ja3, ",")
94 |
95 | version := tokens[0]
96 | ciphers := strings.Split(tokens[1], "-")
97 | extensions := strings.Split(tokens[2], "-")
98 | curves := strings.Split(tokens[3], "-")
99 | if len(curves) == 1 && curves[0] == "" {
100 | curves = []string{}
101 | }
102 | pointFormats := strings.Split(tokens[4], "-")
103 | if len(pointFormats) == 1 && pointFormats[0] == "" {
104 | pointFormats = []string{}
105 | }
106 |
107 | // parse curves
108 | var targetCurves []utls.CurveID
109 | targetCurves = append(targetCurves, utls.CurveID(utls.GREASE_PLACEHOLDER)) //append grease for Chrome browsers
110 | for _, c := range curves {
111 | cid, err := strconv.ParseUint(c, 10, 16)
112 | if err != nil {
113 | return nil, err
114 | }
115 | targetCurves = append(targetCurves, utls.CurveID(cid))
116 | }
117 | extMap["10"] = &utls.SupportedCurvesExtension{Curves: targetCurves}
118 |
119 | // parse point formats
120 | var targetPointFormats []byte
121 | for _, p := range pointFormats {
122 | pid, err := strconv.ParseUint(p, 10, 8)
123 | if err != nil {
124 | return nil, err
125 | }
126 | targetPointFormats = append(targetPointFormats, byte(pid))
127 | }
128 | extMap["11"] = &utls.SupportedPointsExtension{SupportedPoints: targetPointFormats}
129 |
130 | // set extension 43
131 | vid64, err := strconv.ParseUint(version, 10, 16)
132 | if err != nil {
133 | return nil, err
134 | }
135 | vid := uint16(vid64)
136 | // extMap["43"] = &utls.SupportedVersionsExtension{
137 | // Versions: []uint16{
138 | // utls.VersionTLS12,
139 | // },
140 | // }
141 |
142 | // build extenions list
143 | var exts []utls.TLSExtension
144 | //Optionally Add Chrome Grease Extension
145 | if parsedUserAgent == chrome {
146 | exts = append(exts, &utls.UtlsGREASEExtension{})
147 | }
148 | for _, e := range extensions {
149 | te, ok := extMap[e]
150 | if !ok {
151 | return nil, raiseExtensionError(e)
152 | }
153 | //Optionally add Chrome Grease Extension
154 | if e == "21" && parsedUserAgent == chrome {
155 | exts = append(exts, &utls.UtlsGREASEExtension{})
156 | }
157 | exts = append(exts, te)
158 | }
159 | // build SSLVersion
160 | // vid64, err := strconv.ParseUint(version, 10, 16)
161 | // if err != nil {
162 | // return nil, err
163 | // }
164 |
165 | // build CipherSuites
166 | var suites []uint16
167 | //Optionally Add Chrome Grease Extension
168 | if parsedUserAgent == chrome {
169 | suites = append(suites, utls.GREASE_PLACEHOLDER)
170 | }
171 | for _, c := range ciphers {
172 | cid, err := strconv.ParseUint(c, 10, 16)
173 | if err != nil {
174 | return nil, err
175 | }
176 | suites = append(suites, uint16(cid))
177 | }
178 | _ = vid
179 | return &utls.ClientHelloSpec{
180 | // TLSVersMin: vid,
181 | // TLSVersMax: vid,
182 | CipherSuites: suites,
183 | CompressionMethods: []byte{0},
184 | Extensions: exts,
185 | GetSessionID: sha256.Sum256,
186 | }, nil
187 | }
188 |
189 | func genMap() (extMap map[string]utls.TLSExtension) {
190 | extMap = map[string]utls.TLSExtension{
191 | "0": &utls.SNIExtension{},
192 | "5": &utls.StatusRequestExtension{},
193 | // These are applied later
194 | // "10": &tls.SupportedCurvesExtension{...}
195 | // "11": &tls.SupportedPointsExtension{...}
196 | "13": &utls.SignatureAlgorithmsExtension{
197 | SupportedSignatureAlgorithms: []utls.SignatureScheme{
198 | utls.ECDSAWithP256AndSHA256,
199 | utls.ECDSAWithP384AndSHA384,
200 | utls.ECDSAWithP521AndSHA512,
201 | utls.PSSWithSHA256,
202 | utls.PSSWithSHA384,
203 | utls.PSSWithSHA512,
204 | utls.PKCS1WithSHA256,
205 | utls.PKCS1WithSHA384,
206 | utls.PKCS1WithSHA512,
207 | utls.ECDSAWithSHA1,
208 | utls.PKCS1WithSHA1,
209 | },
210 | },
211 | "16": &utls.ALPNExtension{
212 | AlpnProtocols: []string{"h2", "http/1.1"},
213 | },
214 | "18": &utls.SCTExtension{},
215 | "21": &utls.UtlsPaddingExtension{GetPaddingLen: utls.BoringPaddingStyle},
216 | "22": &utls.GenericExtension{Id: 22}, // encrypt_then_mac
217 | "23": &utls.UtlsExtendedMasterSecretExtension{},
218 | "27": &utls.CompressCertificateExtension{
219 | Algorithms: []utls.CertCompressionAlgo{utls.CertCompressionBrotli},
220 | },
221 | "28": &utls.FakeRecordSizeLimitExtension{}, //Limit: 0x4001
222 | "35": &utls.SessionTicketExtension{},
223 | "34": &utls.GenericExtension{Id: 34},
224 | "41": &utls.GenericExtension{Id: 41}, //FIXME pre_shared_key
225 | "43": &utls.SupportedVersionsExtension{Versions: []uint16{
226 | utls.GREASE_PLACEHOLDER,
227 | utls.VersionTLS13,
228 | utls.VersionTLS12,
229 | utls.VersionTLS11,
230 | utls.VersionTLS10}},
231 | "44": &utls.CookieExtension{},
232 | "45": &utls.PSKKeyExchangeModesExtension{Modes: []uint8{
233 | utls.PskModeDHE,
234 | }},
235 | "49": &utls.GenericExtension{Id: 49}, // post_handshake_auth
236 | "50": &utls.GenericExtension{Id: 50}, // signature_algorithms_cert
237 | "51": &utls.KeyShareExtension{KeyShares: []utls.KeyShare{
238 | {Group: utls.CurveID(utls.GREASE_PLACEHOLDER), Data: []byte{0}},
239 | {Group: utls.X25519},
240 |
241 | // {Group: utls.CurveP384}, known bug missing correct extensions for handshake
242 | }},
243 | "30032": &utls.GenericExtension{Id: 0x7550, Data: []byte{0}}, //FIXME
244 | "13172": &utls.NPNExtension{},
245 | "65281": &utls.RenegotiationInfoExtension{
246 | Renegotiation: utls.RenegotiateOnceAsClient,
247 | },
248 | }
249 | return
250 |
251 | }
252 |
--------------------------------------------------------------------------------
/mypy.ini:
--------------------------------------------------------------------------------
1 | [mypy]
2 | ignore_missing_imports = true
3 | no_site_packages = true
4 |
5 | [mypy-tests.*]
6 | strict_optional = false
7 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [tool.black]
2 | line-length = 100
3 |
4 | include = '\.pyi?$'
5 |
6 | exclude = '''
7 | (
8 | __pycache__
9 | | \.git
10 | | \.mypy_cache
11 | | \.pytest_cache
12 | | \.vscode
13 | | \.venv
14 | | \bdist\b
15 | | \bdoc\b
16 | )
17 | '''
18 |
19 | [tool.isort]
20 | profile = "black"
21 | multi_line_output = 3
22 |
23 | [build-system]
24 | requires = ["setuptools", "wheel"]
25 | build-backend = "setuptools.build_meta"
26 |
--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | testpaths = tests/
3 | python_classes = Test* *Test
4 | log_format = %(asctime)s - %(levelname)s - %(name)s - %(message)s
5 | log_level = DEBUG
6 | markers =
7 | filterwarnings =
8 |
--------------------------------------------------------------------------------
/scripts/personalize.py:
--------------------------------------------------------------------------------
1 | """
2 | Run this script once after first creating your project from this template repo to personalize
3 | it for own project.
4 |
5 | This script is interactive and will prompt you for various inputs.
6 | """
7 |
8 | from pathlib import Path
9 | from typing import Generator, List, Tuple
10 |
11 | import click
12 | from click_help_colors import HelpColorsCommand
13 | from rich import print
14 | from rich.markdown import Markdown
15 | from rich.prompt import Confirm
16 | from rich.syntax import Syntax
17 | from rich.traceback import install
18 |
19 | install(show_locals=True, suppress=[click])
20 |
21 | REPO_BASE = (Path(__file__).parent / "..").resolve()
22 |
23 | FILES_TO_REMOVE = {
24 | REPO_BASE / ".github" / "workflows" / "setup.yml",
25 | REPO_BASE / "setup-requirements.txt",
26 | REPO_BASE / "scripts" / "personalize.py",
27 | }
28 |
29 | PATHS_TO_IGNORE = {
30 | REPO_BASE / "README.md",
31 | REPO_BASE / ".git",
32 | REPO_BASE / "docs" / "source" / "_static" / "favicon.ico",
33 | }
34 |
35 | GITIGNORE_LIST = [
36 | line.strip()
37 | for line in (REPO_BASE / ".gitignore").open().readlines()
38 | if line.strip() and not line.startswith("#")
39 | ]
40 |
41 | REPO_NAME_TO_REPLACE = "python-package-template"
42 | BASE_URL_TO_REPLACE = "https://github.com/allenai/python-package-template"
43 |
44 |
45 | @click.command(
46 | cls=HelpColorsCommand,
47 | help_options_color="green",
48 | help_headers_color="yellow",
49 | context_settings={"max_content_width": 115},
50 | )
51 | @click.option(
52 | "--github-org",
53 | prompt="GitHub organization or user (e.g. 'allenai')",
54 | help="The name of your GitHub organization or user.",
55 | )
56 | @click.option(
57 | "--github-repo",
58 | prompt="GitHub repository (e.g. 'python-package-template')",
59 | help="The name of your GitHub repository.",
60 | )
61 | @click.option(
62 | "--package-name",
63 | prompt="Python package name (e.g. 'my-package')",
64 | help="The name of your Python package.",
65 | )
66 | @click.option(
67 | "-y",
68 | "--yes",
69 | is_flag=True,
70 | help="Run the script without prompting for a confirmation.",
71 | default=False,
72 | )
73 | @click.option(
74 | "--dry-run",
75 | is_flag=True,
76 | hidden=True,
77 | default=False,
78 | )
79 | def main(
80 | github_org: str, github_repo: str, package_name: str, yes: bool = False, dry_run: bool = False
81 | ):
82 | repo_url = f"https://github.com/{github_org}/{github_repo}"
83 | package_actual_name = package_name.replace("_", "-")
84 | package_dir_name = package_name.replace("-", "_")
85 |
86 | # Confirm before continuing.
87 | print(f"Repository URL set to: [link={repo_url}]{repo_url}[/]")
88 | print(f"Package name set to: [cyan]{package_actual_name}[/]")
89 | if not yes:
90 | yes = Confirm.ask("Is this correct?")
91 | if not yes:
92 | raise click.ClickException("Aborted, please run script again")
93 |
94 | # Delete files that we don't need.
95 | for path in FILES_TO_REMOVE:
96 | assert path.is_file(), path
97 | if not dry_run:
98 | path.unlink()
99 | else:
100 | print(f"Removing {path}")
101 |
102 | # Personalize remaining files.
103 | replacements = [
104 | (BASE_URL_TO_REPLACE, repo_url),
105 | (REPO_NAME_TO_REPLACE, github_repo),
106 | ("my-package", package_actual_name),
107 | ("my_package", package_dir_name),
108 | ]
109 | if dry_run:
110 | for old, new in replacements:
111 | print(f"Replacing '{old}' with '{new}'")
112 | for path in iterfiles(REPO_BASE):
113 | personalize_file(path, dry_run, replacements)
114 |
115 | # Rename 'my_package' directory to `package_dir_name`.
116 | if not dry_run:
117 | (REPO_BASE / "my_package").replace(REPO_BASE / package_dir_name)
118 | else:
119 | print(f"Renaming 'my_package' directory to '{package_dir_name}'")
120 |
121 | # Start with a fresh README.
122 | readme_contents = f"""# {package_actual_name}\n"""
123 | if not dry_run:
124 | with open(REPO_BASE / "README.md", "w+t") as readme_file:
125 | readme_file.write(readme_contents)
126 | else:
127 | print("Replacing README.md contents with:\n", Markdown(readme_contents))
128 |
129 | install_example = Syntax("pip install -e '.[dev]'", "bash")
130 | print(
131 | "[green]\N{check mark} Success![/] You can now install your package locally in development mode with:\n",
132 | install_example,
133 | )
134 |
135 |
136 | def iterfiles(dir: Path) -> Generator[Path, None, None]:
137 | assert dir.is_dir()
138 | for path in dir.iterdir():
139 | if path in PATHS_TO_IGNORE:
140 | continue
141 |
142 | is_ignored_file = False
143 | for gitignore_entry in GITIGNORE_LIST:
144 | if path.relative_to(REPO_BASE).match(gitignore_entry):
145 | is_ignored_file = True
146 | break
147 | if is_ignored_file:
148 | continue
149 |
150 | if path.is_dir():
151 | yield from iterfiles(path)
152 | else:
153 | yield path
154 |
155 |
156 | def personalize_file(path: Path, dry_run: bool, replacements: List[Tuple[str, str]]):
157 | with path.open("r+t") as file:
158 | filedata = file.read()
159 |
160 | should_update: bool = False
161 | for old, new in replacements:
162 | if filedata.count(old):
163 | should_update = True
164 | filedata = filedata.replace(old, new)
165 |
166 | if should_update:
167 | if not dry_run:
168 | with path.open("w+t") as file:
169 | file.write(filedata)
170 | else:
171 | print(f"Updating {path}")
172 |
173 |
174 | if __name__ == "__main__":
175 | main()
176 |
--------------------------------------------------------------------------------
/scripts/prepare_changelog.py:
--------------------------------------------------------------------------------
1 | from datetime import datetime
2 | from pathlib import Path
3 |
4 | from my_package.version import VERSION
5 |
6 |
7 | def main():
8 | changelog = Path("CHANGELOG.md")
9 |
10 | with changelog.open() as f:
11 | lines = f.readlines()
12 |
13 | insert_index: int = -1
14 | for i in range(len(lines)):
15 | line = lines[i]
16 | if line.startswith("## Unreleased"):
17 | insert_index = i + 1
18 | elif line.startswith(f"## [v{VERSION}]"):
19 | print("CHANGELOG already up-to-date")
20 | return
21 | elif line.startswith("## [v"):
22 | break
23 |
24 | if insert_index < 0:
25 | raise RuntimeError("Couldn't find 'Unreleased' section")
26 |
27 | lines.insert(insert_index, "\n")
28 | lines.insert(
29 | insert_index + 1,
30 | f"## [v{VERSION}](https://github.com/allenai/python-package-template/releases/tag/v{VERSION}) - "
31 | f"{datetime.now().strftime('%Y-%m-%d')}\n",
32 | )
33 |
34 | with changelog.open("w") as f:
35 | f.writelines(lines)
36 |
37 |
38 | if __name__ == "__main__":
39 | main()
40 |
--------------------------------------------------------------------------------
/scripts/release.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -e
4 |
5 | TAG=$(python -c 'from my_package.version import VERSION; print("v" + VERSION)')
6 |
7 | read -p "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt
8 |
9 | if [[ $prompt == "y" || $prompt == "Y" || $prompt == "yes" || $prompt == "Yes" ]]; then
10 | python scripts/prepare_changelog.py
11 | git add -A
12 | git commit -m "Bump version to $TAG for release" || true && git push
13 | echo "Creating new git tag $TAG"
14 | git tag "$TAG" -m "$TAG"
15 | git push --tags
16 | else
17 | echo "Cancelled"
18 | exit 1
19 | fi
20 |
--------------------------------------------------------------------------------
/scripts/release_notes.py:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 |
3 | """
4 | Prepares markdown release notes for GitHub releases.
5 | """
6 |
7 | import os
8 | from typing import List, Optional
9 |
10 | import packaging.version
11 |
12 | TAG = os.environ["TAG"]
13 |
14 | ADDED_HEADER = "### Added 🎉"
15 | CHANGED_HEADER = "### Changed ⚠️"
16 | FIXED_HEADER = "### Fixed ✅"
17 | REMOVED_HEADER = "### Removed 👋"
18 |
19 |
20 | def get_change_log_notes() -> str:
21 | in_current_section = False
22 | current_section_notes: List[str] = []
23 | with open("CHANGELOG.md") as changelog:
24 | for line in changelog:
25 | if line.startswith("## "):
26 | if line.startswith("## Unreleased"):
27 | continue
28 | if line.startswith(f"## [{TAG}]"):
29 | in_current_section = True
30 | continue
31 | break
32 | if in_current_section:
33 | if line.startswith("### Added"):
34 | line = ADDED_HEADER + "\n"
35 | elif line.startswith("### Changed"):
36 | line = CHANGED_HEADER + "\n"
37 | elif line.startswith("### Fixed"):
38 | line = FIXED_HEADER + "\n"
39 | elif line.startswith("### Removed"):
40 | line = REMOVED_HEADER + "\n"
41 | current_section_notes.append(line)
42 | assert current_section_notes
43 | return "## What's new\n\n" + "".join(current_section_notes).strip() + "\n"
44 |
45 |
46 | def get_commit_history() -> str:
47 | new_version = packaging.version.parse(TAG)
48 |
49 | # Get all tags sorted by version, latest first.
50 | all_tags = os.popen("git tag -l --sort=-version:refname 'v*'").read().split("\n")
51 |
52 | # Out of `all_tags`, find the latest previous version so that we can collect all
53 | # commits between that version and the new version we're about to publish.
54 | # Note that we ignore pre-releases unless the new version is also a pre-release.
55 | last_tag: Optional[str] = None
56 | for tag in all_tags:
57 | if not tag.strip(): # could be blank line
58 | continue
59 | version = packaging.version.parse(tag)
60 | if new_version.pre is None and version.pre is not None:
61 | continue
62 | if version < new_version:
63 | last_tag = tag
64 | break
65 | if last_tag is not None:
66 | commits = os.popen(f"git log {last_tag}..{TAG}^ --oneline --first-parent").read()
67 | else:
68 | commits = os.popen("git log --oneline --first-parent").read()
69 | return "## Commits\n\n" + commits
70 |
71 |
72 | def main():
73 | print(get_change_log_notes())
74 | print(get_commit_history())
75 |
76 |
77 | if __name__ == "__main__":
78 | main()
79 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 | import pathlib
3 | # The directory containing this file
4 | HERE = pathlib.Path(__file__).parent
5 |
6 | # The text of the README file
7 | README = (HERE / "README.md").read_text()
8 | setup(
9 | name='cycletls',
10 | version='0.0.1',
11 | packages=find_packages(exclude=['tests*']),
12 | license='none',
13 | description='A python package for spoofing TLS',
14 | long_description_content_type="text/markdown",
15 | long_description=README,
16 | install_requires=[],
17 | url='https://github.com/Danny-Dasilva/cycletls_python',
18 | author='Danny-Dasilva',
19 | author_email='dannydasilva.solutions@gmail.com'
20 | )
21 |
--------------------------------------------------------------------------------
/tests/conftest.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Danny-Dasilva/cycletls_python/43d8db8aa5e31ead0244397eac0014e7ad7b582f/tests/conftest.py
--------------------------------------------------------------------------------
/tests/integration/test_api.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | from cycletls import CycleTLS, Request
3 |
4 | @pytest.fixture
5 | def simple_request():
6 | """returns a simple request interface"""
7 | return Request(url="https://ja3er.com/json", method="get")
8 |
9 | def test_api_call():
10 | cycle = CycleTLS()
11 | result = cycle.get("https://ja3er.com/json")
12 |
13 | cycle.close()
14 | assert result.status_code == 200
15 |
16 |
--------------------------------------------------------------------------------
15 |