├── out └── .gitkeep ├── .gitignore ├── pyproject.toml ├── README.md ├── .github └── workflows │ ├── publish.yml │ └── check.yml ├── generate.py └── thttp.py /out/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | out/*.json 2 | .pyc 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 120 3 | 4 | [tool.black] 5 | line-length = 120 6 | 7 | [tool.isort] 8 | profile = "black" 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Finds Github repositories that multiple people you are following have starred. 2 | Creates a JSON feed that you can subscribe to and deploys it to Github Pages. 3 | 4 | --- 5 | 6 | ## Setup 7 | 8 | In your Github Repository set up the following secrets: 9 | 10 | - `FEED_AUTHOR`: A name for the "Author" field of the JSON Feed 11 | - `FEED_NAME`: The filename for the JSON file that is generated (`.json` will be appended to this) 12 | - `FEED_REPO`: The name of the repository (i.e. `multi-star`) 13 | - `FEED_USERNAME`: The Github username of the user/organisation that owns the repository 14 | - `GH_PAT`: A [Github token](https://github.com/settings/tokens) that has read access to the "followers" and "starring" categories 15 | 16 | Update your repository settings to [allow Github actions read and write permissions to your repository](./settings/actions). 17 | 18 | Once configured manually run the Github Action. 19 | 20 | Finally, jump back into settings to enable Pages. It should deploy from the `gh-pages` branch. 21 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Multi Star 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | schedule: 7 | - cron: '0 0 * * *' 8 | workflow_dispatch: 9 | 10 | jobs: 11 | publish: 12 | runs-on: ubuntu-latest 13 | 14 | permissions: 15 | contents: write 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Set up Python 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: "3.11" 24 | 25 | - name: Download existing feed 26 | run: curl https://sesh.github.io/multi-star/${{ secrets.FEED_NAME }}.json > out/${{ secrets.FEED_NAME }}.json 27 | 28 | - name: Build feed 29 | run: python3 generate.py 30 | env: 31 | FEED_AUTHOR: ${{ secrets.FEED_AUTHOR }} 32 | FEED_NAME: ${{ secrets.FEED_NAME }} 33 | FEED_REPO: ${{ secrets.FEED_REPO }} 34 | FEED_USERNAME: ${{ secrets.FEED_USERNAME }} 35 | GH_PAT: ${{ secrets.GH_PAT }} 36 | 37 | 38 | - name: Deploy to Github Pages 39 | uses: peaceiris/actions-gh-pages@v3 40 | with: 41 | github_token: ${{ secrets.GITHUB_TOKEN }} 42 | publish_dir: out 43 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: Python Checks 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | black: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | 17 | - name: Set up Python 18 | uses: actions/setup-python@v4 19 | with: 20 | python-version: "3.11" 21 | 22 | - name: Install black 23 | run: | 24 | python -m pip install black 25 | 26 | - name: Run black 27 | run: | 28 | black --check . 29 | 30 | isort: 31 | runs-on: ubuntu-latest 32 | 33 | steps: 34 | - uses: actions/checkout@v3 35 | 36 | - name: Set up Python 37 | uses: actions/setup-python@v4 38 | with: 39 | python-version: "3.11" 40 | 41 | - name: Install isort 42 | run: | 43 | python -m pip install isort 44 | 45 | - name: Run isort 46 | run: | 47 | isort --check . 48 | 49 | ruff: 50 | runs-on: ubuntu-latest 51 | 52 | steps: 53 | - uses: actions/checkout@v3 54 | 55 | - name: Set up Python 56 | uses: actions/setup-python@v4 57 | with: 58 | python-version: "3.11" 59 | 60 | - name: Install ruff 61 | run: | 62 | python -m pip install ruff 63 | 64 | - name: Run ruff 65 | run: | 66 | ruff --format=github . 67 | 68 | bandit: 69 | runs-on: ubuntu-latest 70 | 71 | steps: 72 | - uses: actions/checkout@v3 73 | 74 | - name: Set up Python 75 | uses: actions/setup-python@v4 76 | with: 77 | python-version: 3 78 | 79 | - name: Install bandit 80 | run: | 81 | python -m pip install bandit 82 | 83 | - name: Run bandit scan 84 | run: | 85 | bandit -r . 86 | -------------------------------------------------------------------------------- /generate.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | from datetime import datetime, timedelta, timezone 5 | from pathlib import Path 6 | 7 | from thttp import request 8 | 9 | """ 10 | A bunch of small utilities for generating JSON feeds 11 | """ 12 | 13 | AUTHOR_NAME = os.environ["FEED_AUTHOR"] 14 | REPOSITORY_NAME = os.environ["FEED_REPO"] 15 | GITHUB_USERNAME = os.environ["FEED_USERNAME"] 16 | FEED_NAME = os.environ["FEED_NAME"] 17 | GITHUB_TOKEN = os.environ["GH_PAT"] 18 | 19 | 20 | def url_for_post(post): 21 | return post["url"] 22 | 23 | 24 | def content_for_post(post): 25 | return post["html"] 26 | 27 | 28 | def title_for_post(post): 29 | return post.get("title", "") 30 | 31 | 32 | def save_jsonfeed(posts, *, feed_name, skip_existing_feed=False, duplicate_check_key="id", max_items=1000): 33 | items = [] 34 | 35 | if not skip_existing_feed and (Path("out") / feed_name).exists(): 36 | existing_feed = open(f"out/{feed_name}").read() 37 | 38 | try: 39 | items = json.loads(existing_feed)["items"] 40 | except json.decoder.JSONDecodeError: 41 | pass 42 | 43 | for post in posts: 44 | if post[duplicate_check_key] not in [item[duplicate_check_key] for item in items]: 45 | items.append( 46 | { 47 | "id": post["id"], 48 | "url": url_for_post(post), 49 | "content_html": content_for_post(post), 50 | "date_published": post["created_at"] + "Z", 51 | "title": title_for_post(post), 52 | "authors": [ 53 | { 54 | "name": AUTHOR_NAME, 55 | } 56 | ], 57 | } 58 | ) 59 | 60 | j = { 61 | "version": "https://jsonfeed.org/version/1.1", 62 | "title": "Multi Star", 63 | "home_page_url": f"https://github.com/{GITHUB_USERNAME}/{REPOSITORY_NAME}", 64 | "feed_url": f"https://{GITHUB_USERNAME}.github.io/{REPOSITORY_NAME}/{feed_name}", 65 | "items": sorted(items, key=lambda item: item["date_published"], reverse=True)[:max_items], 66 | } 67 | 68 | return j 69 | 70 | 71 | def parse_header_links(value): 72 | """Return a list of parsed link headers proxies. 73 | i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" 74 | :rtype: list 75 | 76 | Taken from psf/requests 77 | """ 78 | 79 | links = [] 80 | replace_chars = " '\"" 81 | value = value.strip(replace_chars) 82 | 83 | if not value: 84 | return links 85 | 86 | for val in re.split(", *<", value): 87 | try: 88 | url, params = val.split(";", 1) 89 | except ValueError: 90 | url, params = val, "" 91 | 92 | link = {"url": url.strip("<> '\"")} 93 | 94 | for param in params.split(";"): 95 | try: 96 | key, value = param.split("=") 97 | except ValueError: 98 | break 99 | 100 | link[key.strip(replace_chars)] = value.strip(replace_chars) 101 | 102 | links.append(link) 103 | 104 | return links 105 | 106 | 107 | def following(): 108 | """ 109 | Get all the followers for the current user 110 | """ 111 | url = "https://api.github.com/user/following" 112 | data = [] 113 | 114 | while url: 115 | response = request(url, headers={"Authorization": f"token {GITHUB_TOKEN}"}) 116 | data.extend(response.json) 117 | 118 | if "next" in response.headers.get("link"): 119 | links = parse_header_links(response.headers.get("link")) 120 | url = [link["url"] for link in links if link["rel"] == "next"][0] 121 | continue 122 | url = None 123 | 124 | return data 125 | 126 | 127 | def starred(user): 128 | """ 129 | Get all stars for a user, no pagination 130 | """ 131 | url = f"https://api.github.com/users/{user}/starred" 132 | params = {"per_page": 100} 133 | 134 | response = request( 135 | url, 136 | params=params, 137 | headers={"Accept": "application/vnd.github.v3.star+json", "Authorization": f"token {GITHUB_TOKEN}"}, 138 | ) 139 | 140 | stars = [] 141 | for star in response.json: 142 | if "starred_at" not in star: 143 | print(star) 144 | 145 | stars.append(star) 146 | 147 | print(user, len(stars)) 148 | return stars 149 | 150 | 151 | def repo_html(repo): 152 | url = f"https://api.github.com/repos/{repo}" 153 | response = request( 154 | url, 155 | headers={"Authorization": f"token {GITHUB_TOKEN}"}, 156 | ) 157 | 158 | html = f"

{response.json['full_name']}

" 159 | html += f"

{response.json['description']}

" 160 | html += f"

{response.json['language']}" 161 | html += f" | Stargazers: {response.json['stargazers_count']}" 162 | html += f" | Homepage: {response.json['homepage']}

" 163 | return html 164 | 165 | 166 | if __name__ == "__main__": 167 | feed_name = f"{FEED_NAME}.json" 168 | 169 | all_usernames = [x["login"] for x in following()] 170 | 171 | stars = [] 172 | for u in all_usernames: 173 | stars.extend(starred(u)) 174 | 175 | # filter to stars in the last ~60 days 176 | print("All stars:", len(stars)) 177 | after = (datetime.utcnow() - timedelta(days=60)).replace(tzinfo=timezone.utc) 178 | stars = [ 179 | star 180 | for star in stars 181 | if datetime.fromisoformat(star["starred_at"]).replace(tzinfo=timezone.utc) > after 182 | if "starred_at" in star 183 | ] 184 | print("Filtered by date:", len(stars)) 185 | 186 | repos = [] 187 | multi_stars = [] 188 | for s in stars: 189 | repo = s["repo"]["full_name"] 190 | if repo in repos and repo not in multi_stars: 191 | multi_stars.append(repo) 192 | repos.append(repo) 193 | 194 | print("Multi stars:", len(multi_stars)) 195 | 196 | posts = [ 197 | { 198 | "html": repo_html(repo), 199 | "id": repo, 200 | "title": repo, 201 | "created_at": datetime.utcnow().isoformat(), 202 | "url": f"https://github.com/{repo}", 203 | } 204 | for repo in multi_stars 205 | ] 206 | 207 | jsonfeed = save_jsonfeed(posts, feed_name=feed_name) 208 | 209 | with open(f"out/{feed_name}", "w") as f: 210 | f.write(json.dumps(jsonfeed, indent=2)) 211 | -------------------------------------------------------------------------------- /thttp.py: -------------------------------------------------------------------------------- 1 | """ 2 | UNLICENSED 3 | This is free and unencumbered software released into the public domain. 4 | 5 | https://github.com/sesh/thttp 6 | """ 7 | 8 | import gzip 9 | import json as json_lib 10 | import ssl 11 | from base64 import b64encode 12 | from collections import namedtuple 13 | from http import HTTPStatus 14 | from http.cookiejar import CookieJar 15 | from urllib.error import HTTPError, URLError 16 | from urllib.parse import urlencode 17 | from urllib.request import ( 18 | HTTPCookieProcessor, 19 | HTTPRedirectHandler, 20 | HTTPSHandler, 21 | Request, 22 | build_opener, 23 | ) 24 | 25 | Response = namedtuple("Response", "request content json status url headers cookiejar") 26 | 27 | 28 | class NoRedirect(HTTPRedirectHandler): 29 | def redirect_request(self, req, fp, code, msg, headers, newurl): 30 | return None 31 | 32 | 33 | def request( 34 | url, 35 | params={}, 36 | json=None, 37 | data=None, 38 | headers={}, 39 | method="GET", 40 | verify=True, 41 | redirect=True, 42 | cookiejar=None, 43 | basic_auth=None, 44 | timeout=None, 45 | ): 46 | """ 47 | Returns a (named)tuple with the following properties: 48 | - request 49 | - content 50 | - json (dict; or None) 51 | - headers (dict; all lowercase keys) 52 | - https://stackoverflow.com/questions/5258977/are-http-headers-case-sensitive 53 | - status 54 | - url (final url, after any redirects) 55 | - cookiejar 56 | """ 57 | method = method.upper() 58 | headers = {k.lower(): v for k, v in headers.items()} # lowercase headers 59 | 60 | if params: 61 | url += "?" + urlencode(params) # build URL from query parameters 62 | 63 | if json and data: 64 | raise Exception("Cannot provide both json and data parameters") 65 | 66 | if method not in ["POST", "PATCH", "PUT"] and (json or data): 67 | raise Exception("Request method must POST, PATCH or PUT if json or data is provided") 68 | 69 | if not timeout: 70 | timeout = 60 71 | 72 | if json: # if we have json, dump it to a string and put it in our data variable 73 | headers["content-type"] = "application/json" 74 | data = json_lib.dumps(json).encode("utf-8") 75 | elif data and not isinstance(data, (str, bytes)): 76 | data = urlencode(data).encode() 77 | elif isinstance(data, str): 78 | data = data.encode() 79 | 80 | if basic_auth and len(basic_auth) == 2 and "authorization" not in headers: 81 | username, password = basic_auth 82 | headers["authorization"] = f'Basic {b64encode(f"{username}:{password}".encode()).decode("ascii")}' 83 | 84 | if not cookiejar: 85 | cookiejar = CookieJar() 86 | 87 | ctx = ssl.create_default_context() 88 | if not verify: # ignore ssl errors 89 | ctx.check_hostname = False 90 | ctx.verify_mode = ssl.CERT_NONE 91 | 92 | handlers = [] 93 | handlers.append(HTTPSHandler(context=ctx)) 94 | handlers.append(HTTPCookieProcessor(cookiejar=cookiejar)) 95 | 96 | if not redirect: 97 | no_redirect = NoRedirect() 98 | handlers.append(no_redirect) 99 | 100 | opener = build_opener(*handlers) 101 | req = Request(url, data=data, headers=headers, method=method) 102 | 103 | try: 104 | with opener.open(req, timeout=timeout) as resp: 105 | status, content, resp_url = (resp.getcode(), resp.read(), resp.geturl()) 106 | headers = {k.lower(): v for k, v in list(resp.info().items())} 107 | 108 | if "gzip" in headers.get("content-encoding", ""): 109 | content = gzip.decompress(content) 110 | 111 | json = ( 112 | json_lib.loads(content) 113 | if "application/json" in headers.get("content-type", "").lower() and content 114 | else None 115 | ) 116 | except HTTPError as e: 117 | status, content, resp_url = (e.code, e.read(), e.geturl()) 118 | headers = {k.lower(): v for k, v in list(e.headers.items())} 119 | 120 | if "gzip" in headers.get("content-encoding", ""): 121 | content = gzip.decompress(content) 122 | 123 | json = ( 124 | json_lib.loads(content) 125 | if "application/json" in headers.get("content-type", "").lower() and content 126 | else None 127 | ) 128 | 129 | return Response(req, content, json, status, resp_url, headers, cookiejar) 130 | 131 | 132 | def pretty(response, headers_only=False): 133 | RESET = "\033[0m" 134 | HIGHLIGHT = "\033[34m" 135 | HTTP_STATUSES = {x.value: x.name for x in HTTPStatus} 136 | 137 | # status code 138 | print(HIGHLIGHT + str(response.status) + " " + RESET + HTTP_STATUSES.get(response.status, "")) 139 | 140 | # headers 141 | for k in sorted(response.headers.keys()): 142 | print(HIGHLIGHT + k + RESET + ": " + response.headers[k]) 143 | 144 | if headers_only: 145 | return 146 | 147 | # blank line 148 | print() 149 | 150 | # response body 151 | if response.json: 152 | print(json_lib.dumps(response.json, indent=2)) 153 | else: 154 | print(response.content.decode()) 155 | 156 | 157 | import contextlib # noqa: E402 158 | import unittest # noqa: E402 159 | from io import StringIO # noqa: E402 160 | 161 | 162 | class RequestTestCase(unittest.TestCase): 163 | def test_cannot_provide_json_and_data(self): 164 | with self.assertRaises(Exception): 165 | request( 166 | "https://httpbingo.org/post", 167 | json={"name": "Brenton"}, 168 | data="This is some form data", 169 | ) 170 | 171 | def test_should_fail_if_json_or_data_and_not_p_method(self): 172 | with self.assertRaises(Exception): 173 | request("https://httpbingo.org/post", json={"name": "Brenton"}) 174 | 175 | with self.assertRaises(Exception): 176 | request("https://httpbingo.org/post", json={"name": "Brenton"}, method="HEAD") 177 | 178 | def test_should_set_content_type_for_json_request(self): 179 | response = request("https://httpbingo.org/post", json={"name": "Brenton"}, method="POST") 180 | self.assertEqual(response.request.headers["Content-type"], "application/json") 181 | 182 | def test_should_work(self): 183 | response = request("https://httpbingo.org/get") 184 | self.assertEqual(response.status, 200) 185 | 186 | def test_should_create_url_from_params(self): 187 | response = request( 188 | "https://httpbingo.org/get", 189 | params={"name": "brenton", "library": "tiny-request"}, 190 | ) 191 | self.assertEqual(response.url, "https://httpbingo.org/get?name=brenton&library=tiny-request") 192 | 193 | def test_should_return_headers(self): 194 | response = request("https://httpbingo.org/response-headers", params={"Test-Header": "value"}) 195 | self.assertEqual(response.headers["test-header"], "value") 196 | 197 | def test_should_populate_json(self): 198 | response = request("https://httpbingo.org/json") 199 | self.assertTrue("slideshow" in response.json) 200 | 201 | def test_should_return_response_for_404(self): 202 | response = request("https://httpbingo.org/404") 203 | self.assertEqual(response.status, 404) 204 | self.assertTrue("text/plain" in response.headers["content-type"]) 205 | 206 | def test_should_fail_with_bad_ssl(self): 207 | with self.assertRaises(URLError): 208 | request("https://expired.badssl.com/") 209 | 210 | def test_should_load_bad_ssl_with_verify_false(self): 211 | response = request("https://expired.badssl.com/", verify=False) 212 | self.assertEqual(response.status, 200) 213 | 214 | def test_should_form_encode_non_json_post_requests(self): 215 | response = request("https://httpbingo.org/post", data={"name": "test-user"}, method="POST") 216 | self.assertEqual(response.json["form"]["name"], ["test-user"]) 217 | 218 | def test_should_follow_redirect(self): 219 | response = request( 220 | "https://httpbingo.org/redirect-to", 221 | params={"url": "https://example.org/"}, 222 | ) 223 | self.assertEqual(response.url, "https://example.org/") 224 | self.assertEqual(response.status, 200) 225 | 226 | def test_should_not_follow_redirect_if_redirect_false(self): 227 | response = request( 228 | "https://httpbingo.org/redirect-to", 229 | params={"url": "https://example.org/"}, 230 | redirect=False, 231 | ) 232 | self.assertEqual(response.status, 302) 233 | 234 | def test_cookies(self): 235 | response = request( 236 | "https://httpbingo.org/cookies/set", 237 | params={"cookie": "test"}, 238 | redirect=False, 239 | ) 240 | response = request("https://httpbingo.org/cookies", cookiejar=response.cookiejar) 241 | self.assertEqual(response.json["cookie"], "test") 242 | 243 | def test_basic_auth(self): 244 | response = request("http://httpbingo.org/basic-auth/user/passwd", basic_auth=("user", "passwd")) 245 | self.assertEqual(response.json["authorized"], True) 246 | 247 | def test_should_handle_gzip(self): 248 | response = request("http://httpbingo.org/gzip", headers={"Accept-Encoding": "gzip"}) 249 | self.assertEqual(response.json["gzipped"], True) 250 | 251 | def test_should_handle_gzip_error(self): 252 | response = request("http://httpbingo.org/status/418", headers={"Accept-Encoding": "gzip"}) 253 | self.assertEqual(response.content, b"I'm a teapot!") 254 | 255 | def test_should_timeout(self): 256 | import socket 257 | 258 | with self.assertRaises((TimeoutError, socket.timeout)): 259 | request("http://httpbingo.org/delay/3", timeout=1) 260 | 261 | def test_should_handle_head_requests(self): 262 | response = request("http://httpbingo.org/head", method="HEAD") 263 | self.assertTrue(response.content == b"") 264 | 265 | def test_should_post_data_string(self): 266 | response = request( 267 | "https://ntfy.sh/thttp-test-ntfy", 268 | data="The thttp test suite was executed!", 269 | method="POST", 270 | ) 271 | self.assertTrue(response.json["topic"] == "thttp-test-ntfy") 272 | 273 | def test_pretty_output(self): 274 | response = request("https://basehtml.xyz") 275 | 276 | f = StringIO() 277 | with contextlib.redirect_stdout(f): 278 | pretty(response) 279 | 280 | f.seek(0) 281 | output = f.read() 282 | 283 | self.assertTrue("text/html; charset=utf-8" in output) 284 | self.assertTrue("

base.html

" in output) 285 | 286 | def test_pretty_output_headers_only(self): 287 | response = request("https://basehtml.xyz") 288 | 289 | f = StringIO() 290 | with contextlib.redirect_stdout(f): 291 | pretty(response, headers_only=True) 292 | 293 | f.seek(0) 294 | output = f.read() 295 | 296 | self.assertTrue("text/html; charset=utf-8" in output) 297 | self.assertTrue("

base.html

" not in output) 298 | --------------------------------------------------------------------------------