├── .env.example ├── .envrc ├── .flake8 ├── .github └── workflows │ └── refresh_data.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── docs ├── assets │ └── android-chrome-96x96.png ├── index.html ├── script.js └── style.css ├── fdroid_data_exporter.py ├── fdroid_html_builder.py ├── imgs └── 2023-07-10_fdroid-insights.png ├── shell.nix ├── template.html └── utils.py /.env.example: -------------------------------------------------------------------------------- 1 | GITHUB_TOKEN=GITHUB_TOKEN_HERE 2 | GITLAB_TOKEN=GITLAB_TOKEN_HERE # for gitlab.com instance 3 | GLTLAB_KDE_TOKEN=GITLAB_KDE_TOKEN_HERE # for invent.kde.org instance 4 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | use nix 2 | dotenv 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=120 3 | per-file-ignores = __init__.py:F401 4 | -------------------------------------------------------------------------------- /.github/workflows/refresh_data.yaml: -------------------------------------------------------------------------------- 1 | name: refresh-data 2 | on: 3 | workflow_dispatch: # allows manual triggering 4 | schedule: 5 | - cron: '16 6 * * 1' # runs every Monday at 6:16 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@v2 13 | - name: Set up Python 3.10 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: '3.10' 17 | - name: Download latest F-Droid data 18 | run: | 19 | curl -O "https://f-droid.org/repo/index-v2.json" 20 | - name: Initialize python env 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install requests pandas pygithub python-gitlab 24 | - name: Create new export 25 | run: | 26 | python fdroid_data_exporter.py 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 29 | GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} 30 | - name: Create new template 31 | run: | 32 | python fdroid_html_builder.py 33 | - name: Pull Remote Changes 34 | run: git pull 35 | - uses: stefanzweifel/git-auto-commit-action@v4 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | *.json 3 | *.csv 4 | todo.txt 5 | .direnv/ 6 | __pycache__/ 7 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: check-yaml 10 | - id: check-added-large-files 11 | - repo: https://github.com/psf/black 12 | rev: 23.3.0 13 | hooks: 14 | - id: black 15 | language_version: python3.10 16 | - repo: https://github.com/pycqa/flake8 17 | rev: 6.0.0 18 | hooks: 19 | - id: flake8 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 David BELEY 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fdroid-insights 2 | 3 | ![F-Droid Insights - Easily explore F-Droid apps with data from external sources](imgs/2023-07-10_fdroid-insights.png) 4 | 5 | [**F-Droid Insights**](https://dbeley.github.io/fdroid-insights) is a simple website that lets you explore F-Droid apps enriched by data coming from external websites such as Github. It facilitates the discovery of popular and well-maintained projects which might be challenging to locate on the regular F-Droid website. 6 | 7 | ## Motivation 8 | 9 | F-Droid is the best app store to find FOSS Android apps. 10 | 11 | It allows thousands of independent developers to freely publish their apps and making them available to the general public. 12 | 13 | Operating solely on donations to sustain their infrastructure, F-Droid offers an ad-free experience, without any premium features or sponsored apps. 14 | 15 | On top of that they also don't track their users, you don't even need an account to download apps. 16 | However, the lack of information regarding app metrics (download count, similar apps) represents a challenge when selecting among multiple options. 17 | 18 | The idea behind **F-Droid Insights** is to leverage external metrics to help finding popular and well-maintained F-Droid apps. 19 | 20 | ## Usage 21 | 22 | The data should be re-generated every monday. 23 | 24 | If you want to manually create an export: 25 | 26 | - Download `index-v2.json` from [F-Droid](https://f-droid.org/en/docs/All_our_APIs) (~35 MB file). 27 | - Copy `.env.example` to `.env` and fill it with your own tokens. 28 | - `python fdroid_data_exporter.py`: will create `export.csv` containing F-Droid apps data. 29 | - `python fdroid_html_builder.py`: will create `docs/index.html` with `export.csv` and `template.html`. 30 | 31 | ## External data 32 | 33 | - Github: number of stars, forks 34 | - Gitlab : number of stars, forks 35 | - Codeberg : number of stars, forks 36 | -------------------------------------------------------------------------------- /docs/assets/android-chrome-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbeley/fdroid-insights/0cf9f13d371791de6ad8c5a1644758f113f2ea94/docs/assets/android-chrome-96x96.png -------------------------------------------------------------------------------- /docs/script.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | //Only needed for the filename of export files. 3 | //Normally set in the title tag of your page. 4 | document.title='F-Droid Insights - Easily explore F-Droid apps with data from external sources'; 5 | var numbersType = $.fn.dataTable.absoluteOrderNumber( [ 6 | { value: 'N/A', position: 'bottom' } 7 | ] ); 8 | 9 | // DataTable initialisation 10 | $('#insights').DataTable( 11 | { 12 | "dom": 'BlfrtipQ', 13 | "buttons": [ 14 | 'copy', 15 | 'csv', 16 | 'excel', 17 | 'print', 18 | 'colvis' 19 | ], 20 | "paging": true, 21 | "autoWidth": true, 22 | "aLengthMenu": [[10, 25, 50, 100, 1000, -1], [10, 25, 50, 100, 1000, "All"]], 23 | "pageLength": 25, 24 | "scrollToTop": true, 25 | "order": [[8, 'desc']], 26 | "columnDefs": [ 27 | { type: numbersType, targets: [2, 3] }, 28 | { targets: [7, 8], render: DataTable.render.datetime('x', 'YYYY-MM-DD', 'en') }, 29 | { width: "20%", targets: 0 }, 30 | ], 31 | "responsive": true 32 | } 33 | ); 34 | }); 35 | -------------------------------------------------------------------------------- /docs/style.css: -------------------------------------------------------------------------------- 1 | body {margin:2em;} 2 | -------------------------------------------------------------------------------- /fdroid_data_exporter.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import logging 4 | import pandas as pd 5 | from github import Github 6 | from gitlab import Gitlab 7 | from utils import get_github_repository_data 8 | from utils import get_gitlab_repository_data 9 | from utils import get_codeberg_repository_data 10 | 11 | logging.basicConfig(level=logging.INFO, format="%(levelname)s :: %(message)s") 12 | logger = logging.getLogger(__name__) 13 | 14 | REPOSITORY_APIS = { 15 | "github.com": Github(os.environ.get("GITHUB_TOKEN")), 16 | "gitlab.com": Gitlab(private_token=os.environ.get("GILAB_TOKEN")), 17 | "invent.kde.org": Gitlab( 18 | url="https://invent.kde.org", private_token=os.environ.get("GILAB_KDE_TOKEN") 19 | ), 20 | } 21 | 22 | 23 | def _extract_repository_name(repository: str, repository_domain: str) -> str: 24 | repository_name = ( 25 | repository.split("://")[-1].split(f"{repository_domain}/")[-1].strip("/") 26 | ) 27 | if repository_name.count("/") > 1: 28 | repository_name = ( 29 | repository_name.split("/")[0] + "/" + repository_name.split("/")[1] 30 | ) 31 | return repository_name 32 | 33 | 34 | def _get_repository_stats(repository: str, repository_domain: str) -> dict[str, str]: 35 | if not repository: 36 | return {} 37 | repository_name = _extract_repository_name(repository, repository_domain) 38 | match repository_domain: 39 | case "github.com": 40 | return get_github_repository_data( 41 | REPOSITORY_APIS["github.com"], repository_name 42 | ) 43 | case "codeberg.org": 44 | return get_codeberg_repository_data(repository_name) 45 | case "gitlab.com": 46 | return get_gitlab_repository_data( 47 | REPOSITORY_APIS["gitlab.com"], repository_name 48 | ) 49 | case "invent.kde.org": 50 | return get_gitlab_repository_data( 51 | REPOSITORY_APIS["invent.kde.org"], repository_name 52 | ) 53 | return {} 54 | 55 | 56 | with open("index-v2.json", "rb") as f: 57 | json_object = json.load(f) 58 | 59 | # two parent objects: repo and packages 60 | # repo object: name, description, icon, address, mirrors, timestamp, antiFeatures, categories, releaseChannels 61 | # packages object: name of all the apps (4137+) 62 | 63 | list_data = [] 64 | json_items = json_object["packages"].items() 65 | logger.info("Fetching data for %s apps...", len(json_items)) 66 | for index, (app_name, app_raw_data) in enumerate(json_items, 1): 67 | logger.info( 68 | "[%s/%s] Fetching data for %s", 69 | str(index).zfill(len(str(len(json_items)))), 70 | len(json_items), 71 | app_name, 72 | ) 73 | metadata = app_raw_data["metadata"] 74 | repository = metadata.get("sourceCode", "") 75 | repository_domain = repository.split("://")[1].split("/")[0] if repository else "" 76 | repository_stats = _get_repository_stats(repository, repository_domain) 77 | summary_en = metadata["summary"].get("en-US", "") if "summary" in metadata else "" 78 | description_en = ( 79 | metadata["description"].get("en-US", "") if "description" in metadata else "" 80 | ) 81 | icon_url = metadata["icon"].get("en-US", "") if "icon" in metadata else "" 82 | if icon_url: 83 | icon_url = f"https://f-droid.org/repo{icon_url['name']}" 84 | 85 | categories = ", ".join(metadata.get("categories", [])) 86 | 87 | list_data.append( 88 | { 89 | "name": metadata["name"]["en-US"], 90 | "icon": icon_url, 91 | "id": app_name, 92 | "repository": repository, 93 | **repository_stats, 94 | "repository_domain": repository_domain, 95 | "summary": summary_en, 96 | # 'description': description_en, 97 | "categories": categories, 98 | "added": metadata["added"], 99 | "last_updated": metadata["lastUpdated"], 100 | "url": f"https://f-droid.org/en/packages/{app_name}", 101 | } 102 | ) 103 | 104 | df = pd.DataFrame.from_records(list_data) 105 | df = df.astype( 106 | { 107 | "repository_stars_count": "Int64", 108 | "repository_forks_count": "Int64", 109 | } 110 | ) 111 | df = df.sort_values(by=["name"]) 112 | df.to_csv("export.csv", index=False) 113 | -------------------------------------------------------------------------------- /fdroid_html_builder.py: -------------------------------------------------------------------------------- 1 | from string import Template 2 | from datetime import datetime 3 | import pandas as pd 4 | 5 | 6 | def read_template(file: str) -> Template: 7 | with open(file, "r") as f: 8 | content = f.read() 9 | return Template(content) 10 | 11 | 12 | df = pd.read_csv("export.csv") 13 | df = df.astype( 14 | { 15 | "repository_stars_count": "Int64", 16 | "repository_forks_count": "Int64", 17 | } 18 | ) 19 | df = df.fillna( 20 | { 21 | # "repository_forks_count": 0, 22 | # "repository_stars_count": 0, 23 | "repository": "", 24 | "repository_domain": "", 25 | "summary": "", 26 | } 27 | ) 28 | 29 | header = ( 30 | "\n" 31 | "\n" 32 | "Name\n" 33 | "Repository\n" 34 | "Repository Stars Count\n" 35 | "Repository Forks Count\n" 36 | "Repository Domain\n" 37 | "Summary\n" 38 | "Categories\n" 39 | "Added\n" 40 | "Last Updated\n" 41 | "\n" 42 | "\n" 43 | ) 44 | 45 | table_data = "\n" 46 | for index, row in df.iterrows(): 47 | name = f"{row['name']}" 48 | table_data += ( 49 | "\n" 50 | "" 51 | f"" 52 | "\n" 53 | f"{name}" 54 | "\n" 55 | f"{row['repository']}" 56 | "\n" 57 | f"{row['repository_stars_count']}" 58 | "\n" 59 | f"{row['repository_forks_count']}" 60 | "\n" 61 | f"{row['repository_domain']}" 62 | "\n" 63 | f"{row['summary']}" 64 | "\n" 65 | f"{row['categories']}" 66 | "\n" 67 | f"{row['added']}" 68 | "\n" 69 | f"{row['last_updated']}" 70 | "\n" 71 | "\n" 72 | ) 73 | table_data += "\n" 74 | 75 | date_update = datetime.today().strftime("%Y-%m-%d") 76 | 77 | formatted_message = read_template("template.html").safe_substitute( 78 | {"date_update": date_update, "header": header, "table_data": table_data} 79 | ) 80 | with open("docs/index.html", "w") as f: 81 | f.write(formatted_message) 82 | -------------------------------------------------------------------------------- /imgs/2023-07-10_fdroid-insights.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbeley/fdroid-insights/0cf9f13d371791de6ad8c5a1644758f113f2ea94/imgs/2023-07-10_fdroid-insights.png -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | with import { }; 2 | 3 | let 4 | pythonPackages = python3Packages; 5 | in pkgs.mkShell rec { 6 | name = "fdroidStatsPythonEnv"; 7 | buildInputs = [ 8 | pythonPackages.python 9 | pythonPackages.pip 10 | 11 | pythonPackages.requests 12 | pythonPackages.pandas 13 | pythonPackages.pygithub 14 | pythonPackages.python-gitlab 15 | 16 | pre-commit 17 | ]; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | F-Droid Insights - Easily explore F-Droid apps with data from external sources 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |

15 | F-Droid logo F-Droid Insights 16 |

17 |

Source code

18 |

Easily explore F-Droid apps with data from external sources. Data was last updated on ${date_update}.

19 | 20 | ${header} 21 | ${table_data} 22 |
23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import calendar 2 | import logging 3 | import requests 4 | import time 5 | from github import Github 6 | from gitlab import Gitlab 7 | from gitlab.exceptions import GitlabGetError 8 | from github.GithubException import UnknownObjectException 9 | from github.GithubException import RateLimitExceededException 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def get_github_repository_data(g: Github, repository_name: str): 15 | try: 16 | repo = g.get_repo(repository_name) 17 | except UnknownObjectException: 18 | logger.warning(f"Repository {repository_name} was not found on Github.") 19 | return {} 20 | except RateLimitExceededException: 21 | core_rate_limit = g.get_rate_limit().core 22 | reset_timestamp = calendar.timegm(core_rate_limit.reset.timetuple()) 23 | sleep_time = reset_timestamp - calendar.timegm(time.gmtime()) + 10 24 | logger.warning(f"Rate-limit detected, sleeping for {sleep_time} seconds.") 25 | time.sleep(sleep_time) 26 | repo = g.get_repo(repository_name) 27 | return { 28 | "repository_stars_count": repo.stargazers_count, 29 | "repository_forks_count": repo.forks_count, 30 | } 31 | 32 | 33 | def get_gitlab_repository_data(gl: Gitlab, repository_name: str): 34 | try: 35 | repo = gl.projects.get(repository_name) 36 | except GitlabGetError: 37 | logger.warning(f"Repository {repository_name} was not found on Gitlab.") 38 | return {} 39 | return { 40 | "repository_stars_count": repo.star_count, 41 | "repository_forks_count": repo.forks_count, 42 | } 43 | 44 | 45 | def get_codeberg_repository_data(repository_name: str): 46 | result = requests.get(f"https://codeberg.org/api/v1/repos/{repository_name}") 47 | if result.status_code == 200: 48 | json_result = result.json() 49 | return { 50 | "repository_stars_count": json_result.get("stars_count"), 51 | "repository_forks_count": json_result.get("forks_count"), 52 | } 53 | return {} 54 | --------------------------------------------------------------------------------