├── .gitattributes ├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── ftlangdetect ├── __init__.py └── detect.py ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py └── test_detect.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.bin filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Fasttext langdetect 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.7", "3.8", "3.9", "3.10"] 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Set up Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v4 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install flake8 pytest 23 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 24 | 25 | - name: Lint with flake8 26 | run: | 27 | # stop the build if there are Python syntax errors or undefined names 28 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 29 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 30 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 31 | 32 | - name: Test with pytest 33 | run: | 34 | pytest 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .DS_Store 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Zafer Çavdar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | publish: 2 | python3 setup.py sdist 3 | twine upload dist/* 4 | 5 | test: 6 | pytest 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fasttext-langdetect 2 | This library is a wrapper for the language detection model trained on fasttext by Facebook. For more information, please visit: https://fasttext.cc/docs/en/language-identification.html 3 | 4 | 5 | ## Supported languages 6 | ``` 7 | af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce cebckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec vep vi vls vo wa war wuu xal xmf yi yo yue zh 8 | ``` 9 | 10 | ## Install 11 | ``` 12 | pip install fasttext-langdetect 13 | ``` 14 | 15 | ## Usage 16 | `detect` method expects UTF-8 data. `low_memory` option enables getting predictions with the compressed version of the fasttext model by sacrificing the accuracy a bit. 17 | 18 | ``` 19 | from ftlangdetect import detect 20 | 21 | result = detect(text="Bugün hava çok güzel", low_memory=False) 22 | print(result) 23 | > {'lang': 'tr', 'score': 1.00} 24 | 25 | result = detect(text="Bugün hava çok güzel", low_memory=True) 26 | print(result) 27 | > {'lang': 'tr', 'score': 0.9982126951217651} 28 | ``` 29 | 30 | ## Benchmark 31 | We benchmarked the fasttext model against [cld2](https://github.com/CLD2Owners/cld2), [langid](https://github.com/saffsd/langid.py), and [langdetect](https://github.com/Mimino666/langdetect) on Wili-2018 dataset. 32 | 33 | | | fasttext | langid | langdetect | cld2 | 34 | |--------------------------|-------------|-------------|-------------|-------------| 35 | | Average time (ms) | 0,158273381 | 1,726618705 | 12,44604317 | **0,028776978** | 36 | | 139 langs - not weighted | 76,8 | 61,6 | 37,6 | **80,8** | 37 | | 139 langs - pop weighted | **95,5** | 93,1 | 86,6 | 92,7 | 38 | | 44 langs - not weighted | **93,3** | 89,2 | 81,6 | 91,5 | 39 | | 44 langs - pop weighted | **96,6** | 94,8 | 89,4 | 93,4 | 40 | 41 | - `pop weighted` means recall for each language is multipled by [its number of speakers](https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers). 42 | - 139 languages = all languages with ISO 639-1 2-letter code 43 | - 44 languages = top 44 languages spoken in the world 44 | 45 | 46 | #### Recall per language 47 | | lang | cld2 | fasttext | langdetect | langid | 48 | |-------------------------|-------|----------|------------|--------| 49 | | Afrikaans | 0,94 | 0,918 | 0,992 | 0,966 | 50 | | Albanian | 0,958 | 0,966 | 0,964 | 0,954 | 51 | | Amharic | 0,976 | 0,982 | 0 | 0,982 | 52 | | Arabic | 0,994 | 0,998 | 0,998 | 0,996 | 53 | | Aragonese | 0 | 0,43 | 0 | 0,788 | 54 | | Armenian | 0,966 | 0,972 | 0 | 0,968 | 55 | | Assamese | 0,946 | 0,956 | 0 | 0,14 | 56 | | Avar | 0 | 0,626 | 0 | 0 | 57 | | Aymara | 0,596 | 0 | 0 | 0 | 58 | | Azerbaijani | 0,97 | 0,988 | 0 | 0,984 | 59 | | Bashkir | 0,97 | 0,97 | 0 | 0 | 60 | | Basque | 0,978 | 0,99 | 0 | 0,962 | 61 | | Belarusian | 0,94 | 0,97 | 0 | 0,964 | 62 | | Bengali | 0,898 | 0,922 | 0,904 | 0,942 | 63 | | Bhojpuri | 0,716 | 0,15 | 0 | 0 | 64 | | Bokmål | 0,852 | 0,966 | 0,976 | 0,95 | 65 | | Bosnian | 0,422 | 0,108 | 0 | 0,054 | 66 | | Breton | 0,946 | 0,974 | 0 | 0,976 | 67 | | Bulgarian | 0,892 | 0,964 | 0,964 | 0,942 | 68 | | Burmese | 0,998 | 0,998 | 0 | 0 | 69 | | Catalan | 0,882 | 0,95 | 0,93 | 0,928 | 70 | | Central Khmer | 0,876 | 0,878 | 0 | 0,876 | 71 | | Chechen | 0 | 0,99 | 0 | 0 | 72 | | Chuvash | 0 | 0,96 | 0 | 0 | 73 | | Cornish | 0 | 0,792 | 0 | 0 | 74 | | Corsican | 0,88 | 0,016 | 0 | 0 | 75 | | Croatian | 0,688 | 0,806 | 0,982 | 0,932 | 76 | | Czech | 0,978 | 0,986 | 0,984 | 0,982 | 77 | | Danish | 0,886 | 0,958 | 0,95 | 0,896 | 78 | | Dhivehi | 0,996 | 0,998 | 0 | 0 | 79 | | Dutch | 0,9 | 0,978 | 0,968 | 0,97 | 80 | | English | 0,992 | 1 | 0,998 | 0,986 | 81 | | Esperanto | 0,936 | 0,978 | 0 | 0,948 | 82 | | Estonian | 0,918 | 0,952 | 0,948 | 0,932 | 83 | | Faroese | 0,912 | 0 | 0 | 0,618 | 84 | | Finnish | 0,988 | 0,998 | 0,998 | 0,994 | 85 | | French | 0,946 | 0,996 | 0,99 | 0,992 | 86 | | Galician | 0,89 | 0,912 | 0 | 0,93 | 87 | | Georgian | 0,974 | 0,976 | 0 | 0,976 | 88 | | German | 0,958 | 0,984 | 0,978 | 0,978 | 89 | | Guarani | 0,968 | 0,728 | 0 | 0 | 90 | | Gujarati | 0,932 | 0,932 | 0,93 | 0,932 | 91 | | Haitian Creole | 0,988 | 0,536 | 0 | 0,99 | 92 | | Hausa | 0,976 | 0 | 0 | 0 | 93 | | Hebrew | 0,994 | 0,996 | 0,998 | 0,998 | 94 | | Hindi | 0,982 | 0,984 | 0,982 | 0,972 | 95 | | Hungarian | 0,96 | 0,988 | 0,968 | 0,986 | 96 | | Icelandic | 0,984 | 0,996 | 0 | 0,996 | 97 | | Ido | 0 | 0,76 | 0 | 0 | 98 | | Igbo | 0,798 | 0 | 0 | 0 | 99 | | Indonesian | 0,88 | 0,946 | 0,958 | 0,836 | 100 | | Interlingua | 0,27 | 0,688 | 0 | 0 | 101 | | Interlingue | 0,198 | 0,192 | 0 | 0 | 102 | | Irish | 0,968 | 0,978 | 0 | 0,984 | 103 | | Italian | 0,866 | 0,948 | 0,932 | 0,936 | 104 | | Japanese | 0,97 | 0,986 | 0,98 | 0,986 | 105 | | Javanese | 0 | 0,864 | 0 | 0,938 | 106 | | Kannada | 0,998 | 0,998 | 0,998 | 0,998 | 107 | | Kazakh | 0,978 | 0,992 | 0 | 0,916 | 108 | | Kinyarwanda | 0,86 | 0 | 0 | 0,44 | 109 | | Kirghiz | 0,974 | 0,99 | 0 | 0,408 | 110 | | Komi | 0 | 0,544 | 0 | 0 | 111 | | Korean | 0,986 | 0,99 | 0,988 | 0,99 | 112 | | Kurdish | 0 | 0,972 | 0 | 0,976 | 113 | | Lao | 0,84 | 0,842 | 0 | 0,85 | 114 | | Latin | 0,778 | 0,864 | 0 | 0,854 | 115 | | Latvian | 0,98 | 0,992 | 0,992 | 0,99 | 116 | | Limburgan | 0 | 0,324 | 0 | 0 | 117 | | Lingala | 0,85 | 0 | 0 | 0 | 118 | | Lithuanian | 0,96 | 0,976 | 0,974 | 0,97 | 119 | | Luganda | 0,952 | 0 | 0 | 0 | 120 | | Luxembourgish | 0,864 | 0,894 | 0 | 0,93 | 121 | | Macedonian | 0,88 | 0,984 | 0,982 | 0,974 | 122 | | Malagasy | 0,99 | 0,99 | 0 | 0,988 | 123 | | Malay | 0,896 | 0,586 | 0 | 0,39 | 124 | | Malayalam | 0,988 | 0,988 | 0,988 | 0,988 | 125 | | Maltese | 0,962 | 0,966 | 0 | 0,964 | 126 | | Manx | 0,972 | 0,294 | 0 | 0 | 127 | | Maori | 0,994 | 0 | 0 | 0 | 128 | | Marathi | 0,958 | 0,966 | 0,964 | 0,942 | 129 | | Modern Greek | 0,99 | 0,992 | 0,99 | 0,992 | 130 | | Mongolian | 0,964 | 0,994 | 0 | 0,996 | 131 | | Navajo | 0 | 0 | 0 | 0 | 132 | | Nepali (macrolanguage) | 0,96 | 0,98 | 0,978 | 0,922 | 133 | | Northern Sami | 0 | 0 | 0 | 0,866 | 134 | | Norwegian Nynorsk | 0,94 | 0,79 | 0 | 0,796 | 135 | | Occitan | 0,66 | 0,48 | 0 | 0,724 | 136 | | Oriya | 0,96 | 0,958 | 0 | 0,96 | 137 | | Oromo | 0,956 | 0 | 0 | 0 | 138 | | Ossetian | 0 | 0,938 | 0 | 0 | 139 | | Panjabi | 0,994 | 0,994 | 0,994 | 0,994 | 140 | | Persian | 0,992 | 0,998 | 0,996 | 0,998 | 141 | | Polish | 0,982 | 0,998 | 0,998 | 0,992 | 142 | | Portuguese | 0,908 | 0,956 | 0,946 | 0,952 | 143 | | Pushto | 0,938 | 0,922 | 0 | 0,754 | 144 | | Quechua | 0,926 | 0,808 | 0 | 0,852 | 145 | | Romanian | 0,932 | 0,986 | 0,984 | 0,984 | 146 | | Romansh | 0,934 | 0,328 | 0 | 0 | 147 | | Russian | 0,728 | 0,986 | 0,984 | 0,988 | 148 | | Sanskrit | 0,964 | 0,976 | 0 | 0 | 149 | | Sardinian | 0 | 0,01 | 0 | 0 | 150 | | Scottish Gaelic | 0,964 | 0,942 | 0 | 0 | 151 | | Serbian | 0,942 | 0,946 | 0 | 0,902 | 152 | | Serbo-Croatian | 0 | 0,402 | 0 | 0 | 153 | | Shona | 0,844 | 0 | 0 | 0 | 154 | | Sindhi | 0,978 | 0,982 | 0 | 0 | 155 | | Sinhala | 0,962 | 0,962 | 0 | 0,962 | 156 | | Slovak | 0,964 | 0,974 | 0,982 | 0,97 | 157 | | Slovene | 0,876 | 0,966 | 0,968 | 0,946 | 158 | | Somali | 0,924 | 0,696 | 0,956 | 0 | 159 | | Spanish | 0,894 | 0,986 | 0,976 | 0,98 | 160 | | Standard Chinese | 0,946 | 0,984 | 0,746 | 0,978 | 161 | | Sundanese | 0,91 | 0,854 | 0 | 0 | 162 | | Swahili (macrolanguage) | 0,924 | 0,92 | 0,938 | 0,934 | 163 | | Swedish | 0,872 | 0,994 | 0,992 | 0,986 | 164 | | Tagalog | 0,928 | 0,972 | 0,974 | 0,964 | 165 | | Tajik | 0,82 | 0,85 | 0 | 0 | 166 | | Tamil | 0,992 | 0,992 | 0,992 | 0,994 | 167 | | Tatar | 0,978 | 0,984 | 0 | 0 | 168 | | Telugu | 0,958 | 0,958 | 0,958 | 0,96 | 169 | | Thai | 0,988 | 0,988 | 0,988 | 0,988 | 170 | | Tibetan | 0,986 | 0,992 | 0 | 0 | 171 | | Tongan | 0,968 | 0 | 0 | 0 | 172 | | Tswana | 0,928 | 0 | 0 | 0 | 173 | | Turkish | 0,968 | 0,986 | 0,982 | 0,976 | 174 | | Turkmen | 0,94 | 0,936 | 0 | 0 | 175 | | Uighur | 0,978 | 0,986 | 0 | 0,964 | 176 | | Ukrainian | 0,97 | 0,988 | 0,986 | 0,986 | 177 | | Urdu | 0,86 | 0,958 | 0,89 | 0,896 | 178 | | Uzbek | 0,984 | 0,99 | 0 | 0 | 179 | | Vietnamese | 0,978 | 0,986 | 0,984 | 0,984 | 180 | | Volapük | 0,994 | 0,982 | 0 | 0,986 | 181 | | Walloon | 0 | 0,664 | 0 | 0,98 | 182 | | Welsh | 0,98 | 0,992 | 0,992 | 0,984 | 183 | | Western Frisian | 0,888 | 0,956 | 0 | 0 | 184 | | Wolof | 0,926 | 0 | 0 | 0 | 185 | | Xhosa | 0,928 | 0 | 0 | 0,912 | 186 | | Yiddish | 0,956 | 0,958 | 0 | 0 | 187 | | Yoruba | 0,75 | 0,262 | 0 | 0 | 188 | 189 | ## Star History 190 | 191 | [![Star History Chart](https://api.star-history.com/svg?repos=zafercavdar/fasttext-langdetect&type=Date)](https://star-history.com/#zafercavdar/fasttext-langdetect&Date) 192 | 193 | ## References 194 | [1] A. Joulin, E. Grave, P. Bojanowski, T. Mikolov, [Bag of Tricks for Efficient Text Classification](https://arxiv.org/abs/1607.01759) 195 | 196 | ``` 197 | @article{joulin2016bag, 198 | title={Bag of Tricks for Efficient Text Classification}, 199 | author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas}, 200 | journal={arXiv preprint arXiv:1607.01759}, 201 | year={2016} 202 | } 203 | ``` 204 | 205 | [2] A. Joulin, E. Grave, P. Bojanowski, M. Douze, H. Jégou, T. Mikolov, [FastText.zip: Compressing text classification models](https://arxiv.org/abs/1612.03651) 206 | 207 | ``` 208 | @article{joulin2016fasttext, 209 | title={FastText.zip: Compressing text classification models}, 210 | author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\'e}gou, H{\'e}rve and Mikolov, Tomas}, 211 | journal={arXiv preprint arXiv:1612.03651}, 212 | year={2016} 213 | } 214 | ``` 215 | -------------------------------------------------------------------------------- /ftlangdetect/__init__.py: -------------------------------------------------------------------------------- 1 | from .detect import detect 2 | 3 | __all__ = ["detect"] 4 | -------------------------------------------------------------------------------- /ftlangdetect/detect.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import Dict, Union 4 | 5 | import fasttext 6 | import requests 7 | 8 | logger = logging.getLogger(__name__) 9 | models = {"low_mem": None, "high_mem": None} 10 | FTLANG_CACHE = os.getenv("FTLANG_CACHE", "/tmp/fasttext-langdetect") 11 | 12 | 13 | def download_model(name: str) -> str: 14 | target_path = os.path.join(FTLANG_CACHE, name) 15 | if not os.path.exists(target_path): 16 | logger.info(f"Downloading {name} model ...") 17 | url = f"https://dl.fbaipublicfiles.com/fasttext/supervised-models/{name}" # noqa 18 | os.makedirs(FTLANG_CACHE, exist_ok=True) 19 | with open(target_path, "wb") as fp: 20 | response = requests.get(url) 21 | fp.write(response.content) 22 | logger.info(f"Downloaded.") 23 | return target_path 24 | 25 | 26 | def get_or_load_model(low_memory=False): 27 | if low_memory: 28 | model = models.get("low_mem", None) 29 | if not model: 30 | model_path = download_model("lid.176.ftz") 31 | model = fasttext.load_model(model_path) 32 | models["low_mem"] = model 33 | return model 34 | else: 35 | model = models.get("high_mem", None) 36 | if not model: 37 | model_path = download_model("lid.176.bin") 38 | model = fasttext.load_model(model_path) 39 | models["high_mem"] = model 40 | return model 41 | 42 | 43 | def detect(text: str, low_memory=False) -> Dict[str, Union[str, float]]: 44 | model = get_or_load_model(low_memory) 45 | labels, scores = model.predict(text) 46 | label = labels[0].replace("__label__", '') 47 | score = min(float(scores[0]), 1.0) 48 | return { 49 | "lang": label, 50 | "score": score, 51 | } 52 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fasttext >= 0.9.1 2 | requests >= 2.22.0 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools import find_packages 3 | 4 | setup(name='fasttext-langdetect', 5 | version='1.0.5', 6 | description='80x faster and 95% accurate language identification with Fasttext', 7 | keywords=['fasttext', 'langdetect', 'language detection', 8 | 'language identification'], 9 | long_description=open("README.md", "r", encoding='utf-8').read(), 10 | long_description_content_type="text/markdown", 11 | url='https://github.com/zafercavdar/fasttext-langdetect.git', 12 | download_url='https://github.com/zafercavdar/fasttext-langdetect/archive/refs/tags/v1.0.5.tar.gz', 13 | author='Zafer Cavdar', 14 | author_email='zafercavdar@yahoo.com', 15 | install_requires=[ 16 | "fasttext>=0.9.1", 17 | "requests>=2.22.0", 18 | ], 19 | license='MIT', 20 | packages=find_packages(), 21 | classifiers=[ 22 | 'Development Status :: 5 - Production/Stable', 23 | 'Intended Audience :: Developers', 24 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 25 | 'Topic :: Scientific/Engineering :: Information Analysis', 26 | 'License :: OSI Approved :: MIT License', 27 | 'Programming Language :: Python :: 3', 28 | 'Programming Language :: Python :: 3.5', 29 | 'Programming Language :: Python :: 3.6', 30 | 'Programming Language :: Python :: 3.7', 31 | 'Programming Language :: Python :: 3.8', 32 | 'Programming Language :: Python :: 3.9', 33 | ]) 34 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zafercavdar/fasttext-langdetect/84529c52262d1af5c2f744943b3e03c457af1e06/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_detect.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from ftlangdetect import detect 4 | 5 | 6 | class TestDetect(TestCase): 7 | 8 | def test_detect_low_mem(self): 9 | result = detect("Bugün hava çok güzel", low_memory=True) 10 | assert "lang" in result 11 | assert "score" in result 12 | assert isinstance(result["lang"], str) 13 | assert isinstance(result["score"], float) 14 | 15 | def test_detect_high_mem(self): 16 | result = detect("Bugün hava çok güzel", low_memory=False) 17 | assert "lang" in result 18 | assert "score" in result 19 | assert isinstance(result["lang"], str) 20 | assert isinstance(result["score"], float) 21 | --------------------------------------------------------------------------------