├── tests ├── __init__.py └── test_language_detector.py ├── MANIFEST.in ├── requirements.txt ├── spacy_cld ├── __init__.py ├── about.py └── spacy_cld.py ├── LICENSE ├── setup.py ├── README.md └── .gitignore /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | spacy>=2.0.0 2 | pycld2==0.31 3 | -------------------------------------------------------------------------------- /spacy_cld/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from .about import __version__ 3 | from .spacy_cld import LanguageDetector 4 | -------------------------------------------------------------------------------- /spacy_cld/about.py: -------------------------------------------------------------------------------- 1 | __title__ = 'spacy-cld' 2 | __version__ = '0.1.0' 3 | __summary__ = 'spaCy pipeline component for guessing the language of Doc and Span objects.' 4 | __url__ = 'https://github.com/nickdavidhaynes/spacy-cld' 5 | __author__ = 'Nick Haynes' 6 | __email__ = 'nickdavidhaynes@gmail.com' 7 | __license__ = 'MIT' 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Nicholas D Haynes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from setuptools import setup, find_packages 3 | 4 | 5 | def setup_package(): 6 | package_name = 'spacy_cld' 7 | root = Path(__file__).parent.resolve() 8 | 9 | # Read in package meta from about.py 10 | about_path = root / package_name / 'about.py' 11 | with about_path.open('r', encoding='utf8') as f: 12 | about = {} 13 | exec(f.read(), about) 14 | 15 | # Get readme 16 | readme_path = root / 'README.md' 17 | with readme_path.open('r', encoding='utf8') as f: 18 | readme = f.read() 19 | 20 | setup( 21 | name=package_name, 22 | description=about['__summary__'], 23 | long_description=readme, 24 | author=about['__author__'], 25 | author_email=about['__email__'], 26 | url=about['__url__'], 27 | version=about['__version__'], 28 | license=about['__license__'], 29 | packages=find_packages(), 30 | install_requires=[ 31 | 'spacy>=2.0.0,<3.0.0', 32 | 'pycld2>=0.31'], 33 | zip_safe=False, 34 | ) 35 | 36 | 37 | if __name__ == '__main__': 38 | setup_package() 39 | -------------------------------------------------------------------------------- /tests/test_language_detector.py: -------------------------------------------------------------------------------- 1 | import spacy 2 | from spacy_cld import LanguageDetector 3 | 4 | nlp = spacy.load('en') 5 | language_detector = LanguageDetector() 6 | nlp.add_pipe(language_detector) 7 | 8 | 9 | def test_english_doc(): 10 | doc = nlp('This is some English text.') 11 | assert 'en' in doc._.languages 12 | assert doc._.language_scores['en'] > 0.9 13 | 14 | 15 | def test_chinese_doc(): 16 | doc = nlp('أفاد مصدر امني في قيادة عمليات صلاح الدين في العراق بأن القوات الامنية تتوقف لليومالثالث على التوالي عن التقدم الى داخل مدينة تكريت بسببانتشار قناصي التنظيم الذي يطلق على نفسه اسم الدولة الاسلامية والعبوات الناسفةوالمنازل المفخخة والانتحاريين، فضلا عن ان القوات الامنية تنتظر وصول تعزيزات اضافية') 17 | assert 'ar' in doc._.languages 18 | assert doc._.language_scores['ar'] > 0.9 19 | 20 | 21 | def test_bilingual_doc(): 22 | doc = nlp("China (simplified Chinese: 中国; traditional Chinese: 中國), officially the People's Republic of China (PRC), is a sovereign state located in East Asia.") 23 | assert 'en' in doc._.languages 24 | assert 'zh-Hant' in doc._.languages 25 | 26 | 27 | def test_english_span(): 28 | doc = nlp('This is a sentence of English text.') 29 | assert 'en' in doc[1:-1]._.languages 30 | 31 | 32 | def test_unknown_not_in_languages(): 33 | doc = nlp('This is a sentence of English text.') 34 | assert 'un' not in doc._.languages 35 | 36 | 37 | def test_unknown_not_in_language_scores(): 38 | doc = nlp('This is a sentence of English text.') 39 | assert 'un' not in doc._.language_scores.keys() 40 | -------------------------------------------------------------------------------- /spacy_cld/spacy_cld.py: -------------------------------------------------------------------------------- 1 | from pycld2 import detect, error as pycld_error 2 | from spacy.tokens import Doc, Span 3 | 4 | 5 | def get_languages(text, cld_results=None): 6 | if cld_results is None: 7 | cld_results = detect_languages(text) 8 | # Ignore 'Unknown' language (code = 'un') 9 | return [lang for (_, lang, _, _) in cld_results if lang != 'un'] 10 | 11 | 12 | def get_scores(text, cld_results=None): 13 | if cld_results is None: 14 | cld_results = detect_languages(text) 15 | return {lang: score / 100. for (_, lang, score, _) 16 | in cld_results if lang != 'un'} 17 | 18 | 19 | def detect_languages(text): 20 | try: 21 | _, _, results = detect(text.text) 22 | except pycld_error as err: 23 | results = [[None, "error", 0.0, None]] 24 | return results 25 | 26 | 27 | class LanguageDetector(object): 28 | 29 | name = 'cld' 30 | 31 | def __init__(self, attrs=('languages', 'language_scores')): 32 | self._languages, self._scores = attrs 33 | Doc.set_extension(self._languages, getter=get_languages) 34 | Doc.set_extension(self._scores, getter=get_scores) 35 | Span.set_extension(self._languages, getter=get_languages) 36 | Span.set_extension(self._scores, getter=get_scores) 37 | 38 | def __call__(self, doc): 39 | '''Apply the language detector as a pipeline component.''' 40 | # Make a single call to the language detector and cache the result. 41 | cld_results = detect_languages(doc) 42 | doc._.set(self._languages, get_languages(doc, cld_results)) 43 | doc._.set(self._scores, get_scores(doc, cld_results)) 44 | return doc 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spaCy-CLD: Bringing simple language detection to spaCy 2 | 3 | This package is a [spaCy 2.0 extension](https://spacy.io/usage/processing-pipelines#section-extensions) that adds language detection to spaCy's text processing pipeline. Inspired from a discussion [here](https://github.com/explosion/spaCy/issues/1172). 4 | 5 | ## Installation 6 | 7 | `pip install spacy_cld` 8 | 9 | ## Usage 10 | 11 | Adding the spaCy-CLD component to the processing pipeline is relatively simple: 12 | 13 | ``` 14 | import spacy 15 | from spacy_cld import LanguageDetector 16 | 17 | nlp = spacy.load('en') 18 | language_detector = LanguageDetector() 19 | nlp.add_pipe(language_detector) 20 | doc = nlp('This is some English text.') 21 | 22 | doc._.languages # ['en'] 23 | doc._.language_scores['en'] # 0.96 24 | ``` 25 | 26 | spaCy-CLD operates on `Doc` and `Span` spaCy objects. When called on a `Doc` or `Span`, the object is given two attributes: `languages` (a list of up to 3 language codes) and `language_scores` (a dictionary mapping language codes to confidence scores between 0 and 1). 27 | 28 | ## Under the hood 29 | 30 | spacy-cld is a little extension that wraps the [PYCLD2](https://github.com/aboSamoor/pycld2) Python library, which in turn wraps the [Compact Language Detector 2](https://github.com/CLD2Owners/cld2) C library originally built at Google for the Chromium project. CLD2 uses character n-grams as features and a Naive Bayes classifier to identify 80+ languages from Unicode text strings (or XML/HTML). It can detect up to 3 different languages in a given document, and reports a confidence score (reported in with each language. 31 | 32 | For additional details, see the linked project pages for PYCLD2 and CLD2. 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | --------------------------------------------------------------------------------