├── documents ├── hi.txt ├── hello.txt ├── welcome.txt └── anomalie.zeta ├── .github └── FUNDING.yml ├── setup.cfg ├── pictures ├── PYSIMILAR.png ├── pyswahili.jpeg ├── Kioo LTD Contacts.docx ├── become_a_patron_button.png ├── codeimg-twitter-instream-image.jpeg ├── codeimg-twitter-instream-image (1).jpeg └── codeimg-twitter-instream-image (2).jpeg ├── LICENSE ├── setup.py ├── .gitignore ├── index.md ├── description.md ├── README.md └── pysimilar └── __init__.py /documents/hi.txt: -------------------------------------------------------------------------------- 1 | Hi -------------------------------------------------------------------------------- /documents/hello.txt: -------------------------------------------------------------------------------- 1 | hello -------------------------------------------------------------------------------- /documents/welcome.txt: -------------------------------------------------------------------------------- 1 | Welcome Hi -------------------------------------------------------------------------------- /documents/anomalie.zeta: -------------------------------------------------------------------------------- 1 | Hi hello Welcome -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | patreon: kalebujordan 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # General setup 2 | 3 | [metadata] 4 | description-file = DESCRIPTION.md -------------------------------------------------------------------------------- /pictures/PYSIMILAR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/PYSIMILAR.png -------------------------------------------------------------------------------- /pictures/pyswahili.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/pyswahili.jpeg -------------------------------------------------------------------------------- /pictures/Kioo LTD Contacts.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/Kioo LTD Contacts.docx -------------------------------------------------------------------------------- /pictures/become_a_patron_button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/become_a_patron_button.png -------------------------------------------------------------------------------- /pictures/codeimg-twitter-instream-image.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/codeimg-twitter-instream-image.jpeg -------------------------------------------------------------------------------- /pictures/codeimg-twitter-instream-image (1).jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/codeimg-twitter-instream-image (1).jpeg -------------------------------------------------------------------------------- /pictures/codeimg-twitter-instream-image (2).jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kalebu/pysimilar/HEAD/pictures/codeimg-twitter-instream-image (2).jpeg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Jordan Kalebu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from setuptools import setup 3 | 4 | # read the contents of your description file 5 | 6 | this_directory = path.abspath(path.dirname(__file__)) 7 | with open(path.join(this_directory, "description.md"), encoding="utf-8") as f: 8 | long_description = f.read() 9 | 10 | setup( 11 | name="pysimilar", 12 | version="0.5", 13 | description="A very light python libary for comparing similarity between text/strings", 14 | long_description=long_description, 15 | long_description_content_type="text/markdown", 16 | url="https://github.com/Kalebu/pysimilar", 17 | download_url="https://github.com/Kalebu/pysimilar/archive/0.2.tar.gz", 18 | author="Jordan Kalebu", 19 | author_email="isaackeinstein@gmail.com", 20 | license="MIT", 21 | packages=["pysimilar"], 22 | keywords=[ 23 | "pysimilar", 24 | "python-plagiarism-library", 25 | "natural language processing", 26 | "NLP libary", 27 | "python-tanzania", 28 | ], 29 | install_requires=[ 30 | "scikit-learn", 31 | ], 32 | python_requires=">=3.6", 33 | classifiers=[ 34 | "Development Status :: 3 - Alpha", 35 | "Intended Audience :: Developers", 36 | "Topic :: Software Development :: Build Tools", 37 | "License :: OSI Approved :: MIT License", 38 | "Programming Language :: Python :: 3.6", 39 | "Programming Language :: Python :: 3.7", 40 | "Programming Language :: Python :: 3.8", 41 | "Programming Language :: Python :: 3.9", 42 | ], 43 | ) 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | # [pysimilar](https://kalebu.github.io/pysimilar/) 2 | 3 | A python library for computing the similarity between two string(text) based on cosine similarity made by [kalebu](https://github.com/Kalebu) 4 | 5 | ## How does it work ? 6 | 7 | It uses Tfidf Vectorizer to transform the text into vectors and then obtained vectors are converted into arrays of numbers and then finally cosine similary computation is employed resulting to output indicating how similar they are. 8 | 9 | ## Example of usage 10 | 11 | Pysimilar allows you to either specify the string you want to compare directly or specify path to files containing string you want to compare. 12 | 13 | ### compare() strings 14 | 15 | Here an example on how to compare strings directly; 16 | 17 | ```python 18 | >>> from pysimilar import compare 19 | >>> compare('very light indeed', 'how fast is light') 20 | 0.17077611319011649 21 | ``` 22 | 23 | ### compare () files 24 | 25 | Here how to compare files with textual documents; 26 | 27 | ```python 28 | >>> compare('README.md', 'LICENSE', isfile=True) 29 | 0.25545580376557886 30 | ``` 31 | 32 | You can also compare documents with particular **extension** in a given directory, for instance let's say I want to compare all the documents with **.txt** in a **documents** directory here is what I will do; 33 | 34 | Directory for documents used by the example below look like this 35 | 36 | ```bash 37 | documents/ 38 | ├── anomalie.zeta 39 | ├── hello.txt 40 | ├── hi.txt 41 | └── welcome.txt 42 | ``` 43 | 44 | ### compare_documents () 45 | 46 | Here how to compare files of a particular extension 47 | 48 | ```python 49 | >>> import pysimilar 50 | >>> from pprint import pprint 51 | >>> pysimilar.extensions = '.txt' 52 | >>> comparison_result = pysimilar.compare_documents('documents') 53 | >>> [['welcome.txt vs hi.txt', 0.6053485081062917], 54 | ['welcome.txt vs hello.txt', 0.0], 55 | ['hi.txt vs hello.txt', 0.0]] 56 | ``` 57 | 58 | ### sorting the outputs 59 | 60 | You can also sort the comparison score based on their score by changing the **ascending** parameter, just as shown below; 61 | 62 | ```python 63 | >>> comparison_result = pysimilar.compare_documents('documents', ascending=True) 64 | >>> pprint(comparison_result) 65 | [['welcome.txt vs hello.txt', 0.0], 66 | ['hi.txt vs hello.txt', 0.0], 67 | ['welcome.txt vs hi.txt', 0.6053485081062917]] 68 | ``` 69 | 70 | ## multiple extensions 71 | 72 | You can also set pysimilar to include files with multiple extensions 73 | 74 | ```python 75 | >>> import pysimilar 76 | >>> from pprint import pprint 77 | >>> pysimilar.extensions = ['.txt', '.zeta'] 78 | >>> comparison_result = pysimilar.compare_documents('documents', ascending=True) 79 | >>> pprint(comparison_result) 80 | [['welcome.txt vs hello.txt', 0.0], 81 | ['hi.txt vs hello.txt', 0.0], 82 | ['anomalie.zeta vs hi.txt', 0.4968161174826459], 83 | ['welcome.txt vs hi.txt', 0.6292275146695526], 84 | ['welcome.txt vs anomalie.zeta', 0.7895651507603823]] 85 | 86 | ``` 87 | 88 | ## Contributions 89 | 90 | If you have anything valuable to add to the *lib*, whether its a documentation, typo error, source code, please don't hesitate to contribute just fork it and submit your pull request and I will try to be as friendly as I can to assist you making the contributions. 91 | 92 | 93 | ## All the Credits 94 | 95 | All the Credits to [kalebu](https://github.com/Kalebu) and other future contributors -------------------------------------------------------------------------------- /description.md: -------------------------------------------------------------------------------- 1 | # [pysimilar](https://kalebu.github.io/pysimilar/) 2 | 3 | A python library for computing the similarity between two string(text) based on cosine similarity made by [kalebu](https://github.com/Kalebu) 4 | 5 | ## How does it work ? 6 | 7 | It uses Tfidf Vectorizer to transform the text into vectors and then obtained vectors are converted into arrays of numbers and then finally cosine similary computation is employed resulting to output indicating how similar they are. 8 | 9 | ## Example of usage 10 | 11 | Pysimilar allows you to either specify the string you want to compare directly or specify path to files containing string you want to compare. 12 | 13 | ### compare() strings 14 | 15 | Here an example on how to compare strings directly; 16 | 17 | ```python 18 | >>> from pysimilar import compare 19 | >>> compare('very light indeed', 'how fast is light') 20 | 0.17077611319011649 21 | ``` 22 | 23 | ### compare () files 24 | 25 | Here how to compare files with textual documents; 26 | 27 | ```python 28 | >>> compare('README.md', 'LICENSE', isfile=True) 29 | 0.25545580376557886 30 | ``` 31 | 32 | You can also compare documents with particular **extension** in a given directory, for instance let's say I want to compare all the documents with **.txt** in a **documents** directory here is what I will do; 33 | 34 | Directory for documents used by the example below look like this 35 | 36 | ```bash 37 | documents/ 38 | ├── anomalie.zeta 39 | ├── hello.txt 40 | ├── hi.txt 41 | └── welcome.txt 42 | ``` 43 | 44 | ### compare_documents () 45 | 46 | Here how to compare files of a particular extension 47 | 48 | ```python 49 | >>> import pysimilar 50 | >>> from pprint import pprint 51 | >>> pysimilar.extensions = '.txt' 52 | >>> comparison_result = pysimilar.compare_documents('documents') 53 | >>> [['welcome.txt vs hi.txt', 0.6053485081062917], 54 | ['welcome.txt vs hello.txt', 0.0], 55 | ['hi.txt vs hello.txt', 0.0]] 56 | ``` 57 | 58 | ### sorting the outputs 59 | 60 | You can also sort the comparison score based on their score by changing the **ascending** parameter, just as shown below; 61 | 62 | ```python 63 | >>> comparison_result = pysimilar.compare_documents('documents', ascending=True) 64 | >>> pprint(comparison_result) 65 | [['welcome.txt vs hello.txt', 0.0], 66 | ['hi.txt vs hello.txt', 0.0], 67 | ['welcome.txt vs hi.txt', 0.6053485081062917]] 68 | ``` 69 | 70 | ## multiple extensions 71 | 72 | You can also set pysimilar to include files with multiple extensions 73 | 74 | ```python 75 | >>> import pysimilar 76 | >>> from pprint import pprint 77 | >>> pysimilar.extensions = ['.txt', '.zeta'] 78 | >>> comparison_result = pysimilar.compare_documents('documents', ascending=True) 79 | >>> pprint(comparison_result) 80 | [['welcome.txt vs hello.txt', 0.0], 81 | ['hi.txt vs hello.txt', 0.0], 82 | ['anomalie.zeta vs hi.txt', 0.4968161174826459], 83 | ['welcome.txt vs hi.txt', 0.6292275146695526], 84 | ['welcome.txt vs anomalie.zeta', 0.7895651507603823]] 85 | 86 | ``` 87 | 88 | ## Contributions 89 | 90 | If you have anything valuable to add to the *lib*, whether its a documentation, typo error, source code, please don't hesitate to contribute just fork it and submit your pull request and I will try to be as friendly as I can to assist you making the contributions. 91 | 92 | 93 | ## All the Credits 94 | 95 | All the Credits to [kalebu](https://github.com/Kalebu) and other future contributors -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [pysimilar](https://pypi.org/project/pysimilar) 2 | 3 | [![Downloads](https://pepy.tech/badge/pysimilar)](https://pepy.tech/project/pysimilar) 4 | [![Downloads](https://pepy.tech/badge/pysimilar/month)](https://pepy.tech/project/pysimilar) 5 | [![Downloads](https://pepy.tech/badge/pysimilar/week)](https://pepy.tech/project/pysimilar) 6 | 7 | A python library for computing the similarity between two string(text) based on cosine similarity made by [kalebu](https://github.com/Kalebu) 8 | 9 | [![Become a patron](pictures/become_a_patron_button.png)](https://www.patreon.com/kalebujordan) 10 | 11 | How does it work ? 12 | ------------------ 13 | 14 | It uses Tfidf Vectorizer to transform the text into vectors and then obtained vectors are converted into arrays of numbers and then finally cosine similary computation is employed resulting to output indicating how similar they are. 15 | 16 | Installation 17 | ------------- 18 | You can either install it directly from *Github* or use *pip* to install it, here is how you to install it directly from github; 19 | 20 | ```bash 21 | $ git clone https://github.com/Kalebu/pysimilar 22 | $ cd pysimilar 23 | $ pysimilar -> python setup.py install 24 | 25 | ``` 26 | 27 | Installation with pip 28 | ---------------------- 29 | 30 | ```python 31 | $ pip install pysimilar 32 | ``` 33 | 34 | Example of usage 35 | ---------------- 36 | Pysimilar allows you to either specify the string you want to compare directly or specify path to files containing string you want to compare. 37 | 38 | Here an example on how to compare strings directly; 39 | 40 | ```python 41 | >>> from pysimilar import compare 42 | >>> compare('very light indeed', 'how fast is light') 43 | 0.17077611319011649 44 | ``` 45 | 46 | Here how to compare files with textual documents; 47 | 48 | ```python 49 | >>> compare('README.md', 'LICENSE', isfile=True) 50 | 0.25545580376557886 51 | ``` 52 | 53 | You can also compare documents with particular **extension** in a given directory, for instance let's say I want to compare all the documents with **.txt** in a **documents** directory here is what I will do; 54 | 55 | Directory for documents used by the example below look like this 56 | 57 | ```bash 58 | documents/ 59 | ├── anomalie.zeta 60 | ├── hello.txt 61 | ├── hi.txt 62 | └── welcome.txt 63 | ``` 64 | 65 | Here how to compare files of a particular extension 66 | 67 | ```python 68 | >>> import pysimilar 69 | >>> from pprint import pprint 70 | >>> pysimilar.extensions = '.txt' 71 | >>> comparison_result = pysimilar.compare_documents('documents') 72 | >>> [['welcome.txt vs hi.txt', 0.6053485081062917], 73 | ['welcome.txt vs hello.txt', 0.0], 74 | ['hi.txt vs hello.txt', 0.0]] 75 | ``` 76 | 77 | You can also sort the comparison score based on their score by changing the **ascending** parameter, just as shown below; 78 | 79 | ```python 80 | >>> comparison_result = pysimilar.compare_documents('documents', ascending=True) 81 | >>> pprint(comparison_result) 82 | [['welcome.txt vs hello.txt', 0.0], 83 | ['hi.txt vs hello.txt', 0.0], 84 | ['welcome.txt vs hi.txt', 0.6053485081062917]] 85 | ``` 86 | 87 | You can also set pysimilar to include files with multiple extensions 88 | 89 | ```python 90 | >>> import pysimilar 91 | >>> from pprint import pprint 92 | >>> pysimilar.extensions = ['.txt', '.zeta'] 93 | >>> comparison_result = pysimilar.compare_documents('documents', ascending=True) 94 | >>> pprint(comparison_result) 95 | [['welcome.txt vs hello.txt', 0.0], 96 | ['hi.txt vs hello.txt', 0.0], 97 | ['anomalie.zeta vs hi.txt', 0.4968161174826459], 98 | ['welcome.txt vs hi.txt', 0.6292275146695526], 99 | ['welcome.txt vs anomalie.zeta', 0.7895651507603823]] 100 | 101 | ``` 102 | 103 | Contributions 104 | ------------- 105 | If you have anything valuable to add to the *lib*, whether its a documentation, typo error, source code, please don't hesitate to contribute just fork it and submit your pull request and I will try to be as friendly as I can to assist you making the contributions. 106 | 107 | 108 | Give it a star 109 | -------------- 110 | Did you find this repo useful to you ? then give it a star so as more people can be aware of it and use it, Share that love * 111 | 112 | All the Credits 113 | --------------- 114 | 115 | All the Credits to [kalebu](https://github.com/Kalebu) and other future contributors -------------------------------------------------------------------------------- /pysimilar/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path 4 | from typing import Union, List, Dict 5 | from sklearn.metrics.pairwise import cosine_similarity 6 | from sklearn.feature_extraction.text import TfidfVectorizer 7 | 8 | 9 | class Pysimilar(object): 10 | """Very light library for computing similarity between two string or text documents 11 | 12 | Args: 13 | object ([type]): [description] 14 | """ 15 | 16 | VALID_EXTENSION: List[str] = ['.doc', '.txt', '.docx'] 17 | 18 | @property 19 | def extensions(self) -> List[str]: 20 | """extensions Returns allowed extensions 21 | 22 | Returns: 23 | [List]: [Allowed file extensions] 24 | """ 25 | return self.VALID_EXTENSION 26 | 27 | @extensions.setter 28 | def extensions(self, new_extensions: Union[str, List[str]]): 29 | """extensions [Set new allowed extensions] 30 | 31 | Args: 32 | new_extensions (Union[str, list]): [description] 33 | 34 | Raises: 35 | TypeError: [description] 36 | 37 | Returns: 38 | [type]: [description] 39 | """ 40 | 41 | if not isinstance(new_extensions, (str, list)): 42 | raise TypeError( 43 | f'New extensions must be of either type or not {type(new_extensions)}') 44 | if isinstance(new_extensions, str): 45 | new_extensions: List[str] = [new_extensions] 46 | self.VALID_EXTENSION = new_extensions 47 | 48 | def get_files(self, path_to_files: Union[Path, str]) -> List[str]: 49 | """get_files [Returns available files paths] 50 | 51 | Returns: 52 | List[Path]: [description] 53 | """ 54 | 55 | all_files_and_dirs = os.listdir(path_to_files) 56 | available_files: List[str] = [] 57 | for file_or_dir in all_files_and_dirs: 58 | full_path = os.path.join(path_to_files, file_or_dir) 59 | if os.path.isfile((full_path)) and any([full_path.endswith(ext) for ext in self.VALID_EXTENSION]): 60 | available_files.append(full_path) 61 | return available_files 62 | 63 | @staticmethod 64 | def load_file(path_to_file: Union[Path, str]): 65 | with open(path_to_file, 'r', encoding='latin-1') as document: 66 | content = document.read() 67 | return content 68 | 69 | def load_files(self, path_to_folder: Union[Path, str]): 70 | path_to_files: List[str] = self.get_files(path_to_folder) 71 | load_documents: List[str] = [self.load_file(path_to_document) 72 | for path_to_document in path_to_files] 73 | file_names = [path_to_file.split('/')[-1] 74 | for path_to_file in path_to_files] 75 | document_dictionary = dict(zip(file_names, load_documents)) 76 | return document_dictionary 77 | 78 | def compare(self, string_i: Union[str, Path], string_j: Union[str, Path], isfile=False) -> float: 79 | """Returns the similarity score between string i and string j 80 | 81 | 82 | Args: 83 | string_i (Union[str, Path]): [description] 84 | string_j (Union[str, Path]): [description] 85 | 86 | Returns: 87 | float: [Similarity score between string i and string j ] 88 | """ 89 | 90 | if not isinstance(string_i, (str, Path)) or not isinstance(string_j, (str, Path)): 91 | raise TypeError( 92 | 'Both string i and string j must be of type either string or Path') 93 | 94 | if isfile: 95 | string_i, string_j = self.load_file( 96 | string_i), self.load_file(string_j) 97 | 98 | corpus = [string_i, string_j] 99 | vector_i, vector_j = self.string_to_vector(corpus) 100 | return self.compute_similarity(vector_i, vector_j) 101 | 102 | def compare_documents(self, path_to_documents: Union[str, Path], sort=True, ascending=False) -> list: 103 | """compare_documents [compare group of documents in a particular folder] 104 | 105 | Args: 106 | path_to_documents (Union[str, Path]): [description] 107 | sort (bool, optional): [description]. Defaults to True. 108 | ascending (bool, optional): [description]. Defaults to True. 109 | 110 | Returns: 111 | list: [description] 112 | """ 113 | if not os.path.exists(path_to_documents): 114 | raise FileNotFoundError( 115 | f'Path <{path_to_documents}> Does not exist') 116 | 117 | loaded_documents: Dict = self.load_files(path_to_documents) 118 | vectorized_documents = self.vectorize_dict(loaded_documents) 119 | compared_documents: List[set] = [] 120 | comparison_results: List[list] = [] 121 | for current_document_name, current_content in vectorized_documents.items(): 122 | for document_name, content in vectorized_documents.items(): 123 | current_comparison = f'{current_document_name} vs {document_name}' 124 | if (current_document_name == document_name) or (set(current_comparison) in compared_documents): 125 | continue 126 | result = self.compute_similarity(current_content, content) 127 | displayable_result = [current_comparison, result] 128 | comparison_results.append(displayable_result) 129 | # print(displayable_result) 130 | compared_documents.append(set(current_comparison)) 131 | 132 | if not sort: 133 | return comparison_results 134 | 135 | sorted_results = sorted( 136 | comparison_results, key=lambda x: x[1], reverse=not ascending) 137 | return sorted_results 138 | 139 | def compute_similarity(self, vector_a: list, vector_b: list) -> float: 140 | """Compute the similarity between vector a and vector b 141 | 142 | Args: 143 | vector_a (list): [description] 144 | vector_b (list): [description] 145 | 146 | Returns: 147 | float: [ int: [similarity score between vector a, string b]] 148 | """ 149 | 150 | return cosine_similarity([vector_a, vector_b])[0][1] 151 | 152 | def vectorize_dict(self, documents: Dict) -> Dict: 153 | """vectorize_dict [summary] 154 | 155 | Args: 156 | documents (Dict): [description] 157 | 158 | Returns: 159 | Dict: [description] 160 | """ 161 | file_names = list(documents.keys()) 162 | corpus = list(documents.values()) 163 | vectorized_corpus = self.string_to_vector(corpus) 164 | return dict(zip(file_names, vectorized_corpus)) 165 | 166 | def string_to_vector(self, corpus: List[str]) -> list: 167 | """Convert a list string to vectors using TfidfVectorizer 168 | 169 | Args: 170 | List (str): [description] 171 | 172 | Returns: 173 | list: [arrays of vectorized text] 174 | """ 175 | return TfidfVectorizer().fit_transform(corpus).toarray() 176 | 177 | 178 | sys.modules[__name__] = Pysimilar() 179 | --------------------------------------------------------------------------------