├── .gitattributes ├── finbert_embedding ├── __init__.py ├── downloader.py └── embedding.py ├── setup.py ├── LICENSE └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | finbertTRC2/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text 2 | finbert_embedding/model/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text 3 | -------------------------------------------------------------------------------- /finbert_embedding/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | finbert_embedding is pypi package for getting bert based embedding for financial 3 | statements 4 | """ 5 | 6 | from finbert_embedding.embedding import FinbertEmbedding 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="finbert-embedding", 8 | packages=['finbert_embedding'], 9 | version="0.1.4", 10 | author="Abhijeet Kumar", 11 | author_email="abhijeetchar@gmail.com", 12 | description="Embeddings from Financial BERT", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | url="https://github.com/abhijeet3922/finbert_embedding", 16 | download_url="https://github.com/abhijeet3922/finbert_embedding/archive/v0.1.4.tar.gz", 17 | install_requires=[ 18 | 'torch>=1.1.0', 19 | 'pytorch-pretrained-bert==0.6.2', 20 | 'tensorflow', 21 | ], 22 | 23 | classifiers=[ 24 | "Programming Language :: Python :: 3", 25 | "License :: OSI Approved :: MIT License", 26 | "Operating System :: OS Independent", 27 | ], 28 | python_requires='>=3.6', 29 | ) 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Abhijeet Kumar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /finbert_embedding/downloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tarfile 3 | import requests 4 | import tensorflow as tf 5 | from pathlib import Path 6 | 7 | def download_file_from_google_drive(id, destination): 8 | 9 | URL = "https://docs.google.com/uc?export=download" 10 | session = requests.Session() 11 | 12 | response = session.get(URL, params = { 'id' : id }, stream = True) 13 | token = get_confirm_token(response) 14 | 15 | if token: 16 | params = { 'id' : id, 'confirm' : token } 17 | response = session.get(URL, params = params, stream = True) 18 | 19 | save_response_content(response, destination) 20 | 21 | def get_confirm_token(response): 22 | for key, value in response.cookies.items(): 23 | if key.startswith('download_warning'): 24 | return value 25 | 26 | return None 27 | 28 | def save_response_content(response, destination): 29 | CHUNK_SIZE = 32768 30 | 31 | with open(destination, "wb") as f: 32 | for chunk in response.iter_content(CHUNK_SIZE): 33 | if chunk: # filter out keep-alive new chunks 34 | f.write(chunk) 35 | 36 | def get_Finbert(location): 37 | 38 | model_path = Path.cwd()/'fin_model' 39 | 40 | if location == 'dropbox': 41 | if not os.path.isdir(model_path): 42 | os.makedirs(model_path) 43 | dataset = tf.keras.utils.get_file(fname=model_path/"fin_model.tar.gz", 44 | origin="https://www.dropbox.com/s/6oeprcqae7tc459/fin_model.tar.gz?dl=1") 45 | tar = tarfile.open(model_path/"fin_model.tar.gz") 46 | tar.extractall() 47 | 48 | else: 49 | if not os.path.exists(model_path/"fin_model.tar.gz"): 50 | dataset = tf.keras.utils.get_file(fname=model_path/"fin_model.tar.gz", 51 | origin="https://www.dropbox.com/s/6oeprcqae7tc459/fin_model.tar.gz?dl=1") 52 | if not os.path.exists(model_path/"pytorch_model.bin"): 53 | print("Extracting finbert model tar.gz") 54 | tar = tarfile.open(model_path/"fin_model.tar.gz") 55 | tar.extractall() 56 | 57 | if location == 'google drive': 58 | if not os.path.isdir(model_path): 59 | os.makedirs(model_path) 60 | print("Downloading the finbert model, will take a minute...") 61 | fileid = "1feMhKmiW2FNQ9107GLeMYbrijgp3Fv8l" 62 | download_file_from_google_drive(fileid, model_path/"fin_model.tar.gz") 63 | tar = tarfile.open(model_path/"fin_model.tar.gz") 64 | tar.extractall() 65 | else: 66 | if not os.path.exists(model_path/"fin_model.tar.gz"): 67 | download_file_from_google_drive(fileid, model_path/"fin_model.tar.gz") 68 | if not os.path.exists(model_path/"pytorch_model.bin"): 69 | print("Extracting finbert model tar.gz") 70 | tar = tarfile.open(model_path/"fin_model.tar.gz") 71 | tar.extractall() 72 | 73 | return model_path 74 | 75 | 76 | if __name__ == "__main__": 77 | print("package from downloading finbert model") 78 | -------------------------------------------------------------------------------- /finbert_embedding/embedding.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import logging 4 | import tensorflow as tf 5 | from finbert_embedding import downloader 6 | from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM 7 | 8 | __author__ = 'Abhijeet Kumar' 9 | 10 | #Create and configure logger 11 | logging.basicConfig(filename='app.log', filemode='w',format='%(asctime)s %(message)s', level=logging.INFO) 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | class FinbertEmbedding(object): 16 | """ 17 | Encoding from FinBERT model (BERT LM finetuned on 47K Financial news articles). 18 | 19 | Parameters 20 | ---------- 21 | 22 | model : str, default finbertTRC2. 23 | pre-trained BERT model 24 | """ 25 | 26 | def __init__(self, model_path=None): 27 | 28 | if model_path is not None: 29 | self.model_path = model_path 30 | else: 31 | self.model_path = downloader.get_Finbert('dropbox') 32 | 33 | self.tokens = "" 34 | self.sentence_tokens = "" 35 | self.tokenizer = BertTokenizer.from_pretrained(self.model_path) 36 | # Load pre-trained model (weights) 37 | self.model = BertModel.from_pretrained(self.model_path) 38 | logger.info("Initialization Done !!") 39 | 40 | def process_text(self, text): 41 | 42 | tokenized_text = ['[CLS]'] + self.tokenizer.tokenize(text)[:510] + ['[SEP]'] 43 | # Tokenize our sentence with the BERT tokenizer 44 | return tokenized_text 45 | 46 | 47 | def handle_oov(self, tokenized_text, word_embeddings): 48 | embeddings = [] 49 | tokens = [] 50 | oov_len = 1 51 | for token,word_embedding in zip(tokenized_text, word_embeddings): 52 | if token.startswith('##'): 53 | token = token[2:] 54 | tokens[-1] += token 55 | oov_len += 1 56 | embeddings[-1] += word_embedding 57 | else: 58 | if oov_len > 1: 59 | embeddings[-1] /= oov_len 60 | tokens.append(token) 61 | embeddings.append(word_embedding) 62 | return tokens,embeddings 63 | 64 | 65 | def eval_fwdprop_finbert(self, tokenized_text): 66 | 67 | # Mark each of the tokens as belonging to sentence "1". 68 | segments_ids = [1] * len(tokenized_text) 69 | # Map the token strings to their vocabulary indeces. 70 | indexed_tokens = self.tokenizer.convert_tokens_to_ids(tokenized_text) 71 | 72 | # Convert inputs to PyTorch tensors 73 | tokens_tensor = torch.tensor([indexed_tokens]) 74 | segments_tensors = torch.tensor([segments_ids]) 75 | 76 | # Put the model in "evaluation" mode, meaning feed-forward operation. 77 | self.model.eval() 78 | # Predict hidden states features for each layer 79 | with torch.no_grad(): 80 | encoded_layers, _ = self.model(tokens_tensor, segments_tensors) 81 | 82 | return encoded_layers 83 | 84 | 85 | def word_vector(self, text, handle_oov=True, filter_extra_tokens=True): 86 | 87 | tokenized_text = self.process_text(text) 88 | 89 | encoded_layers = self.eval_fwdprop_finbert(tokenized_text) 90 | 91 | # Concatenate the tensors for all layers. We use `stack` here to 92 | # create a new dimension in the tensor. 93 | token_embeddings = torch.stack(encoded_layers, dim=0) 94 | token_embeddings = torch.squeeze(token_embeddings, dim=1) 95 | # Swap dimensions 0 and 1. 96 | token_embeddings = token_embeddings.permute(1,0,2) 97 | 98 | # Stores the token vectors, with shape [22 x 768] 99 | word_embeddings = [] 100 | logger.info("Summing last 4 layers for each token") 101 | # For each token in the sentence... 102 | for token in token_embeddings: 103 | 104 | # `token` is a [12 x 768] tensor 105 | # Sum the vectors from the last four layers. 106 | sum_vec = torch.sum(token[-4:], dim=0) 107 | 108 | # Use `sum_vec` to represent `token`. 109 | word_embeddings.append(sum_vec) 110 | 111 | self.tokens = tokenized_text 112 | if filter_extra_tokens: 113 | # filter_spec_tokens: filter [CLS], [SEP] tokens. 114 | word_embeddings = word_embeddings[1:-1] 115 | self.tokens = tokenized_text[1:-1] 116 | 117 | if handle_oov: 118 | self.tokens, word_embeddings = self.handle_oov(self.tokens,word_embeddings) 119 | logger.info(self.tokens) 120 | logger.info("Shape of Word Embeddings = %s",str(len(word_embeddings))) 121 | return word_embeddings 122 | 123 | 124 | 125 | def sentence_vector(self,text): 126 | 127 | logger.info("Taking last layer embedding of each word.") 128 | logger.info("Mean of all words for sentence embedding.") 129 | tokenized_text = self.process_text(text) 130 | self.sentence_tokens = tokenized_text 131 | encoded_layers = self.eval_fwdprop_finbert(tokenized_text) 132 | 133 | # `encoded_layers` has shape [12 x 1 x 22 x 768] 134 | # `token_vecs` is a tensor with shape [22 x 768] 135 | token_vecs = encoded_layers[11][0] 136 | 137 | # Calculate the average of all 22 token vectors. 138 | sentence_embedding = torch.mean(token_vecs, dim=0) 139 | logger.info("Shape of Sentence Embeddings = %s",str(len(sentence_embedding))) 140 | return sentence_embedding 141 | 142 | 143 | if __name__ == "__main__": 144 | 145 | text = "Another PSU bank, Punjab National Bank which also reported numbers " \ 146 | "managed to see a slight improvement in asset quality." 147 | 148 | finbert = FinbertEmbedding() 149 | word_embeddings = finbert.word_vector(text) 150 | sentence_embedding = finbert.sentence_vector(text) 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # finbert_embedding 2 | Token and sentence level embeddings from FinBERT model (Financial Domain). 3 | 4 | [BERT](https://arxiv.org/abs/1810.04805), published by Google, is conceptually simple and empirically powerful as it obtained state-of-the-art results on eleven natural language processing tasks. 5 | 6 | The objective of this project is to obtain the word or sentence embeddings from [FinBERT](https://github.com/ProsusAI/finBERT), pre-trained model by Dogu Tan Araci (University of Amsterdam). FinBERT, which is a BERT language model further trained on Financial news articles for adapting financial domain. It achieved the state-of-the-art on FiQA sentiment scoring and Financial PhraseBank dataset. Paper [here](https://arxiv.org/abs/1908.10063). 7 | 8 | Instead of building and do fine-tuning for an end-to-end NLP model, You can directly utilize word embeddings from Financial BERT to build NLP models for various downstream tasks eg. Financial text classification, Text clustering, Extractive summarization or Entity extraction etc. 9 | 10 | 11 | 12 | ## Features 13 | * Creates an abstraction to remove dealing with inferencing pre-trained FinBERT model. 14 | * Require only two lines of code to get sentence/token-level encoding for a text sentence. 15 | * The package takes care of OOVs (out of vocabulary) inherently. 16 | * Downloads and installs FinBERT pre-trained model (first initialization, usage in next section). 17 | 18 | ## Install 19 | (Recommended to create a conda env to have isolation and avoid dependency clashes) 20 | 21 | ``` 22 | pip install finbert-embedding==0.1.4 23 | ``` 24 | 25 | Note: If you get error in installing this package (common error with Tf):
26 | 27 | Installing collected packages: wrapt, tensorflow
28 | Found existing installation: wrapt 1.10.11
29 | ERROR: Cannot uninstall 'wrapt'. It is a distutils installed project.... 30 | 31 | then, just do this: 32 | ``` 33 | pip install wrapt --upgrade --ignore-installed 34 | pip install finbert-embedding==0.1.4 35 | ``` 36 | 37 | ## Usage 1 38 | 39 | word embeddings generated are list of 768 dimensional embeddings for each word.
40 | sentence embedding generated is 768 dimensional embedding which is average of each token. 41 | 42 | ```python 43 | from finbert_embedding.embedding import FinbertEmbedding 44 | 45 | text = "Another PSU bank, Punjab National Bank which also reported numbers managed to see a slight improvement in asset quality." 46 | 47 | # Class Initialization (You can set default 'model_path=None' as your finetuned BERT model path while Initialization) 48 | finbert = FinbertEmbedding() 49 | 50 | word_embeddings = finbert.word_vector(text) 51 | sentence_embedding = finbert.sentence_vector(text) 52 | 53 | print("Text Tokens: ", finbert.tokens) 54 | # Text Tokens: ['another', 'psu', 'bank', ',', 'punjab', 'national', 'bank', 'which', 'also', 'reported', 'numbers', 'managed', 'to', 'see', 'a', 'slight', 'improvement', 'in', 'asset', 'quality', '.'] 55 | 56 | print ('Shape of Word Embeddings: %d x %d' % (len(word_embeddings), len(word_embeddings[0]))) 57 | # Shape of Word Embeddings: 21 x 768 58 | 59 | print("Shape of Sentence Embedding = ",len(sentence_embedding)) 60 | # Shape of Sentence Embedding = 768 61 | ``` 62 | 63 | ## Usage 2 64 | 65 | A decent representation for a downstream task doesn't mean that it will be meaningful in terms of cosine distance. Since cosine distance is a linear space where all dimensions are weighted equally. if you want to use cosine distance anyway, then please focus on the rank not the absolute value. 66 | 67 | Namely, do not use:
68 | if cosine(A, B) > 0.9, then A and B are similar 69 | 70 | Please consider the following instead:
71 | if cosine(A, B) > cosine(A, C), then A is more similar to B than C. 72 | 73 | ```python 74 | from finbert_embedding.embedding import FinbertEmbedding 75 | 76 | text = "After stealing money from the bank vault, the bank robber was seen fishing on the Mississippi river bank." 77 | finbert = FinbertEmbedding() 78 | word_embeddings = finbert.word_vector(text) 79 | 80 | from scipy.spatial.distance import cosine 81 | diff_bank = 1 - cosine(word_embeddings[9], word_embeddings[18]) 82 | same_bank = 1 - cosine(word_embeddings[9], word_embeddings[5]) 83 | 84 | print('Vector similarity for similar bank meanings (bank vault & bank robber): %.2f' % same_bank) 85 | print('Vector similarity for different bank meanings (bank robber & river bank): %.2f' % diff_bank) 86 | 87 | # Vector similarity for similar bank meanings (bank vault & bank robber): 0.92 88 | # Vector similarity for different bank meanings (bank robber & river bank): 0.64 89 | ``` 90 | 91 | ### Warning 92 | 93 | According to BERT author Jacob Devlin: 94 | ```I'm not sure what these vectors are, since BERT does not generate meaningful sentence vectors. It seems that this is doing average pooling over the word tokens to get a sentence vector, but we never suggested that this will generate meaningful sentence representations. And even if they are decent representations when fed into a DNN trained for a downstream task, it doesn't mean that they will be meaningful in terms of cosine distance. (Since cosine distance is a linear space where all dimensions are weighted equally).``` 95 | 96 | However, with the [CLS] token, it does become meaningful if the model has been fine-tuned, where the last hidden layer of this token is used as the “sentence vector” for downstream sequence classification task. This package encode sentence in similar manner. 97 | 98 | ### To Do (Next Version) 99 | 100 | * Extend it to give word embeddings for a paragram/Document (Currently, it takes one sentence as input). Chunkize your paragraph or text document into sentences using Spacy or NLTK before using finbert_embedding. 101 | * Adding batch processing feature. 102 | * More ways of handing OOVs (Currently, uses average of all tokens of a OOV word) 103 | * Ingesting and extending it to more pre-trained financial models. 104 | 105 | ### Future Goal 106 | 107 | * Create generic downstream framework using various FinBERT language model for any financial labelled text classifcation task like sentiment classification, Financial news classification, Financial Document classification. 108 | --------------------------------------------------------------------------------