├── config └── shopping-list │ └── config_spacy.json ├── LICENSE ├── .gitignore ├── shopping_bot.py ├── command.py ├── intent.py ├── README.md └── data └── shopping-list └── rasa └── shopping-list-small.json /config/shopping-list/config_spacy.json: -------------------------------------------------------------------------------- 1 | { 2 | "pipeline": "spacy_sklearn", 3 | "path" : "./projects", 4 | "data" : "./data/shopping-list/rasa/shopping-list-small.json" 5 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Iaroslav Omelianenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /shopping_bot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sat Oct 7 22:52:04 2017 5 | 6 | The shopping bot main class which provides routines to maintain NLU model, 7 | parse user's intents and execute corresponding commands in response. 8 | 9 | @author: yaric 10 | """ 11 | 12 | from rasa_nlu.converters import load_data 13 | from rasa_nlu.config import RasaNLUConfig 14 | from rasa_nlu.model import Trainer 15 | 16 | from intent import HelloIntent, AddItemsIntent, ClearListIntent, ShowItemsIntent, ShowStatsIntent 17 | 18 | class ShoppingBot(object): 19 | def __init__(self, training_data_file = "./data/shopping-list/rasa/shopping-list-small.json", 20 | config_file = "./config/shopping-list/config_spacy.json"): 21 | training_data = load_data(training_data_file) 22 | 23 | trainer = Trainer(RasaNLUConfig(config_file)) 24 | self.interpreter = trainer.train(training_data) 25 | self.shopping_list = {} 26 | 27 | # Create supported intents 28 | context = {'confidence_threshold':0.8} 29 | self.intents = { 30 | "greet" : HelloIntent(self, "greet", context), 31 | "add_item" : AddItemsIntent(self, "add_item", context), 32 | "clear_list": ClearListIntent(self, "clear_list", context), 33 | "show_items": ShowItemsIntent(self, "show_items", context), 34 | "_num_items": ShowStatsIntent(self, "_num_items", context) 35 | } 36 | 37 | 38 | def handle(self, message): 39 | """ 40 | Handles incoming message using trained NLU model and prints response to 41 | the system out 42 | Arguments: 43 | message the message from user to be handled with known intents 44 | (greet, add_item, clear_list, show_items, _num_items) 45 | """ 46 | if message == '_num_items': 47 | self.intents['_num_items'].execute(None) 48 | else: 49 | nlu_data = self.interpreter.parse(message) 50 | intent = nlu_data['intent']['name'] 51 | if self.intents[intent] is not None: 52 | self.intents[intent].execute(nlu_data) 53 | 54 | 55 | -------------------------------------------------------------------------------- /command.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sun Oct 8 15:24:39 2017 5 | 6 | The stateless command allows to encapsulate processing logic for specic 7 | intent's command. Allows to easy build response processing pipelines with 8 | multiple stages of intent data processing. The stateless commands may be shared 9 | among various intents. 10 | 11 | @author: yaric 12 | """ 13 | 14 | import random 15 | 16 | class Command(object): 17 | 18 | def do(self, bot, entity): 19 | """ 20 | Execute command's action for specified intent. 21 | Arguments: 22 | bot the chatbot 23 | entity the parsed NLU entity 24 | """ 25 | pass 26 | 27 | 28 | class GreetCommand(Command): 29 | """ 30 | The command to greet user 31 | """ 32 | 33 | def __init__(self): 34 | """ 35 | Default constructor which will create list of gretings to be picked 36 | randomly to make our bot more human-like 37 | """ 38 | self.greetings = ["Hey!", "Hello!", "Hi there!", "Hallo!", "How are you!"] 39 | 40 | def do(self, bot, entity): 41 | print(random.choice(self.greetings)) 42 | 43 | class AddItemCommand(Command): 44 | """ 45 | The command to add item to the list 46 | """ 47 | def do(self, bot, entity): 48 | count = 0 49 | if entity in bot.shopping_list: 50 | count = bot.shopping_list[entity] 51 | 52 | bot.shopping_list[entity] = count + 1 53 | 54 | 55 | class ShowItemsCommand(Command): 56 | """ 57 | The command to display shopping list 58 | """ 59 | 60 | def do(self, bot, entity): 61 | if len(bot.shopping_list) == 0: 62 | print("Your shopping list is empty!") 63 | return 64 | 65 | print("Shopping list items:") 66 | for k, v in bot.shopping_list.items(): 67 | print("%s - quantity: %d" % (k, v)) 68 | 69 | class ClearListCommand(Command): 70 | """ 71 | The command to clear shopping list 72 | """ 73 | 74 | def do(self, bot, entity): 75 | bot.shopping_list.clear() 76 | print("Items removed from your list!") 77 | 78 | class ShowStatsCommand(Command): 79 | """ 80 | The command to show shopping list statistics 81 | """ 82 | 83 | def do(self, bot, entity): 84 | unique = len(bot.shopping_list) 85 | if unique == 0: 86 | print("shopping list is empty") 87 | 88 | total = 0 89 | for v in bot.shopping_list.values(): 90 | total += v 91 | 92 | print("# of unique items: %d, total # of items: %d" % (unique, total)) 93 | -------------------------------------------------------------------------------- /intent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sun Oct 8 15:15:26 2017 5 | 6 | The user's intents definitions. It serves as abstraction to separate NLU processing 7 | logic from response processing routines. Allows to easy register in the system as many 8 | intents as it needed. 9 | 10 | @author: yaric 11 | """ 12 | 13 | from command import GreetCommand, AddItemCommand, ShowItemsCommand, ClearListCommand, ShowStatsCommand 14 | 15 | class Intent(object): 16 | 17 | def __init__(self, bot, intent_name, context): 18 | """ 19 | Creates new intent for specified chatbot with given name 20 | Arguments: 21 | bot the chatbot 22 | name the intent name 23 | context the execution context holding configuration parameters 24 | """ 25 | self.chatbot = bot 26 | self.name = intent_name 27 | self.context = context 28 | self.commands = [] 29 | self.initCommands() 30 | 31 | def execute(self, nlu_data): 32 | """ 33 | Executes given intent by applying appropriate command to the given 34 | parsed NLU data response 35 | """ 36 | for c in self.commands: 37 | c.do(self.chatbot, None) 38 | 39 | def initCommands(self): 40 | """ 41 | The method to init specific to particular intent. 42 | """ 43 | pass 44 | 45 | class AddItemsIntent(Intent): 46 | 47 | def initCommands(self): 48 | self.commands.append(AddItemCommand()) 49 | self.commands.append(ShowItemsCommand()) 50 | 51 | def execute(self, data): 52 | confidence = data['intent']['confidence'] 53 | confidence_threshold = self.context['confidence_threshold'] 54 | if confidence < confidence_threshold: 55 | print('I\'m sorry! Could you please paraphrase!') 56 | return 57 | 58 | # add all intent entities 59 | for entity in data['entities']: 60 | self.commands[0].do(self.chatbot, entity['value']) 61 | 62 | # show items list 63 | self.commands[1].do(self.chatbot, None) 64 | 65 | class HelloIntent(Intent): 66 | def initCommands(self): 67 | self.commands.append(GreetCommand()) 68 | 69 | class ShowItemsIntent(Intent): 70 | def initCommands(self): 71 | self.commands.append(ShowItemsCommand()) 72 | 73 | class ClearListIntent(Intent): 74 | def initCommands(self): 75 | self.commands.append(ClearListCommand()) 76 | self.commands.append(ShowItemsCommand()) 77 | 78 | class ShowStatsIntent(Intent): 79 | def initCommands(self): 80 | self.commands.append(ShowStatsCommand()) 81 | 82 | 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is simple chatbot application with Natural Language Understanding to maintain very simple shopping list. It's built using [RASA NLU][rasa_nlu_home] Python library with [Spacy NLP][spacy_nlp_home] backend. 2 | The bot which we will create assumes that anything our bot's users say can be categorized into one of the following intents: 3 | 1. Greetings (**greet**) 4 | 2. Adding item to the shopping list (**add_item**) 5 | 3. Removing all items from shopping list (**clear_list**) 6 | 4. Displaying shopping list (**show_items**) 7 | 5. The debug command to return number of items in the list (**_num_items**) 8 | 9 | ## System Requirements ## 10 | As it was already mentined we are going to use RASA NLU and Spacy NLP Python libraries which should be installed. 11 | 12 | To install current stable version of [RASA NLU][rasa_nlu_home] just run 13 | ```python 14 | pip install rasa_nlu 15 | ``` 16 | 17 | To install current version of [Spacy NLP][spacy_nlp_home] execute 18 | ```python 19 | pip install -U spacy 20 | ``` 21 | 22 | We will use English language Spacy model which can be installed as following: 23 | ```python 24 | python -m spacy download en 25 | ``` 26 | 27 | Also please make sure that sklearn_crfsuite installed on the system or install it: 28 | ```python 29 | pip install sklearn_crfsuite 30 | ``` 31 | 32 | ## Preparing the Training Data ## 33 | To build NLU model able to parse user's intents first we need to create appropriate training data which will allow our chatbot to interpret user's inputs and created structured data (intent/entities). The best way to get training texts is from real users, and the best way to get the structured data is to pretend to be the bot yourself. For the purpose of this experiment we will create small training data corpus in format described at: https://rasa-nlu.readthedocs.io/en/stable/dataformat.html 34 | 35 | The best way to create training data in rasa's format and to validate it is to use great [online tool](https://rasahq.github.io/rasa-nlu-trainer/) created by [@azazdeaz](https://github.com/azazdeaz). 36 | 37 | The resulting training data file in JSON format can be found in the *data* folder. 38 | 39 | ## Training a New NLU Model ## 40 | There are two modes to use Rasa NLU models - as a HTTP server or directly from Python. In this experiment taking into account small size of our training data we will train and use NLU model directly from Python. 41 | 42 | To do this first we need to create corresponding Rasa NLU configuration file. We will use Spacy NLP as backend thus configuration file will be: 43 | ```json 44 | { 45 | "pipeline": "spacy_sklearn", 46 | "path" : "./projects", 47 | "data" : "./data/shopping-list/rasa/shopping-list-small.json" 48 | } 49 | ``` 50 | 51 | For more details as how to run Rasa NLU models directly from Python refer to: https://rasa-nlu.readthedocs.io/en/stable/python.html 52 | 53 | ## The Shopping List Chatbot ## 54 | The training data includes extremelly limited knowledge of grocery items to be purchased - eggs, milk, and butter. But it can be easy extended by modifying training data file mentioned above. 55 | 56 | The chatbot can be used as following: 57 | ```python 58 | In [1]: from shopping_bot import ShoppingBot 59 | 60 | In [2]: bot = ShoppingBot() 61 | 62 | In [3]: bot.handle("Hello") 63 | How are you! 64 | 65 | In [4]: bot.handle("add milk") 66 | Shopping list items: 67 | milk - quantity: 1 68 | 69 | In [5]: bot.handle("i need eggs") 70 | Shopping list items: 71 | milk - quantity: 1 72 | eggs - quantity: 1 73 | 74 | In [6]: bot.handle("i need eggs and milk") 75 | Shopping list items: 76 | milk - quantity: 2 77 | eggs - quantity: 2 78 | 79 | In [7]: bot.handle("my list") 80 | Shopping list items: 81 | milk - quantity: 2 82 | eggs - quantity: 2 83 | 84 | In [8]: bot.handle("_num_items") 85 | # of unique items: 2, total # of items: 4 86 | 87 | In [9]: bot.handle("clear list") 88 | Items removed from your list! 89 | Your shopping list is empty! 90 | 91 | In [10]: bot.handle("_num_items") 92 | shopping list is empty 93 | # of unique items: 0, total # of items: 0 94 | ``` 95 | 96 | The supported grocery items: 97 | * eggs 98 | * milk 99 | * butter 100 | 101 | [rasa_nlu_home]:https://rasa-nlu.readthedocs.io/en/stable/ 102 | [spacy_nlp_home]:https://spacy.io 103 | -------------------------------------------------------------------------------- /data/shopping-list/rasa/shopping-list-small.json: -------------------------------------------------------------------------------- 1 | { 2 | "rasa_nlu_data": { 3 | "common_examples": [ 4 | { 5 | "text": "hey", 6 | "intent": "greet", 7 | "entities": [] 8 | }, 9 | { 10 | "text": "howdy", 11 | "intent": "greet", 12 | "entities": [] 13 | }, 14 | { 15 | "text": "hey there", 16 | "intent": "greet", 17 | "entities": [] 18 | }, 19 | { 20 | "text": "hello", 21 | "intent": "greet", 22 | "entities": [] 23 | }, 24 | { 25 | "text": "hi", 26 | "intent": "greet", 27 | "entities": [] 28 | }, 29 | { 30 | "text": "hallo", 31 | "intent": "greet", 32 | "entities": [] 33 | }, 34 | { 35 | "text": "add eggs", 36 | "intent": "add_item", 37 | "entities": [ 38 | { 39 | "start": 4, 40 | "end": 8, 41 | "value": "eggs", 42 | "entity": "shopping_item" 43 | } 44 | ] 45 | }, 46 | { 47 | "text": "add eggs to the list", 48 | "intent": "add_item", 49 | "entities": [ 50 | { 51 | "start": 4, 52 | "end": 8, 53 | "value": "eggs", 54 | "entity": "shopping_item" 55 | } 56 | ] 57 | }, 58 | { 59 | "text": "i also need eggs", 60 | "intent": "add_item", 61 | "entities": [ 62 | { 63 | "start": 12, 64 | "end": 16, 65 | "value": "eggs", 66 | "entity": "shopping_item" 67 | } 68 | ] 69 | }, 70 | { 71 | "text": "need eggs to buy", 72 | "intent": "add_item", 73 | "entities": [ 74 | { 75 | "start": 5, 76 | "end": 9, 77 | "value": "eggs", 78 | "entity": "shopping_item" 79 | } 80 | ] 81 | }, 82 | { 83 | "text": "add milk", 84 | "intent": "add_item", 85 | "entities": [ 86 | { 87 | "start": 4, 88 | "end": 8, 89 | "value": "milk", 90 | "entity": "shopping_item" 91 | } 92 | ] 93 | }, 94 | { 95 | "text": "need some milk", 96 | "intent": "add_item", 97 | "entities": [ 98 | { 99 | "start": 10, 100 | "end": 14, 101 | "value": "milk", 102 | "entity": "shopping_item" 103 | } 104 | ] 105 | }, 106 | { 107 | "text": "add butter to the list", 108 | "intent": "add_item", 109 | "entities": [ 110 | { 111 | "start": 4, 112 | "end": 10, 113 | "value": "butter", 114 | "entity": "shopping_item" 115 | } 116 | ] 117 | }, 118 | { 119 | "text": "and butter please", 120 | "intent": "add_item", 121 | "entities": [ 122 | { 123 | "start": 4, 124 | "end": 10, 125 | "value": "butter", 126 | "entity": "shopping_item" 127 | } 128 | ] 129 | }, 130 | { 131 | "text": "clear list", 132 | "intent": "clear_list", 133 | "entities": [] 134 | }, 135 | { 136 | "text": "clear my list", 137 | "intent": "clear_list", 138 | "entities": [] 139 | }, 140 | { 141 | "text": "remove all items", 142 | "intent": "clear_list", 143 | "entities": [] 144 | }, 145 | { 146 | "text": "delete all from list", 147 | "intent": "clear_list", 148 | "entities": [] 149 | }, 150 | { 151 | "text": "i need eggs and milk", 152 | "intent": "add_item", 153 | "entities": [ 154 | { 155 | "start": 7, 156 | "end": 11, 157 | "value": "eggs", 158 | "entity": "shopping_item" 159 | }, 160 | { 161 | "start": 16, 162 | "end": 20, 163 | "value": "milk", 164 | "entity": "shopping_item" 165 | } 166 | ] 167 | }, 168 | { 169 | "text": "put more eggs", 170 | "intent": "add_item", 171 | "entities": [ 172 | { 173 | "start": 9, 174 | "end": 13, 175 | "value": "eggs", 176 | "entity": "shopping_item" 177 | } 178 | ] 179 | }, 180 | { 181 | "text": "add more eggs", 182 | "intent": "add_item", 183 | "entities": [ 184 | { 185 | "start": 9, 186 | "end": 13, 187 | "value": "eggs", 188 | "entity": "shopping_item" 189 | } 190 | ] 191 | }, 192 | { 193 | "text": "i need more milk", 194 | "intent": "add_item", 195 | "entities": [ 196 | { 197 | "start": 12, 198 | "end": 16, 199 | "value": "milk", 200 | "entity": "shopping_item" 201 | } 202 | ] 203 | }, 204 | { 205 | "text": "show my list", 206 | "intent": "show_items", 207 | "entities": [] 208 | }, 209 | { 210 | "text": "check list", 211 | "intent": "show_items", 212 | "entities": [] 213 | }, 214 | { 215 | "text": "display list", 216 | "intent": "show_items", 217 | "entities": [] 218 | }, 219 | { 220 | "text": "my shopping list", 221 | "intent": "show_items", 222 | "entities": [] 223 | }, 224 | { 225 | "text": "shopping list", 226 | "intent": "show_items", 227 | "entities": [] 228 | }, 229 | { 230 | "text": "add milk to the list", 231 | "intent": "add_item", 232 | "entities": [ 233 | { 234 | "start": 4, 235 | "end": 8, 236 | "value": "milk", 237 | "entity": "shopping_item" 238 | } 239 | ] 240 | } 241 | ] 242 | } 243 | } --------------------------------------------------------------------------------