├── .codeclimate.yml ├── .coveragerc ├── .gitignore ├── .travis.yml ├── LICENSE.md ├── MANIFEST.in ├── README.rst ├── config ├── irc.ini ├── irc.service ├── socket_io.ini ├── socket_io.service ├── vexbot.ini ├── vexbot.service ├── xmpp.ini ├── xmpp.service ├── youtube.ini └── youtube.service ├── data ├── add_extensions ├── cpu_count ├── get_help ├── get_installed_extensions ├── get_last_error ├── get_log ├── get_services ├── restart_process ├── start_process ├── status_process ├── stop_process ├── swap ├── virtual_memory_percent └── virtual_memory_used ├── docs ├── Makefile ├── adapter_configuration.rst ├── adapter_development.rst ├── conf.py ├── extension_development.rst ├── extension_discovery.rst ├── extension_management.rst ├── index.rst └── packages.rst ├── setup.py ├── tests ├── __init__.py ├── test_command_managers.py ├── test_messaging.py └── test_subprocess_manager.py └── vexbot ├── __init__.py ├── __main__.py ├── _logging.py ├── _version.py ├── adapters ├── __init__.py ├── irc │ ├── __init__.py │ ├── __main__.py │ ├── echo_to_message.py │ └── observer.py ├── messaging.py ├── shell │ ├── __init__.py │ ├── __main__.py │ ├── completers.py │ ├── intents.py │ ├── interfaces.py │ ├── observers.py │ ├── parser.py │ └── shell.py ├── socket_io │ ├── __init__.py │ ├── __main__.py │ └── observer.py ├── stackoverflow.py ├── xmpp │ ├── __init__.py │ └── __main__.py └── youtube │ ├── __init__.py │ └── __main__.py ├── command.py ├── command_observer.py ├── dynamic_entities.py ├── entity_extraction.py ├── extension_metadata.py ├── extensions ├── __init__.py ├── admin.py ├── develop.py ├── digitalocean.py ├── dynamic_loading.py ├── extensions.py ├── help.py ├── hidden.py ├── intents.py ├── log.py ├── modules.py ├── news.py ├── subprocess.py └── system.py ├── intents.py ├── language.py ├── messaging.py ├── observer.py ├── robot.py ├── scheduler.py ├── subprocess_manager.py └── util ├── __init__.py ├── create_cache_filepath.py ├── create_config_file.py ├── create_vexdir.py ├── generate_certificates.py ├── generate_config_file.py ├── get_cache_filepath.py ├── get_certificate_filepath.py ├── get_classifier_filepath.py ├── get_config.py ├── get_config_filepath.py ├── get_kwargs.py ├── get_vexdir_filepath.py ├── lru_cache.py ├── messaging.py └── socket_factory.py /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | languages: 2 | Python: true 3 | 4 | exclude_paths: 5 | - 'tests/*' 6 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = */tests/*, */distutils/*, /tmp/*, */site-packages/* 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generic files to ignore 2 | *~ 3 | *.lock 4 | *.DS_Store 5 | *.swp 6 | *.out 7 | ENV* 8 | 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | venv/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff/Logfiles 57 | *.log 58 | 59 | # Sphinx documentation 60 | docs/_build/ 61 | 62 | # PyBuilder 63 | target/ 64 | 65 | # Coverage reports for a specific version 66 | .coverage 67 | 68 | # SublimeLinter config file 69 | .sublimelinterrc 70 | 71 | settings.yml 72 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | notifications: 4 | email: false 5 | 6 | python: 7 | - "3.5" 8 | 9 | # command to install dependencies 10 | install: 11 | - pip install flake8 12 | - pip install coverage 13 | - pip install python-coveralls 14 | - pip install -e . 15 | 16 | # command to run tests 17 | script: 18 | - coverage run -m unittest discover 19 | - flake8 vexbot 20 | - flake8 tests 21 | 22 | after_success: 23 | - coverage report 24 | - coveralls 25 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include data/* 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | vexbot 3 | ====== 4 | 5 | |readthedocs| |codeclimate| 6 | 7 | .. |readthedocs| image:: https://readthedocs.org/projects/vexbot/badge/?version=latest 8 | :target: http://vexbot.readthedocs.io/en/latest/?badge=latest 9 | :alt: Documentation Status 10 | 11 | .. |codeclimate| image:: https://api.codeclimate.com/v1/badges/c17129f65b11dfef34c0/maintainability.svg 12 | :target: https://codeclimate.com/github/benhoff/vexbot/badges/gpa.svg 13 | :alt: Climate Status 14 | 15 | 16 | Pluggable bot 17 | 18 | Under development. Very useable but currently not feature complete. 19 | 20 | Requirements 21 | ------------ 22 | 23 | Requires python 3.5 for asyncio and only runs on linux. 24 | 25 | If you're a python developer, you can probably get this to run on not linux. 26 | 27 | Installation 28 | ------------ 29 | 30 | You will need an active DBus user session bus. Depending on your distro, you might already have one (Arch linux, for example). 31 | 32 | For Ubuntu: 33 | 34 | .. code-block:: bash 35 | 36 | $ apt-get install dbus-user-session python3-gi python3-dev python3-pip build-essential 37 | 38 | For everyone: 39 | 40 | .. code-block:: bash 41 | 42 | $ python3 -m venv 43 | 44 | .. code-block:: bash 45 | 46 | $ source /bin/activate 47 | 48 | .. code-block:: bash 49 | 50 | $ ln -s /usr/lib/python3/dist-packages/gi /lib/python3.5/site-packages/ 51 | 52 | .. code-block:: bash 53 | 54 | $ pip install vexbot[process_manager] 55 | 56 | Configuring 57 | ----------- 58 | 59 | Make sure your virtual environment is activated. Then run: 60 | 61 | .. code-block:: bash 62 | 63 | $ vexbot_generate_certificates 64 | 65 | .. code-block:: bash 66 | 67 | $ vexbot_generate_unit_file 68 | 69 | .. code-block:: bash 70 | 71 | $ systemctl --user daemon-reload 72 | 73 | Your bot is ready to run! 74 | 75 | Running 76 | ------- 77 | 78 | .. code-block:: bash 79 | 80 | $ systemctl --user start vexbot 81 | 82 | Or 83 | 84 | .. code-block:: bash 85 | 86 | $ vexbot_robot 87 | 88 | Please note that vexbot has a client/server architecture. The above commands will launch the server. To launch the command line client: 89 | 90 | .. code-block:: bash 91 | 92 | $ vexbot 93 | 94 | Exit the command line client by typing `!exit` or using `ctl+D`. 95 | -------------------------------------------------------------------------------- /config/irc.ini: -------------------------------------------------------------------------------- 1 | [bot] 2 | nick= 3 | password= 4 | host= 5 | port = 6 | # ssl = True 7 | # level = 30 8 | inlcudes = 9 | irc3.plugins.core 10 | irc3.plugins.autojoins 11 | autojoins = 12 | cmd = ! 13 | service_name= 14 | 15 | #[connection] 16 | #address= 17 | #publish_address= 18 | #subscribe_address= 19 | #command_address= 20 | #control_address= 21 | -------------------------------------------------------------------------------- /config/irc.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=IRC Client for Vexbot 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/path/to/virtual/env/binary /path/to/config 7 | 8 | StandardOutput=syslog 9 | StandardError=syslog 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /config/socket_io.ini: -------------------------------------------------------------------------------- 1 | [socket_io] 2 | streamer_name= 3 | namespace= 4 | website_url= 5 | service_name= 6 | 7 | #[connection] 8 | #address= 9 | #publish_address= 10 | #subscribe_address= 11 | #command_address= 12 | #control_address= 13 | -------------------------------------------------------------------------------- /config/socket_io.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Socket IO Adpater for Vexbot 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/path/to/socket_io/binary /path/to/config 7 | StandardOutput=syslog 8 | StandardError=syslog 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /config/vexbot.ini: -------------------------------------------------------------------------------- 1 | # version: "0.0.4" 2 | [vexbot] 3 | bot_name='vexbot' 4 | 5 | #[connection] 6 | #address = * 7 | #publish_address = 4000 8 | #subscribe_address = [4001,] 9 | #command_address = 4002 10 | #control_address = 4003 11 | -------------------------------------------------------------------------------- /config/vexbot.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Helper robot 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/path/to/vexbot/script 7 | StandardOutput=syslog 8 | StandardError=syslog 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /config/xmpp.ini: -------------------------------------------------------------------------------- 1 | [xmpp] 2 | # local is username 3 | local= 4 | domain= 5 | room= 6 | resource= 7 | nick= 8 | password= 9 | service_name= 10 | 11 | #[connection] 12 | #address= 13 | #publish_address= 14 | #subscribe_address= 15 | #command_address= 16 | #control_address= 17 | -------------------------------------------------------------------------------- /config/xmpp.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=XMPP Adapter for vexbot 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/path/to/xmpp/script /path/to/config/file 7 | StandardOutput=syslog 8 | StandardError=syslog 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /config/youtube.ini: -------------------------------------------------------------------------------- 1 | [youtube] 2 | client_secret_filepath= 3 | service_name= 4 | 5 | #[connection] 6 | #address = 7 | #publish_address= 8 | #subscribe_address= 9 | #command_address= 10 | #control_address= 11 | -------------------------------------------------------------------------------- /config/youtube.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Youtube Adapter for Vexbot 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/path/to/youtube/script /path/to/config/file 7 | StandardOutput=syslog 8 | StandardError=syslog 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /data/add_extensions: -------------------------------------------------------------------------------- 1 | load commands [x,y,z] 2 | -------------------------------------------------------------------------------- /data/cpu_count: -------------------------------------------------------------------------------- 1 | how many cores do we have 2 | get cpu cores 3 | -------------------------------------------------------------------------------- /data/get_help: -------------------------------------------------------------------------------- 1 | help me please 2 | what does this do 3 | what can I do 4 | what can you do 5 | -------------------------------------------------------------------------------- /data/get_installed_extensions: -------------------------------------------------------------------------------- 1 | show me what we've got installed 2 | -------------------------------------------------------------------------------- /data/get_last_error: -------------------------------------------------------------------------------- 1 | show me the error 2 | what's wrong with you 3 | tell me how you're broken 4 | what's wrong 5 | what happened 6 | -------------------------------------------------------------------------------- /data/get_log: -------------------------------------------------------------------------------- 1 | get log 2 | get logging 3 | get logging value 4 | get logs 5 | what is the logging 6 | what is the logging? 7 | show me the log 8 | show me the logging 9 | what is the logging value? 10 | -------------------------------------------------------------------------------- /data/get_services: -------------------------------------------------------------------------------- 1 | what services are up 2 | services up 3 | services available 4 | what is running 5 | running 6 | -------------------------------------------------------------------------------- /data/restart_process: -------------------------------------------------------------------------------- 1 | restart adapter [x] 2 | restart process [x] 3 | restart program [x] 4 | -------------------------------------------------------------------------------- /data/start_process: -------------------------------------------------------------------------------- 1 | start adapter [x] 2 | start process [x] 3 | start program [x] 4 | -------------------------------------------------------------------------------- /data/status_process: -------------------------------------------------------------------------------- 1 | status adapter 2 | status process 3 | status program 4 | -------------------------------------------------------------------------------- /data/stop_process: -------------------------------------------------------------------------------- 1 | stop adapter [x] 2 | stop process [x] 3 | stop program [x] 4 | -------------------------------------------------------------------------------- /data/swap: -------------------------------------------------------------------------------- 1 | show me the swap 2 | -------------------------------------------------------------------------------- /data/virtual_memory_percent: -------------------------------------------------------------------------------- 1 | how much memory are we using 2 | what's our memory utilization 3 | -------------------------------------------------------------------------------- /data/virtual_memory_used: -------------------------------------------------------------------------------- 1 | how much memory do we have 2 | how much memory 3 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = vexbot 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/adapter_configuration.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Adapter Configuration 3 | ===================== 4 | 5 | There are two things that need to be configured for most adapters. The `.service` file which systemd uses to launch the service, and an option configuration file, which can be used to pass in configurations that need to be persisted. 6 | 7 | See `this link`_ for example configurations for the packaged adapters. And the below for a primer on ZMQ addresses, if you desire to change the configuration of anything from running locally on the loopback address. 8 | 9 | .. _`this link`: https://github.com/benhoff/vexbot/tree/master/config 10 | 11 | Please note that the `.service` files should be put in `~/.config/systemd/user/*`. The `.ini` files may be placed almost anywhere, as long as they are referred to properly in the `.service` file, but recommend they be placed in `!/.config/vexbot/*` for consistency. 12 | 13 | 14 | Configuring ZMQ Addresses 15 | ------------------------- 16 | 17 | Addresses can be configured for the adapters and the bot itself in the .ini files. This is a bit more advanced and probably not recommended. 18 | 19 | The address expected is in the format of `tcp://[ADDRESS]:[PORT_NUMBER]`. 20 | 21 | For example `tcp://127.0.0.1:5617` is a valid address. 127.0.0.1 is the ADDRESS and 5617 is the PORT_NUMBER. 22 | 23 | 127.0.0.1 was chosen specifially as an example because for IPV4 it is the "localhost". Localhost is the computer the program is being run on. So if you want the program to connect to a socket on your local computer (you probably do), use 127.0.0.1. 24 | 25 | Port numbers range from 0-65536, and can be mostly aribratry chosen. For linux ports 0-1024 are reserved, so best to stay away from those. Port 5555 is usually used as an example port for coding examples, so probably best to stay away from that as well. 26 | 27 | The value of the `publish_address` and `subscribe_address` at the top of the settings file are likely what you want to copy for the `publish_address` and `subscribe_address` under shell, irc, xmpp, youtube, and socket_io if you're running everything locally on one computer. But you don't have to. You could run all the services on one computer and the main robot on a different computer. You would just need to configure the address and ports correctly, as well as work through any networking/port issues going across the local area network (LAN). 28 | -------------------------------------------------------------------------------- /docs/adapter_development.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Adapter Development 3 | =================== 4 | 5 | Create A New Adapter 6 | -------------------- 7 | 8 | Create a messaging instance and pass in a service name that will uniquely identify it. 9 | 10 | .. code-block:: python 11 | 12 | from vexbot.adapters.messaging import Messaging 13 | 14 | messaging = Messaging('unique_service_name', run_control_loop=True) 15 | messaging.run(blocking=False) 16 | # Your messaging code here 17 | 18 | # Some sort of loop, using a `while` loop for illustration purposes 19 | while True: 20 | author = '' 21 | message = '' 22 | # optional 23 | # channel = '' 24 | 25 | messaging.send_chatter(author=author, 26 | message=message) 27 | 28 | # NOTE: Alternate implementation 29 | """ 30 | messaging.send_chatter(author=author, 31 | message=message, 32 | channel=channel) 33 | """ 34 | 35 | Dope. But what about something that sends commands to the robot? 36 | 37 | .. code-block:: python 38 | 39 | from vexbot.adapters.messaging import Messaging 40 | 41 | messaging = Messaging('unique_service_name', run_control_loop=True) 42 | messaging.run(blocking=False) 43 | 44 | # Your code here. You would probably want this in a loop as well. 45 | command = '' 46 | args = [] 47 | kwargs = {} 48 | 49 | messaging.send_command(command, *args, **kwargs) 50 | 51 | You probably want a response back out of that command, huh? 52 | 53 | .. code-block:: python 54 | 55 | from vexbot.observer import Observer 56 | from vexbot.adapters.messaging import Messaging 57 | 58 | class MyObserver(Observer): 59 | def on_next(self, request): 60 | result = request.kwargs.get('result') 61 | # your code here 62 | 63 | def on_error(self, *args, **kwargs): 64 | pass 65 | 66 | def on_completed(*args, **kwargs): 67 | pass 68 | 69 | messaging = Messaging('unique_service_name', run_control_loop=True) 70 | messaging.run(blocking=False) 71 | 72 | my_observer = MyObserver() 73 | 74 | messaging.command.subscribe(my_observer) 75 | # You can also pass in methods to the `subscribe` method 76 | messaging.command.subscribe(your_custom_method_here) 77 | 78 | Actually you probably want the ability to dynamically load commands, persist your dynamic commands, and see all the installed commands available. 79 | 80 | .. code-block:: python 81 | 82 | import shelve 83 | from os import path 84 | from vexbot.observer import Observer 85 | from vexbot.extensions import extensions 86 | 87 | from vexbot.util.get_cache_filepath import get_cache 88 | from vexbot.util.get_cache_filepath import get_cache_filepath as get_cache_dir 89 | 90 | class MyObserver(Observer): 91 | extensions = (extensions.add_extensions, 92 | extensions.remove_extension, 93 | # NOTE: you can pass in dict's here to change the behavior 94 | {'method': your_method_here, 95 | 'hidden': True, 96 | 'name': 'some_alternate_method_name', 97 | 'alias': ['method_name2', 98 | 'method_name3']}, 99 | 100 | extensions.get_extensions, 101 | extensions.get_installed_extensions) 102 | 103 | def __init__(self): 104 | super().__init__() 105 | self._commands = {} 106 | cache_dir = get_cache_dir() 107 | mkdir = not path.isdir(cache_dir) 108 | if mkdir: 109 | os.makedirs(cache_dir, exist_ok=True) 110 | 111 | filepath = get_cache(__name__ + '.pickle') 112 | init = not path.isfile(filepath) 113 | 114 | self._config = shelve.open(filepath, flag='c', writeback=True) 115 | 116 | if init: 117 | self._config['extensions'] = {} 118 | self._config['disabled'] = {} 119 | self._config['modules'] = {} 120 | 121 | # NOTE: Here's our command handeling 122 | def handle_command(self, command: str, *args, **kwargs): 123 | callback = self._commands.get(command) 124 | if callback is None: 125 | return 126 | 127 | # Wrap our callback to catch errors 128 | try: 129 | result = callback(*args, **kwargs) 130 | except Exception as e: 131 | self.on_error(command, e, args, kwargs) 132 | 133 | print(result) 134 | 135 | def on_next(self, request): 136 | # NOTE: Here's our responses back from the bot 137 | result = request.kwargs.get('result') 138 | # your code here 139 | 140 | def on_error(self, *args, **kwargs): 141 | pass 142 | 143 | def on_completed(*args, **kwargs): 144 | pass 145 | 146 | >> observer = MyObserver() 147 | >> observer.handle_command('get_extensions') 148 | >> [] 149 | >> observer.handle_command('add_extensions', 'log_level') 150 | >> observer.handle_command('get_extensions') 151 | >> ['log_level'] 152 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # vexbot documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Dec 26 19:40:36 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc', 35 | 'sphinx.ext.viewcode'] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # 43 | # source_suffix = ['.rst', '.md'] 44 | source_suffix = '.rst' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = 'vexbot' 51 | copyright = '2017, Ben Hoff' 52 | author = 'Ben Hoff' 53 | 54 | # The version info for the project you're documenting, acts as replacement for 55 | # |version| and |release|, also used in various other places throughout the 56 | # built documents. 57 | # 58 | # The short X.Y version. 59 | version = '0.4.0' 60 | # The full version, including alpha/beta/rc tags. 61 | release = '0.4.0' 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | # 66 | # This is also used if you do content translation via gettext catalogs. 67 | # Usually you set "language" from the command line for these cases. 68 | language = None 69 | 70 | # List of patterns, relative to source directory, that match files and 71 | # directories to ignore when looking for source files. 72 | # This patterns also effect to html_static_path and html_extra_path 73 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 74 | 75 | # The name of the Pygments (syntax highlighting) style to use. 76 | pygments_style = 'sphinx' 77 | 78 | # If true, `todo` and `todoList` produce output, else they produce nothing. 79 | todo_include_todos = False 80 | 81 | 82 | # -- Options for HTML output ---------------------------------------------- 83 | 84 | # The theme to use for HTML and HTML Help pages. See the documentation for 85 | # a list of builtin themes. 86 | # 87 | html_theme = 'alabaster' 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # html_theme_options = {} 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | # Custom sidebar templates, must be a dictionary that maps document names 101 | # to template names. 102 | # 103 | # This is required for the alabaster theme 104 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 105 | html_sidebars = { 106 | '**': [ 107 | 'relations.html', # needs 'show_related': True theme option to display 108 | 'searchbox.html', 109 | ] 110 | } 111 | 112 | 113 | # -- Options for HTMLHelp output ------------------------------------------ 114 | 115 | # Output file base name for HTML help builder. 116 | htmlhelp_basename = 'vexbotdoc' 117 | 118 | 119 | # -- Options for LaTeX output --------------------------------------------- 120 | 121 | latex_elements = { 122 | # The paper size ('letterpaper' or 'a4paper'). 123 | # 124 | # 'papersize': 'letterpaper', 125 | 126 | # The font size ('10pt', '11pt' or '12pt'). 127 | # 128 | # 'pointsize': '10pt', 129 | 130 | # Additional stuff for the LaTeX preamble. 131 | # 132 | # 'preamble': '', 133 | 134 | # Latex figure (float) alignment 135 | # 136 | # 'figure_align': 'htbp', 137 | } 138 | 139 | # Grouping the document tree into LaTeX files. List of tuples 140 | # (source start file, target name, title, 141 | # author, documentclass [howto, manual, or own class]). 142 | latex_documents = [ 143 | (master_doc, 'vexbot.tex', 'vexbot Documentation', 144 | 'Ben Hoff', 'manual'), 145 | ] 146 | 147 | 148 | # -- Options for manual page output --------------------------------------- 149 | 150 | # One entry per manual page. List of tuples 151 | # (source start file, name, description, authors, manual section). 152 | man_pages = [ 153 | (master_doc, 'vexbot', 'vexbot Documentation', 154 | [author], 1) 155 | ] 156 | 157 | 158 | # -- Options for Texinfo output ------------------------------------------- 159 | 160 | # Grouping the document tree into Texinfo files. List of tuples 161 | # (source start file, target name, title, author, 162 | # dir menu entry, description, category) 163 | texinfo_documents = [ 164 | (master_doc, 'vexbot', 'vexbot Documentation', 165 | author, 'vexbot', 'One line description of project.', 166 | 'Miscellaneous'), 167 | ] 168 | 169 | 170 | 171 | -------------------------------------------------------------------------------- /docs/extension_development.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Extension Development 3 | ===================== 4 | 5 | Foreword 6 | -------- 7 | 8 | Currently extensions are tied very closely to python's packaging ecosystem. This is unlikely to change, honestly. 9 | 10 | 11 | Setup a Packging Environment 12 | ---------------------------- 13 | 14 | .. code-block:: bash 15 | 16 | $ # Note that the directory these commands are run from is `~/my_vexbot_extensions`! 17 | 18 | $ mkdir my_vexbot_extenstions 19 | $ cd my_vexbot_extensions 20 | 21 | Create a `setup.py` file. 22 | 23 | .. code-block:: python 24 | 25 | # filepath: my_vexbot_extensions/setup.py 26 | from setuptools import setup 27 | 28 | setup(name='my_vexbot_extensions') 29 | 30 | Now take a breather. Python packaging is hard, and we're almost done! 31 | 32 | Extensions Development 33 | ---------------------- 34 | 35 | Done with your breather? Good. Now let's say we've got a `hello_world` function that we want to use such as the below. 36 | 37 | .. code-block:: python 38 | 39 | # filepath: my_vexbot_extensions/hello_world.py 40 | 41 | def hello(*args, **kwargs): 42 | print('Hello World!') 43 | 44 | Dope. Let's add this as an extension in our `setup.py` file. 45 | 46 | .. code-block:: python 47 | 48 | # filepath: my_vexbot_extensions/setup.py 49 | from setuptools import setup 50 | 51 | setup(name='my_vexbot_extensions', 52 | entry_points={'vexbot_extensions':['hello=hello_world:hello']) 53 | 54 | That's it. Let's make sure the package is installed on our path. Make sure you're in the directory `my_vexbot_extensions` (or whatever you named your folder). 55 | 56 | .. code-block:: bash 57 | 58 | $ # Note that the directory these commands are run from is `~/my_vexbot_extensions`! 59 | 60 | $ ls 61 | setup.py hello_world.py 62 | 63 | $ python setup.py develop 64 | 65 | You might say, wait, that's not a normal python packaging structure! And you'd be right, so let's look at how it'll change for that. We'll make a directory called `my_src` and create a file in there known as `goodbye_world.py` 66 | 67 | .. code-block:: python 68 | 69 | # filepath: my_vexbot_extensions/my_src/goodbye_world.py 70 | 71 | def angsty(*args, **kwargs): 72 | print('Goodbye, cruel world!') 73 | 74 | Note how I'm taking an arbitrary amount of arguments and key word arguments using `*args and **kwargs`_? You should do that for every extension, or your program will error out at somepoint when it gets called with metadata that it's not ready for. 75 | 76 | .. code-block:: python 77 | 78 | # filepath: my_vexbot_extensions/my_src/goodbye_world.py 79 | 80 | def angsty(*args, **kwargs): 81 | print('Goodbye, cruel world!') 82 | 83 | # Note: Do NOT make extensions without using `*args, **kwargs` 84 | def function_that_will_fail_on_metadata(): 85 | print('Please notice the lack of flexibility in this function for taking arguments') 86 | print('This type of extension will inevitabley throw `TypeError` exceptions if put in a codebase') 87 | 88 | 89 | But back to registering our extension in our `setup.py` file. Remember that the filepath for this is `my_vexbot_extensions/my_src/goodbye_world.py`. 90 | 91 | .. code-block:: python 92 | 93 | # filepath: my_vexbot_extensions/setup.py 94 | from setuptools import setup 95 | 96 | setup(name='my_vexbot_extensions', 97 | entry_points={'vexbot_extensions':['hello=hello_world:hello', 98 | 'angsty=my_src.goodbye_world:angsty']) 99 | 100 | Notice how we use the `.` operator to represent the folder directory, and the `:` to specify the method name? That's important. 101 | 102 | We can have multiple methods in a file that way. 103 | 104 | .. code-block:: python 105 | 106 | # filepath: my_vexbot_extensions/my_src/goodbye_world.py 107 | 108 | def angsty(*args, **kwargs): 109 | print('Goodbye, cruel world!') 110 | 111 | def crocodile(*args, **kwargs): 112 | print('Goodbye, crocodile!') 113 | 114 | .. code-block:: python 115 | 116 | # filepath: my_vexbot_extensions/setup.py 117 | from setuptools import setup 118 | 119 | setup(name='my_vexbot_extensions', 120 | entry_points={'vexbot_extensions':['hello=hello_world:hello', 121 | 'angsty=my_src.goodbye_world:angsty', 122 | 'crocodile=my_src.goodbye_world:crocodile']) 123 | 124 | If you have a deeply python nested file, such as one in `my_vexbot_extensions/my_src/how/deep/we/go.py`... 125 | .. code-block:: python 126 | 127 | # filepath: my_vexbot_extensions/setup.py 128 | from setuptools import setup 129 | 130 | setup(name='my_vexbot_extensions', 131 | entry_points={'vexbot_extensions':['hello=hello_world:hello', 132 | 'angsty=my_src.goodbye_world:angsty', 133 | 'crocodile=my_src.goodbye_world:crocodile', 134 | 'nested_func=my_src.how.deep.we.go:waay_to_much']) 135 | 136 | Note that each folder is separated by the `.` operator, and the function name in the above example is `waay_to_much`, which is how deeply I feel that function is nested for a simple example such as this. 137 | 138 | Remember, remember the 5th of November. 139 | And also to re-run `python setup.py develop` once you've added an entry point/extension to your `setup.py` file. 140 | 141 | .. code-block:: bash 142 | 143 | $ # Note that the directory these commands are run from is `~/my_vexbot_extensions`! 144 | 145 | $ ls 146 | setup.py hello_world.py my_src 147 | 148 | $ python setup.py develop 149 | 150 | The string used before the path decleration, I.e the `nested_func` in the string `nested_func=my_src.how.deep.we.go:waay_to_much` is the name you will use in vexbot itself or the `hello` in `hello=hellow_world:hello`. 151 | 152 | Let's add our hello world greeting to our command line interface. 153 | 154 | 155 | .. code-block:: bash 156 | 157 | $ vexbot 158 | 159 | vexbot: !add_extensions hello 160 | vexbot: !hello 161 | Hello World! 162 | 163 | You can also add the hello world to the robot instance as well. 164 | 165 | .. code-block:: bash 166 | 167 | $ vexbot 168 | 169 | vexbot: !add_extensions hello --remote 170 | 171 | 172 | Last, but not least, you can specify command name alias's, specify if the command should be hidden, and give a short description for what the command does by using the `command` decorator. 173 | 174 | .. code-block:: python 175 | 176 | # filepath: my_vexbot_extensions/hello_world.py 177 | from vexbot.commands import command 178 | 179 | 180 | @command(alias=['hello_world', 'world_hello'], 181 | hidden=True, # default is `False` 182 | short='Prints `Hello World`!') 183 | def hello(*args, **kwargs): 184 | print('Hello World!') 185 | 186 | .. _`*args and **kwargs`: https://stackoverflow.com/questions/3394835/args-and-kwargs 187 | -------------------------------------------------------------------------------- /docs/extension_discovery.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Extension Discovery 3 | =================== 4 | 5 | See Installed Extensions 6 | ------------------------ 7 | 8 | Installed extensions are on your path, but not neccesairly attached to anything running currently 9 | 10 | .. code-block:: bash 11 | 12 | vexbot: !get_installed_extensions 13 | 14 | This command displays a list of every extension installed with a short doc string on what it does. 15 | 16 | For example, the command `power_off` will be displayed as 17 | 18 | 'power_off: Power off a digital ocean droplet' 19 | 20 | You can also use a module name to see all the extensions that are provided by that module. `vexbot.extensions.subprocess` is an installed module for vexbot. To see all the extensions that are provided by that module: 21 | 22 | .. code-block:: bash 23 | 24 | vexbot: !get_installed_modules vexbot.extensions.subprocess 25 | 26 | 27 | 28 | See Installed Modules 29 | --------------------- 30 | 31 | There are a lot of installed extensions and it's hard to figure out what each one does. 32 | You can break them up into modules 33 | 34 | .. code-block:: bash 35 | 36 | vexbot: !get_installed_modules 37 | 38 | This is helpful because the installed modules can be used with the `get_installed_extensions` to narrow down what is shown. For example, the every extensions in the module `vexbot.extensions.digitalocean` can be shown by using the following command: 39 | 40 | .. code-block:: bash 41 | 42 | vexbot: !get_installed_modules vexbot.extensions.digitalocean 43 | 44 | .. TODO See extension code would be helpful. Also implementing typing information. Also seeing the documentation. Also seeing some sort of use documentation 45 | -------------------------------------------------------------------------------- /docs/extension_management.rst: -------------------------------------------------------------------------------- 1 | ==================== 2 | Extension Management 3 | ==================== 4 | 5 | 6 | For the most part it's assumed that you are running in the command line program for this documentation. 7 | 8 | .. code-block:: bash 9 | 10 | $ vexbot 11 | 12 | vexbot: 13 | 14 | 15 | See Current Commands 16 | -------------------- 17 | .. code-block:: bash 18 | 19 | vexbot: !commands 20 | 21 | See bot commands 22 | 23 | .. code-block:: bash 24 | 25 | vexbot: !commands --remote 26 | 27 | 28 | .. TODO I'm not sure how you get commands for a service right now? Think I might have regressed that functionality during development 29 | 30 | See Extensions In Use 31 | --------------------- 32 | 33 | .. code-block:: bash 34 | 35 | vexbot: !extensions 36 | 37 | From bot: 38 | 39 | .. code-block:: bash 40 | 41 | vexbot: !extensions --remote 42 | 43 | Remove Extensions In Use 44 | ------------------------ 45 | Let's say you've the `get_cache` (shows you your configuration cache) and the `cpu_count` extensions in use and you'd like to remove them. 46 | 47 | 48 | .. code-block:: bash 49 | 50 | vexbot: !remove_extension get_cache cpu_count 51 | 52 | Alternatively just removing one: 53 | 54 | .. code-block:: bash 55 | 56 | vexbot: !remove_extension get_cache 57 | 58 | See Installed Extensions 59 | ------------------------ 60 | 61 | Installed extensions are on your path, but not neccesairly attached to anything running currently 62 | 63 | .. code-block:: bash 64 | 65 | vexbot: !get_installed_extensions 66 | 67 | This command displays a list of every extension installed with a short doc string on what it does. 68 | 69 | For example, the command `power_off` will be displayed as 70 | 71 | 'power_off: Power off a digital ocean droplet' 72 | 73 | You can also use a module name to see all the extensions that are provided by that module. `vexbot.extensions.subprocess` is an installed module for vexbot. To see all the extensions that are provided by that module: 74 | 75 | .. code-block:: bash 76 | 77 | vexbot: !get_installed_modules vexbot.extensions.subprocess 78 | 79 | 80 | 81 | See Installed Modules 82 | --------------------- 83 | 84 | There are a lot of installed extensions and it's hard to figure out what each one does. 85 | You can break them up into modules 86 | 87 | .. code-block:: bash 88 | 89 | vexbot: !get_installed_modules 90 | 91 | This is helpful because the installed modules can be used with the `get_installed_extensions` to narrow down what is shown. For example, the every extensions in the module `vexbot.extensions.digitalocean` can be shown by using the following command: 92 | 93 | .. code-block:: bash 94 | 95 | vexbot: !get_installed_modules vexbot.extensions.digitalocean 96 | 97 | 98 | Add Extensions 99 | -------------- 100 | 101 | .. code-block:: bash 102 | 103 | vexbot: !add_extensions get_code delete_cache 104 | 105 | To add commands to the robot instance: 106 | 107 | .. code-block:: bash 108 | 109 | vexbot: !add_extensions get_code delete_cache --remote 110 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to vexbot's documentation! 2 | ================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | extension_development 9 | extension_management 10 | extension_discovery 11 | adapter_development 12 | adapter_configuration 13 | packages 14 | 15 | 16 | .. TODO role_management 17 | .. Should commands be called extensions? 18 | 19 | 20 | Installation 21 | ------------ 22 | 23 | You will need an active DBus user session bus. Depending on your distro, you might already have one (Arch linux, for example). 24 | 25 | For Ubuntu: 26 | 27 | .. code-block:: bash 28 | 29 | $ apt-get install dbus-user-session python3-gi python3-dev python3-pip build-essential 30 | 31 | For everyone: 32 | 33 | .. code-block:: bash 34 | 35 | $ python3 -m venv 36 | 37 | .. code-block:: bash 38 | 39 | $ source /bin/activate 40 | 41 | .. code-block:: bash 42 | 43 | $ ln -s /usr/lib/python3/dist-packages/gi /lib/python3.5/site-packages/ 44 | 45 | .. code-block:: bash 46 | 47 | $ pip install vexbot[process_manager] 48 | 49 | Make sure your virtual environment is activated. Then run: 50 | 51 | .. code-block:: bash 52 | 53 | $ vexbot_generate_certificates 54 | 55 | .. code-block:: bash 56 | 57 | $ vexbot_generate_unit_file 58 | 59 | .. code-block:: bash 60 | 61 | $ systemctl --user daemon-reload 62 | 63 | Your bot is ready to run! 64 | 65 | Running 66 | ------- 67 | 68 | .. code-block:: bash 69 | 70 | $ systemctl --user start vexbot 71 | 72 | Or 73 | 74 | .. code-block:: bash 75 | 76 | $ vexbot_robot 77 | 78 | Please note that vexbot has a client/server architecture. The above commands will launch the server. To launch the command line client: 79 | 80 | .. code-block:: bash 81 | 82 | $ vexbot 83 | 84 | Exit the command line client by typing `!exit` or using `ctl+D`. 85 | 86 | 87 | Indices and tables 88 | ================== 89 | 90 | * :ref:`genindex` 91 | * :ref:`modindex` 92 | * :ref:`search` 93 | -------------------------------------------------------------------------------- /docs/packages.rst: -------------------------------------------------------------------------------- 1 | Packages 2 | ======== 3 | 4 | +===================+=========+ 5 | | required packages | License | 6 | +===================+=========+ 7 | | vexmessage | GPL3 | 8 | +-------------------+---------+ 9 | | pyzmq | BSD | 10 | +-------------------+---------+ 11 | | rx | Apache | 12 | +-------------------+---------+ 13 | | tblib | BSD | 14 | +-------------------+---------+ 15 | | tornado | Apache | 16 | +-------------------+---------+ 17 | 18 | Optional Packages 19 | ----------------- 20 | 21 | +==================+=========+ 22 | | nlp | License | 23 | +==================+=========+ 24 | | spacy | | 25 | +------------------+---------+ 26 | | sklearn | | 27 | +------------------+---------+ 28 | | sklearn_crfsuite | | 29 | +------------------+---------+ 30 | | wheel | | 31 | +------------------+---------+ 32 | 33 | +==================+=========+ 34 | | socket_io | License | 35 | +==================+=========+ 36 | | requests | | 37 | +------------------+---------+ 38 | | websocket-client | | 39 | +------------------+---------+ 40 | 41 | +===============+=========+ 42 | | summarization | License | 43 | +===============+=========+ 44 | | gensim | | 45 | +---------------+---------+ 46 | | newspaper3k | | 47 | +---------------+---------+ 48 | 49 | +==========================+=========+ 50 | | youtube | License | 51 | +==========================+=========+ 52 | | google-api-python-client | | 53 | +--------------------------+---------+ 54 | 55 | +========+=========+ 56 | | dev | License | 57 | +========+=========+ 58 | | flake8 | | 59 | +--------+---------+ 60 | | twine | | 61 | +--------+---------+ 62 | | wheel | | 63 | +--------+---------+ 64 | 65 | +===========+=========+ 66 | | xmpp | License | 67 | +===========+=========+ 68 | | sleekxmpp | | 69 | +-----------+---------+ 70 | | dnspython | | 71 | +-----------+---------+ 72 | 73 | 74 | +==============+=========+ 75 | | process_name | License | 76 | +==============+=========+ 77 | | setproctitle | | 78 | +--------------+---------+ 79 | 80 | 81 | +==============+=========+ 82 | | speechtotext | License | 83 | +==============+=========+ 84 | | speechtotext | | 85 | +--------------+---------+ 86 | 87 | 88 | +=================+=========+ 89 | | process_manager | License | 90 | +=================+=========+ 91 | | pydus | | 92 | +-----------------+---------+ 93 | 94 | 95 | +=================+=========+ 96 | | gui | License | 97 | +=================+=========+ 98 | | chatimusmaximus | | 99 | +-----------------+---------+ 100 | 101 | 102 | +======+=========+ 103 | | irc | License | 104 | +======+=========+ 105 | | irc3 | | 106 | +------+---------+ 107 | 108 | 109 | +============+=========+ 110 | | microphone | License | 111 | +============+=========+ 112 | | microphone | | 113 | +------------+---------+ 114 | 115 | 116 | +==============+=========+ 117 | | speechtotext | License | 118 | +==============+=========+ 119 | | speechtotext | | 120 | +--------------+---------+ 121 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from setuptools import find_packages, setup 5 | from vexbot.extension_metadata import extensions 6 | 7 | 8 | VERSIONFILE = 'vexbot/_version.py' 9 | verstrline = open(VERSIONFILE, 'rt').read() 10 | VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" 11 | mo = re.search(VSRE, verstrline, re.M) 12 | if mo: 13 | verstr = mo.group(1) 14 | else: 15 | raise RuntimeError("Unable to find version string in {}".format(VERSIONFILE)) 16 | 17 | 18 | # directory = os.path.abspath(os.path.dirname(__file__)) 19 | """ 20 | with open(os.path.join(directory, 'README.rst')) as f: 21 | long_description = f.read() 22 | """ 23 | 24 | str_form = '{}={}{}' 25 | extensions_ = [] 26 | for name, extension in extensions.items(): 27 | extras = extension.get('extras') 28 | if extras is None: 29 | extras = '' 30 | # FIXME: This will error out weirdly if there's not a list 31 | else: 32 | extras = ', '.join(extras) 33 | extras = ' [' + extras + ']' 34 | line = str_form.format(name, extension['path'], extras) 35 | extensions_.append(line) 36 | 37 | setup( 38 | name="vexbot", 39 | version=verstr, 40 | description='Python personal assistant', 41 | # long_description=long_description, 42 | url='https://github.com/benhoff/vexbot', 43 | license='GPL3', 44 | classifiers=[ 45 | 'Development Status :: 2 - Pre-Alpha', 46 | 'Intended Audience :: Developers', 47 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 48 | 'Programming Language :: Python :: 3', 49 | 'Programming Language :: Python :: 3.5', 50 | 'Programming Language :: Python :: 3.6', 51 | 'Operating System :: POSIX :: Linux'], 52 | 53 | author='Ben Hoff', 54 | author_email='beohoff@gmail.com', 55 | entry_points={'console_scripts': ['vexbot=vexbot.adapters.shell.__main__:main', 56 | 'vexbot_robot=vexbot.__main__:main', 57 | 'vexbot_irc=vexbot.adapters.irc.__main__:main', 58 | 'vexbot_xmpp=vexbot.adapters.xmpp:main', 59 | 'vexbot_socket_io=vexbot.adapters.socket_io.__main__:main', 60 | 'vexbot_youtube=vexbot.adapters.youtube:main', 61 | 'vexbot_stackoverflow=vexbot.adapters.stackoverflow:main', 62 | 'vexbot_generate_certificates=vexbot.util.generate_certificates:main', 63 | 'vexbot_generate_unit_file=vexbot.util.generate_config_file:main'], 64 | 'vexbot_extensions': extensions_}, 65 | packages=find_packages(), # exclude=['docs', 'tests'] 66 | 67 | install_requires=[ 68 | # 'pluginmanager>=0.4.1', 69 | 'pyzmq', 70 | 'vexmessage>=0.4.0', 71 | 'rx', 72 | 'tblib', # traceback serilization 73 | 'tornado', # zmq asnyc framework 74 | 'prompt_toolkit>=2.0.0', # shell 75 | ], 76 | 77 | extras_require={ 78 | 'nlp': ['wheel', 'spacy', 'sklearn', 'sklearn_crfsuite', 'scipy'], 79 | 'socket_io': ['requests', 'websocket-client'], 80 | 'summarization': ['gensim', 'newspaper3k'], 81 | 'youtube': ['google-api-python-client'], 82 | 'dev': ['flake8', 'twine', 'wheel', 'pygments', 'sphinx'], 83 | 'xmpp': ['sleekxmpp', 'dnspython'], 84 | 'process_name': ['setproctitle'], 85 | 'speechtotext': ['speechtotext'], 86 | 'digitalocean': ['python-digitalocean'], 87 | 'process_manager': ['pydbus'], 88 | 'command_line': ['pygments'], 89 | 'microphone': ['microphone'], 90 | 'database': ['vexstorage'], 91 | 'gui': ['chatimusmaximus'], 92 | 'entity': ['duckling'], 93 | 'irc': ['irc3'], 94 | 'system': ['psutil'], 95 | } 96 | ) 97 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_command_managers.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import vexmessage 3 | from vexbot.command_managers import (AdapterCommandManager, 4 | BotCommandManager, 5 | CommandManager) 6 | 7 | from vexbot.subprocess_manager import SubprocessManager 8 | 9 | 10 | class Message: 11 | def __init__(self): 12 | self.response = None 13 | self.status = None 14 | self.commands = None 15 | 16 | def send_response(self, *args, **kwargs): 17 | self.response = (args, kwargs) 18 | 19 | def send_status(self, status): 20 | self.status = status 21 | 22 | def send_command(self, *args, **kwargs): 23 | self.commands = [args, kwargs] 24 | 25 | 26 | class MockRobot: 27 | def __init__(self): 28 | self.messaging = Message() 29 | self.subprocess_manager = SubprocessManager() 30 | 31 | 32 | class TestCommandManager(unittest.TestCase): 33 | def setUp(self): 34 | self.command_manager = CommandManager(Message()) 35 | 36 | def test_register_command(self): 37 | def _test(): 38 | pass 39 | 40 | test_dict = {'blah': _test} 41 | self.command_manager.register_command('test', test_dict) 42 | self.assertIn('test', self.command_manager._commands) 43 | self.assertFalse(self.command_manager.is_command('test')) 44 | self.assertFalse(self.command_manager.is_command('blah')) 45 | self.command_manager.register_command('blah', _test) 46 | self.assertTrue(self.command_manager.is_command('blah')) 47 | # FIXME: ? 48 | self.assertTrue(self.command_manager.is_command('test blah')) 49 | 50 | def test_help(self): 51 | message = vexmessage.Message('target', 'source', 'CMD', command='help') 52 | self.command_manager.parse_commands(message) 53 | response_sent = self.command_manager._messaging.response[1] 54 | self.assertIn('response', response_sent) 55 | message = vexmessage.Message('target', 56 | 'source', 57 | 'CMD', 58 | command='help commands') 59 | 60 | self.command_manager.parse_commands(message) 61 | response_sent = self.command_manager._messaging.response[1]['response'] 62 | self.assertIn('help', response_sent) 63 | self.assertIn('commands', response_sent) 64 | 65 | def test_help_specific(self): 66 | message = vexmessage.Message('target', 67 | 'source', 68 | 'CMD', 69 | command='help', 70 | args=['commands']) 71 | 72 | self.command_manager.parse_commands(message) 73 | response = self.command_manager._messaging.response 74 | self.assertIsNotNone(response) 75 | 76 | def test_commands(self): 77 | message = vexmessage.Message('target', 78 | 'source', 79 | 'CMD', 80 | command='commands') 81 | 82 | self.command_manager.parse_commands(message) 83 | response_sent = self.command_manager._messaging.response 84 | answer = response_sent[1]['response'] 85 | self.assertIn('commands', answer) 86 | self.assertIn('help', answer) 87 | 88 | def test_is_command_with_call(self): 89 | 90 | def t(*args): 91 | pass 92 | 93 | self.command_manager.register_command('t', t) 94 | called = self.command_manager.is_command('t', True) 95 | self.assertTrue(called) 96 | not_called = self.command_manager.is_command('f', True) 97 | self.assertFalse(not_called) 98 | # FIXME: not sure what actually want here 99 | called_w_args = self.command_manager.is_command('t these are args', 100 | True) 101 | 102 | self.assertTrue(called_w_args) 103 | 104 | def test_get_callback_recursively_none(self): 105 | result = self.command_manager._get_callback_recursively(None) 106 | self.assertEqual(len(result), 3) 107 | for r in result: 108 | self.assertIsNone(r) 109 | 110 | def test_cmd_commands(self): 111 | def t(): 112 | pass 113 | self.command_manager.register_command('test', t) 114 | nested = {'test': t} 115 | self.command_manager.register_command('nested', nested) 116 | commands = self.command_manager._cmd_commands(None) 117 | self.assertIn('test', commands) 118 | self.assertIn('nested test', commands) 119 | self.assertNotIn('blah', commands) 120 | # TODO: test `help` and `commands`? 121 | self.assertEqual(len(commands), 4) 122 | 123 | 124 | class MockProcess: 125 | def __init__(self): 126 | self.killed = False 127 | 128 | def poll(self): 129 | return None 130 | 131 | def terminate(self): 132 | pass 133 | 134 | def kill(self): 135 | self.killed = True 136 | 137 | 138 | class TestBotCommandManager(unittest.TestCase): 139 | def setUp(self): 140 | robot = MockRobot() 141 | self.command_manager = BotCommandManager(robot) 142 | self.command_manager._messaging = Message() 143 | self.subprocess_manager = robot.subprocess_manager 144 | self.messaging = self.command_manager._messaging 145 | 146 | def test_settings(self): 147 | message = vexmessage.Message('target', 148 | 'source', 149 | 'CMD', 150 | command='subprocess', 151 | args='settings test', 152 | parsed_args=['test', ]) 153 | 154 | self.subprocess_manager.update_settings('test', 155 | {'value': 'test_value'}) 156 | 157 | self.command_manager.parse_commands(message) 158 | response = self.messaging.response[1]['response'] 159 | self.assertIn('value', response) 160 | 161 | def test_kill(self): 162 | mock = MockProcess() 163 | self.subprocess_manager._subprocess['test'] = mock 164 | message = vexmessage.Message('target', 165 | 'source', 166 | 'CMD', 167 | command='kill test') 168 | 169 | self.command_manager.parse_commands(message) 170 | self.assertTrue(mock.killed) 171 | 172 | def test_killall(self): 173 | def _mock_kill(*args, **kwargs): 174 | # due to scope issues, it's easiest to just tack on to the original 175 | # message to show that it was called correctly 176 | args[0].contents['killed'] = True 177 | 178 | self.subprocess_manager.killall = _mock_kill 179 | self.command_manager._commands['killall'] = _mock_kill 180 | 181 | message = vexmessage.Message('target', 182 | 'source', 183 | 'CMD', 184 | command='killall') 185 | 186 | self.command_manager.parse_commands(message) 187 | self.assertTrue(message.contents.get('killed')) 188 | 189 | def test_restart_bot(self): 190 | pass 191 | """ 192 | message = vexmessage.Message('target', 193 | 'source', 194 | 'CMD', 195 | command='restart_bot') 196 | 197 | with self.assertRaises(SystemExit): 198 | self.command_manager.parse_commands(message) 199 | """ 200 | 201 | def test_alive(self): 202 | self.subprocess_manager.register('test2', object()) 203 | message = vexmessage.Message('tgt', 'source', 'CMD', command='alive') 204 | self.command_manager.parse_commands(message) 205 | commands = self.messaging.commands 206 | self.assertEqual(commands[1].get('command'), 'alive') 207 | 208 | def test_start(self): 209 | # TODO: finish 210 | message = vexmessage.Message('tgt', 'source', 'CMD', command='start') 211 | self.command_manager.parse_commands(message) 212 | 213 | def test_subprocesses(self): 214 | self.subprocess_manager.register('test', object()) 215 | message = vexmessage.Message('tgt', 216 | 'source', 217 | 'CMD', 218 | command='subprocesses') 219 | 220 | self.command_manager.parse_commands(message) 221 | response = self.command_manager._messaging.response[1]['response'] 222 | self.assertIn('test', response) 223 | 224 | def test_restart(self): 225 | # TODO: finish 226 | message = vexmessage.Message('tgt', 'source', 'CMD', command='restart') 227 | self.command_manager.parse_commands(message) 228 | 229 | def test_terminate(self): 230 | # TODO: finish 231 | message = vexmessage.Message('target', 232 | 'source', 233 | 'CMD', 234 | command='terminate') 235 | 236 | self.command_manager.parse_commands(message) 237 | 238 | def test_running(self): 239 | 240 | self.subprocess_manager._subprocess['test'] = MockProcess() 241 | message = vexmessage.Message('tgt', 'source', 'CMD', command='running') 242 | self.command_manager.parse_commands(message) 243 | response = self.command_manager._messaging.response[1]['response'] 244 | self.assertIn('test', response) 245 | 246 | 247 | class TestAdapapterCommandManager(unittest.TestCase): 248 | def setUp(self): 249 | self.command_manager = AdapterCommandManager(Message()) 250 | 251 | def test_alive(self): 252 | messaging = self.command_manager._messaging 253 | 254 | self.command_manager._alive() 255 | self.assertIsNotNone(messaging.status) 256 | self.assertEqual(messaging.status, 'CONNECTED') 257 | -------------------------------------------------------------------------------- /tests/test_messaging.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from time import sleep 3 | 4 | import zmq 5 | from vexmessage import decode_vex_message 6 | from vexbot.messaging import Messaging 7 | 8 | 9 | class TestMessaging(unittest.TestCase): 10 | def __init__(self, *args, **kwargs): 11 | self.subscribe_address = 'tcp://127.0.0.1:4006' 12 | self.publish_address = 'tcp://127.0.0.1:4007' 13 | self.settings = {'subscribe_address': self.subscribe_address, 14 | 'publish_address': self.publish_address} 15 | 16 | context = zmq.Context() 17 | self.messaging = Messaging(self.settings, context) 18 | 19 | self.test_publish_socket = context.socket(zmq.PUB) 20 | self.test_publish_socket.connect(self.publish_address) 21 | 22 | self.test_subscribe_socket = context.socket(zmq.SUB) 23 | self.test_subscribe_socket.connect(self.subscribe_address) 24 | self.test_subscribe_socket.setsockopt_string(zmq.SUBSCRIBE, '') 25 | sleep(.2) 26 | super().__init__(*args, **kwargs) 27 | 28 | """ 29 | def test_send_message(self): 30 | self.messaging.send_message('', test='blue') 31 | frame = self.test_subscribe_socket.recv_multipart() 32 | message = decode_vex_message(frame) 33 | self.assertEqual(message.target, '') 34 | self.assertEqual(message.source, 'robot') 35 | self.assertEqual(message.contents['test'], 'blue') 36 | self.assertEqual(message.type, 'MSG') 37 | """ 38 | 39 | def test_send_targeted_message(self): 40 | self.test_subscribe_socket.setsockopt_string(zmq.SUBSCRIBE, 'test') 41 | self.messaging.send_message('test', test='blue') 42 | frame = self.test_subscribe_socket.recv_multipart() 43 | message = decode_vex_message(frame) 44 | self.assertEqual(message.target, 'test') 45 | self.assertEqual(message.source, 'robot') 46 | self.assertEqual(message.contents['test'], 'blue') 47 | self.assertEqual(message.type, 'MSG') 48 | -------------------------------------------------------------------------------- /tests/test_subprocess_manager.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from vexbot.subprocess_manager import SubprocessManager 4 | 5 | 6 | class TestSubprocessManager(unittest.TestCase): 7 | def setUp(self): 8 | self.subprocess_manager = SubprocessManager() 9 | # register a subprocess? With a name? 10 | 11 | def test_registered_subprocesses(self): 12 | test_obj = object() 13 | self.subprocess_manager.register('test', test_obj) 14 | registered = self.subprocess_manager.registered_subprocesses() 15 | self.assertIn('test', registered) 16 | 17 | def test_register_with_blacklist(self): 18 | blacklisted = 'black' 19 | self.subprocess_manager.blacklist.append(blacklisted) 20 | self.subprocess_manager.register(blacklisted, None) 21 | registered = self.subprocess_manager.registered_subprocesses() 22 | self.assertNotIn(blacklisted, registered) 23 | 24 | def test_update_setting_value(self): 25 | self.subprocess_manager.update_settings('test', {'test_value': 'old'}) 26 | self.subprocess_manager.update_setting_value('test', 27 | 'test_value', 28 | 'new') 29 | 30 | result = self.subprocess_manager.get_settings('test') 31 | self.assertEqual(result['test_value'], 'new') 32 | self.assertNotEqual(result['test_value'], 'old') 33 | 34 | def test_update_setting(self): 35 | settings = {'setting': 'value'} 36 | self.subprocess_manager.update_settings('test', settings) 37 | gotten_settings = self.subprocess_manager.get_settings('test') 38 | self.assertEqual(settings, gotten_settings) 39 | -------------------------------------------------------------------------------- /vexbot/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from vexbot._version import __version__ # flake8: noqa 3 | 4 | # FIXME: probably don't want to add NullHandler here 5 | # This is here to prevent the default handler being added upon init of the program 6 | logging.getLogger(__name__).addHandler(logging.NullHandler()) 7 | 8 | 9 | def _port_configuration_helper(configuration: dict) -> dict: 10 | default_port_config = _get_default_port_config() 11 | 12 | # get the port settings out of the configuration, falling back on defaults 13 | port_config = configuration.get('connection', default_port_config) 14 | 15 | # update the defaults with the retrived port configs 16 | default_port_config.update(port_config) 17 | # Overwrite the port config to be the updated default port config dict 18 | port_config = default_port_config 19 | 20 | return port_config 21 | 22 | 23 | # TODO: Rename this to be `_get_default_robot_port_config` or similar 24 | def _get_default_port_config() -> dict: 25 | """ 26 | protocol: 'tcp' 27 | address: '*' 28 | chatter_publish_port: 4000 29 | chatter_subscription_port: [4001,] 30 | command_port: 4002 31 | request_port: 4003 32 | control_port: 4005 33 | """ 34 | # Setup some default port configurations 35 | default_port_config= {'protocol': 'tcp', 36 | 'address': '*', 37 | 'chatter_publish_port': 4000, 38 | # FIXME: There's probably not a good reason to have 39 | # this be an iterable 40 | 'chatter_subscription_port': [4001,], 41 | 'request_port': 4003, 42 | 'command_port': 4004, 43 | 'control_port': 4005} 44 | 45 | return default_port_config 46 | 47 | 48 | def _get_default_adapter_config() -> dict: 49 | """ 50 | Returns: 51 | dict: The default adapter config for a locally run helper 52 | { 53 | protocol: 'tcp' 54 | address: '127.0.0.1' 55 | chatter_publish_port: 4000 56 | chatter_subscription_port: [4001,] 57 | command_port: 4002 58 | request_port: 4003 59 | control_port: 4005 60 | } 61 | """ 62 | config = _get_default_port_config() 63 | config['address'] = '127.0.0.1' 64 | return config 65 | -------------------------------------------------------------------------------- /vexbot/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # import atexit 4 | 5 | try: 6 | import setproctitle as _setproctitle 7 | except ImportError: 8 | _setproctitle = False 9 | 10 | from vexbot import _port_configuration_helper 11 | 12 | from vexbot.robot import Robot 13 | from vexbot.util.get_config_filepath import get_config_filepath 14 | from vexbot.util.get_kwargs import get_kwargs as _get_kwargs 15 | from vexbot.util.get_config import get_config as _get_config 16 | 17 | 18 | def _configuration_sane_defaults(configuration: dict) -> (dict, str): 19 | default_vexbot_settings = {'bot_name': 'vexbot'} 20 | # Get the settings out of the configuration, falling back on the defaults 21 | vexbot_settings = configuration.get('vexbot', default_vexbot_settings) 22 | # Get the robot name out of the configuration, falling back on the default 23 | robot_name = vexbot_settings.get('bot_name', 'vexbot') 24 | 25 | return robot_name 26 | 27 | 28 | def main(*args, **kwargs): 29 | """ 30 | `kwargs`: 31 | 32 | `configuration_filepath`: filepath for the `ini` configuration 33 | """ 34 | kwargs = {**kwargs, **_get_kwargs()} 35 | # FIXME: This filepath handeling is messed up and not transparent as it should be 36 | default_filepath = get_config_filepath() 37 | configuration_filepath = kwargs.get('configuration_filepath') 38 | if configuration_filepath is None: 39 | configuration_filepath = default_filepath 40 | # configuration is from an `ini` file 41 | configuration = _get_config(configuration_filepath) 42 | # setup some sane defaults 43 | robot_name = _configuration_sane_defaults(configuration) 44 | # Get the port configuration out of the configuration 45 | port_config = _port_configuration_helper(configuration) 46 | # create the settings manager using the port config 47 | if _setproctitle: 48 | _setproctitle.setproctitle('vexbot') 49 | 50 | robot = Robot(robot_name, port_config) 51 | robot.run() 52 | 53 | 54 | if __name__ == "__main__": 55 | main() 56 | -------------------------------------------------------------------------------- /vexbot/_logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from tblib import Traceback 3 | 4 | 5 | class LoopPubHandler(logging.Handler): 6 | def __init__(self, messaging, level=logging.NOTSET): 7 | super().__init__(level) 8 | self.messaging = messaging 9 | 10 | def emit(self, record): 11 | args = record.args 12 | if isinstance(args, tuple): 13 | args = [str(x) for x in record.args] 14 | elif isinstance(args, dict): 15 | args = str(args) 16 | # NOTE: Might need more specific handling in the future 17 | else: 18 | args = str(args) 19 | 20 | info = {'name': record.name, 21 | 'level': record.levelno, 22 | 'pathname': record.pathname, 23 | 'lineno': record.lineno, 24 | 'msg': record.msg, 25 | 'args': args, 26 | 'exc_info': record.exc_info, 27 | 'func': record.funcName, 28 | 'sinfo': record.stack_info, 29 | 'type': 'log'} 30 | exc_info = info['exc_info'] 31 | 32 | if exc_info is not None: 33 | # FIXME: It's hard to transmit an exception because the Error type 34 | # might be defined in a library that is not installed on the 35 | # receiving side. Not sure what the best way to do this is. 36 | info['exc_info'] = Traceback(exc_info[2]).to_dict() 37 | """ 38 | new_exc_info = [] 39 | first = exc_info[0] 40 | if isinstance(first, Exception): 41 | try: 42 | first = first.__name__ 43 | except AttributeError: 44 | first = first.__class__.__name__ 45 | print(first, dir(first), isinstance(first, Exception)) 46 | new_exc_info.append(first) 47 | new_exc_info.append([str(x) for x in exc_info[1].args]) 48 | new_exc_info.append(Traceback(exc_info[2]).to_dict()) 49 | info['exc_info'] = new_exc_info 50 | """ 51 | 52 | 53 | self.messaging.send_log(**info) 54 | 55 | 56 | class MessagingLogger: 57 | def __init__(self, service_name: str): 58 | name = service_name + '.messaging' 59 | self.command = logging.getLogger(name + '.command') 60 | self.control = logging.getLogger(name + '.control') 61 | self.publish = logging.getLogger(name + '.publish') 62 | self.subscribe = logging.getLogger(name + '.subscribe') 63 | self.request = logging.getLogger(name + '.request') 64 | -------------------------------------------------------------------------------- /vexbot/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.4.5' 2 | -------------------------------------------------------------------------------- /vexbot/adapters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/vexbot/adapters/__init__.py -------------------------------------------------------------------------------- /vexbot/adapters/irc/__init__.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import argparse 3 | import logging 4 | import pkg_resources 5 | 6 | from threading import Thread 7 | try: 8 | import irc3 9 | except ImportError: 10 | irc3 = False 11 | 12 | if not irc3: 13 | logging.exception('irc3 not installed!') 14 | raise ImportError('irc3 is not installed. Install irc3 using `pip install ' 15 | 'irc3` on the command line') 16 | 17 | from vexbot._logging import LoopPubHandler 18 | from vexbot.adapters.messaging import Messaging as _Messaging 19 | from vexbot.adapters.irc.observer import IrcObserver as _IrcObserver 20 | 21 | 22 | class IrcInterface: 23 | def __init__(self, 24 | service_name: str, 25 | irc_config: dict=None, 26 | connection: dict=None, 27 | **kwargs): 28 | 29 | 30 | if connection is None: 31 | connection = {} 32 | 33 | self.messaging = _Messaging(service_name, run_control_loop=True, **connection) 34 | 35 | self.root_handler = LoopPubHandler(self.messaging) 36 | self.root_logger = logging.getLogger() 37 | self.root_logger.addHandler(self.root_handler) 38 | 39 | self._scheduler_thread = Thread(target=self.messaging.start, 40 | daemon=True) 41 | 42 | self.irc_bot = irc3.IrcBot.from_config(irc_config) 43 | # Duck type messaging to irc bot. Could also subclass `irc3.IrcBot`, 44 | # but seems like overkill 45 | self.irc_bot.messaging = self.messaging 46 | self.command_parser = _IrcObserver(self.irc_bot, self.messaging, self) 47 | self.messaging.command.subscribe(self.command_parser) 48 | 49 | def run(self): 50 | self._scheduler_thread.start() 51 | 52 | self.irc_bot.create_connection() 53 | self.irc_bot.add_signal_handlers() 54 | event_loop = asyncio.get_event_loop() 55 | 56 | """ 57 | handle_close = _handle_close(messaging, 58 | event_loop) 59 | signal.signal(signal.SIGINT, handle_close) 60 | signal.signal(signal.SIGTERM, handle_close) 61 | """ 62 | event_loop.run_forever() 63 | event_loop.close() 64 | -------------------------------------------------------------------------------- /vexbot/adapters/irc/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import atexit 3 | import signal 4 | from vexbot._version import __version__ as version 5 | 6 | from vexbot.adapters.irc import IrcInterface 7 | 8 | """ 9 | try: 10 | pkg_resources.get_distribution('irc3') 11 | except pkg_resources.DistributionNotFound: 12 | _IRC3_INSTALLED = False 13 | 14 | if _IRC3_INSTALLED: 15 | import irc3 16 | 17 | else: 18 | pass 19 | """ 20 | 21 | import irc3 22 | from irc3 import utils 23 | 24 | 25 | def main(**kwargs): 26 | """ 27 | if not _IRC3_INSTALLED: 28 | logging.error('vexbot_irc requires `irc3` to be installed. Please install ' 29 | 'using `pip install irc3`') 30 | 31 | sys.exit(1) 32 | """ 33 | 34 | config = _from_argv(irc3.IrcBot, kwargs=kwargs) 35 | if not 'includes' in config: 36 | config['includes'] = [] 37 | 38 | message_plug = 'vexbot.adapters.irc.echo_to_message' 39 | if not message_plug in config['includes']: 40 | config['includes'].append(message_plug) 41 | service_name = config.get('service_name', 'irc') 42 | connection = config.get('connection', {}) 43 | interface = IrcInterface(service_name, irc_config=config, connection=connection) 44 | 45 | interface.run() 46 | sys.exit() 47 | 48 | 49 | # NOTE: This code is from `irc3` 50 | def _from_argv(cls, argv=None, **kwargs) -> dict: 51 | prog = cls.server and 'irc3d' or 'irc3' 52 | # TODO: Add in publish ports and all that jazz. 53 | doc = """ 54 | Run an __main__.py instance from a config file 55 | 56 | Usage: __main__.py [options] ... 57 | 58 | Options: 59 | -h, --help Display this help and exit 60 | --version Output version information and exit 61 | --logdir DIRECTORY Log directory to use instead of stderr 62 | --logdate Show datetimes in console output 63 | --host HOST Server name or ip 64 | --port PORT Server port 65 | -v,--verbose Increase verbosity 66 | -r,--raw Show raw irc log on the console 67 | -d,--debug Add some debug commands/utils 68 | """ 69 | import os 70 | import docopt 71 | import textwrap 72 | args = argv or sys.argv[1:] 73 | args = docopt.docopt(doc, args, version=version) 74 | cfg = utils.parse_config( 75 | cls.server and 'server' or 'bot', *args['']) 76 | cfg.update( 77 | verbose=args['--verbose'], 78 | debug=args['--debug'], 79 | ) 80 | cfg.update(kwargs) 81 | if args['--host']: # pragma: no cover 82 | host = args['--host'] 83 | cfg['host'] = host 84 | if host in ('127.0.0.1', 'localhost'): 85 | cfg['ssl'] = False 86 | if args['--port']: # pragma: no cover 87 | cfg['port'] = args['--port'] 88 | if args['--logdir'] or 'logdir' in cfg: 89 | logdir = os.path.expanduser(args['--logdir'] or cfg.get('logdir')) 90 | cls.logging_config = config.get_file_config(logdir) 91 | if args['--logdate']: # pragma: no cover 92 | fmt = cls.logging_config['formatters']['console'] 93 | fmt['format'] = config.TIMESTAMPED_FMT 94 | if args.get('--help-page'): # pragma: no cover 95 | for v in cls.logging_config['handlers'].values(): 96 | v['level'] = 'ERROR' 97 | if args['--raw']: 98 | cfg['raw'] = True 99 | 100 | return cfg 101 | 102 | 103 | if __name__ == '__main__': 104 | main() 105 | -------------------------------------------------------------------------------- /vexbot/adapters/irc/echo_to_message.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import irc3 3 | 4 | 5 | @irc3.plugin 6 | class EchoToMessage: 7 | requires = ['irc3.plugins.core', 8 | 'irc3.plugins.command'] 9 | 10 | def __init__(self, bot): 11 | self.bot = bot 12 | self.logger = logging.getLogger(__name__) 13 | 14 | @irc3.event(irc3.rfc.PRIVMSG) 15 | def message(self, mask, event, target, data): 16 | nick = mask.nick 17 | nick = str(nick) 18 | message = str(data) 19 | target = str(target) 20 | self.logger.info(' message %s %s %s', nick, message, target) 21 | self.bot.messaging.send_chatter(author=nick, 22 | message=message, 23 | channel=target) 24 | -------------------------------------------------------------------------------- /vexbot/adapters/irc/observer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import inspect as _inspect 3 | import logging 4 | 5 | from vexmessage import Request 6 | from vexbot.observer import Observer 7 | from vexbot.extensions import develop, admin 8 | from vexbot.extensions import help as vexhelp 9 | 10 | 11 | class IrcObserver(Observer): 12 | extensions = (develop.get_code, vexhelp.help, admin.get_commands) 13 | def __init__(self, bot, messaging, irc_interface): 14 | super().__init__() 15 | self.bot = bot 16 | self.messaging = messaging 17 | self.irc_interface = irc_interface 18 | self._commands = self._get_commands() 19 | self.logger = logging.getLogger(__name__) 20 | 21 | def _get_commands(self) -> dict: 22 | result = {} 23 | for name, method in _inspect.getmembers(self): 24 | if name.startswith('do_'): 25 | result[name[3:]] = method 26 | 27 | return result 28 | 29 | def do_log_level(self, *args, **kwargs): 30 | if not args: 31 | # FIXME 32 | return self.irc_interface.root_logger.level 33 | 34 | def do_debug(self, *args, **kwargs): 35 | self.irc_interface.root_logger.setLevel(logging.DEBUG) 36 | self.irc_interface.root_handler.setLevel(logging.DEBUG) 37 | 38 | def do_info(self, *args, **kwargs): 39 | self.irc_interface.root_logger.setLevel(logging.INFO) 40 | self.irc_interface.root_handler.setLevel(logging.INFO) 41 | 42 | def do_MSG(self, message, channel, *args, **kwargs): 43 | msg_target = kwargs.get('msg_target') 44 | if msg_target: 45 | message = msg_target + ', ' + message 46 | self.bot.privmsg(channel, message) 47 | 48 | def do_join(self, *args, **kwargs): 49 | self.bot.join(*args) 50 | 51 | def do_part(self, *args, **kwargs): 52 | self.bot.part(*args) 53 | 54 | def do_kick(self, channel, target, reason=None, *args, **kwargs): 55 | self.bot.kick(channel, target, reason) 56 | 57 | def do_away(self, message=None, *args, **kwargs): 58 | self.bot.away(message) 59 | 60 | def do_unaway(self, *args, **kwargs): 61 | self.bot.unaway() 62 | 63 | def do_topic(self, channel, topic=None, *args, **kwargs): 64 | self.bot.topic(channel, topic) 65 | 66 | def do_get_nick(self, *args, **kwargs): 67 | return self.bot.get_nick() 68 | 69 | def do_get_ip(self, *args, **kwargs): 70 | return str(self.bot.ip) 71 | 72 | def on_next(self, item: Request): 73 | command = item.command 74 | args = item.args 75 | kwargs = item.kwargs 76 | self.logger.debug(' command recieved: %s %s %s', command, args, kwargs) 77 | try: 78 | callback = self._commands[command] 79 | except KeyError: 80 | self.logger.info(' command not found: %s', command) 81 | return 82 | 83 | try: 84 | result = callback(*args, **kwargs) 85 | except Exception as e: 86 | self.on_error(e, command, args) 87 | return 88 | 89 | if result is None: 90 | self.logger.info('no result for callback: %s', command) 91 | return 92 | 93 | source = item.source 94 | # NOTE: probably need more here 95 | self.logger.debug(' send command response %s %s %s', source, command, result) 96 | service = self.messaging._service_name 97 | self.messaging.send_command_response(source, command, result=result, service=service, *args, **kwargs) 98 | 99 | def on_completed(self, *args, **kwargs): 100 | pass 101 | 102 | def on_error(self, *args, **kwargs): 103 | self.logger.exception('command failed') 104 | -------------------------------------------------------------------------------- /vexbot/adapters/shell/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/vexbot/adapters/shell/__init__.py -------------------------------------------------------------------------------- /vexbot/adapters/shell/__main__.py: -------------------------------------------------------------------------------- 1 | from vexbot.adapters.shell.shell import Shell 2 | 3 | 4 | def main(**kwargs): 5 | shell = Shell(**kwargs) 6 | return shell.run() 7 | 8 | 9 | if __name__ == '__main__': 10 | main() 11 | -------------------------------------------------------------------------------- /vexbot/adapters/shell/completers.py: -------------------------------------------------------------------------------- 1 | import re 2 | from six import string_types 3 | from prompt_toolkit.completion import Completer, Completion 4 | 5 | 6 | _WORD = re.compile(r'([!#a-zA-Z0-9_]+|[^!#a-zA-Z0-9_\s]+)') 7 | 8 | 9 | def _get_previous_word(document): 10 | # reverse the text before the curosr, in order to do an efficient 11 | # backwards search 12 | text_before_cursor = document.text_before_cursor[::-1] 13 | iterator = _WORD.finditer(text_before_cursor) 14 | count = 1 15 | try: 16 | for i, match in enumerate(iterator): 17 | if i == 0 and match.start(1) == 0: 18 | count += 1 19 | if i + 1 == count: 20 | return (-match.end(1), -match.start(1)) 21 | except StopIteration: 22 | pass 23 | return None, None 24 | 25 | 26 | class ServiceCompleter(Completer): 27 | def __init__(self, command_completer: Completer): 28 | self._command_completer = command_completer 29 | self.service_command_map = {} 30 | 31 | def _handle_services(self, previous_word: str, document, complete_event): 32 | if previous_word in self.service_command_map: 33 | return self.service_command_map[previous_word].get_completions(document, complete_event) 34 | else: 35 | return self._command_completer.get_completions(document, complete_event) 36 | 37 | def get_completions(self, document, complete_event): 38 | start, end = _get_previous_word(document) 39 | if start is None: 40 | return self._command_completer.get_completions(document, complete_event) 41 | 42 | offset = document.cursor_position 43 | start += offset 44 | end += offset 45 | previous_word = document.text[start:end] 46 | 47 | if previous_word in self.service_command_map: 48 | return self._handle_services(previous_word, document, complete_event) 49 | else: 50 | return self._command_completer.get_completions(document, complete_event) 51 | 52 | def set_service_completer(self, service: str, completer: Completer): 53 | self.service_command_map[service] = completer 54 | 55 | 56 | class WordCompleter(Completer): 57 | """ 58 | Simple autocompletion on a list of words. 59 | :param words: List of words. 60 | :param ignore_case: If True, case-insensitive completion. 61 | :param meta_dict: Optional dict mapping words to their meta-information. 62 | :param WORD: When True, use WORD characters. 63 | :param sentence: When True, don't complete by comparing the word before the 64 | cursor, but by comparing all the text before the cursor. In this case, 65 | the list of words is just a list of strings, where each string can 66 | contain spaces. (Can not be used together with the WORD option.) 67 | :param match_middle: When True, match not only the start, but also in the 68 | middle of the word. 69 | """ 70 | def __init__(self, words, ignore_case=False, meta_dict=None, WORD=False, 71 | sentence=False, match_middle=False): 72 | assert not (WORD and sentence) 73 | 74 | self.words = set(words) 75 | self.ignore_case = ignore_case 76 | self.meta_dict = meta_dict or {} 77 | self.WORD = WORD 78 | self.sentence = sentence 79 | self.match_middle = match_middle 80 | assert all(isinstance(w, string_types) for w in self.words) 81 | 82 | def get_completions(self, document, complete_event): 83 | # Get word/text before cursor. 84 | if self.sentence: 85 | word_before_cursor = document.text_before_cursor 86 | else: 87 | word_before_cursor = document.get_word_before_cursor(WORD=self.WORD) 88 | 89 | if self.ignore_case: 90 | word_before_cursor = word_before_cursor.lower() 91 | 92 | def word_matches(word): 93 | """ True when the word before the cursor matches. """ 94 | if self.ignore_case: 95 | word = word.lower() 96 | 97 | if self.match_middle: 98 | return word_before_cursor in word 99 | else: 100 | return word.startswith(word_before_cursor) 101 | 102 | for a in self.words: 103 | if word_matches(a): 104 | display_meta = self.meta_dict.get(a, '') 105 | yield Completion(a, -len(word_before_cursor), display_meta=display_meta) 106 | -------------------------------------------------------------------------------- /vexbot/adapters/shell/intents.py: -------------------------------------------------------------------------------- 1 | import inspect as _inspect 2 | 3 | class CommandItents: 4 | def get_intents(self) -> dict: 5 | result = {} 6 | for name, method in _inspect.getmembers(self): 7 | if name.startswith('do_'): 8 | result[name[3:]] = method 9 | elif method._vex_intent: 10 | result[method._vex_intent_name] = method 11 | 12 | return result 13 | 14 | def do_stop_chatter(self): 15 | pass 16 | 17 | def do_start_chatter(self): 18 | pass 19 | 20 | def do_change_color(self): 21 | pass 22 | -------------------------------------------------------------------------------- /vexbot/adapters/shell/interfaces.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from vexbot.adapters.shell.observers import AuthorObserver 4 | from vexbot.adapters.shell.observers import ServiceObserver 5 | from vexbot.util.lru_cache import LRUCache as _LRUCache 6 | 7 | 8 | def _add_word(completer): 9 | """ 10 | Used to add words to the completors 11 | """ 12 | def inner(word: str): 13 | completer.words.add(word) 14 | return inner 15 | 16 | 17 | def _remove_word(completer): 18 | """ 19 | Used to remove words from the completors 20 | """ 21 | def inner(word: str): 22 | try: 23 | completer.words.remove(word) 24 | except Exception: 25 | pass 26 | return inner 27 | 28 | 29 | 30 | class AuthorInterface: 31 | def __init__(self, word_completer, messaging): 32 | self.author_observer = AuthorObserver(self) 33 | 34 | messaging.chatter.subscribe(self.author_observer) 35 | self.authors = _LRUCache(100, 36 | _add_word(word_completer), 37 | _remove_word(word_completer)) 38 | self.author_metadata = _LRUCache(100) 39 | self.metadata_words = ['channel', ] 40 | 41 | def add_author(self, author: str, source: str, **metadata: dict): 42 | # NOTE: this fixes the parsing for usernames 43 | author = author.replace(' ', '_') 44 | 45 | self.authors[author] = source 46 | self.author_metadata[author] = {k: v for k, v in metadata.items() 47 | if k in self.metadata_words} 48 | 49 | def is_author(self, author: str): 50 | return author in self.authors 51 | 52 | def get_metadata(self, author: str, kwargs: dict) -> dict: 53 | source = self.author_observer.authors[author] 54 | metadata = self.author_observer.author_metadata[author] 55 | # TODO: Determine if this makes the most sense to update in 56 | metadata.update(kwargs) 57 | kwargs = metadata 58 | kwargs['target'] = source 59 | kwargs['msg_target'] = author 60 | return kwargs 61 | 62 | 63 | class ServiceInterface: 64 | def __init__(self, word_completer, messaging): 65 | self.service_observer = ServiceObserver(self) 66 | messaging.chatter.subscribe(self.service_observer) 67 | 68 | self.services = set() 69 | self.channels = _LRUCache(100, 70 | _add_word(word_completer), 71 | _remove_word(word_completer)) 72 | 73 | def add_service(self, source: str, channel: str=None): 74 | if source not in self.services: 75 | self.services.add(source) 76 | # FIXME: hack to append word to completer 77 | self.channels.add_callback(source) 78 | 79 | if channel is not None and channel not in self.channels: 80 | self.channels[channel] = source 81 | 82 | def is_service(self, service: str): 83 | in_service = service in self.services 84 | in_channel = service in self.channels 85 | return in_service or in_channel 86 | 87 | 88 | def get_metadata(self, service: str, kwargs: dict) -> dict: 89 | if service in self.channels: 90 | channel = service 91 | kwargs['channel'] = channel 92 | service = self.channels[channel] 93 | kwargs['target'] = service 94 | return kwargs 95 | 96 | 97 | class EntityInterface: 98 | def __init__(self, author_interface, service_interface): 99 | self.author_interface = author_interface 100 | self.service_interface = service_interface 101 | 102 | def get_entities(self, text: str): 103 | result = [] 104 | for service in self.service_interface.services: 105 | for match in re.finditer(service, text): 106 | s = {'start': match.start(), 107 | 'end': match.end(), 108 | 'name': 'service', 109 | 'value': service} 110 | result.append(s) 111 | for channel in self.service_interface.channels.keys(): 112 | for match in re.finditer(channel, text): 113 | c = {'start': match.start(), 114 | 'end': match.end(), 115 | 'name': 'channel', 116 | 'value': channel} 117 | result.append(c) 118 | 119 | for author in self.author_interface.authors.keys(): 120 | for match in re.finditer(author, text): 121 | a = {'start': match.start(), 122 | 'end': match.end(), 123 | 'name': 'author', 124 | 'value': author} 125 | result.append(a) 126 | 127 | return result 128 | -------------------------------------------------------------------------------- /vexbot/adapters/shell/parser.py: -------------------------------------------------------------------------------- 1 | def _is_kwarg(string: str): 2 | if string.startswith('-') or string.startswith('--') and not string == '-': 3 | return True 4 | return False 5 | 6 | 7 | def parse(strings: list): 8 | args = [] 9 | 10 | # Parse args first 11 | for string in strings: 12 | if _is_kwarg(string): 13 | break 14 | args.append(string) 15 | # Remove the args we've parsed 16 | strings = strings[len(args):] 17 | 18 | kwargs = {} 19 | while strings: 20 | string = strings.pop(0) 21 | if string.startswith('--'): 22 | string = string[2:] 23 | elif string.startswith('-'): 24 | string = string[1:] 25 | try: 26 | value = strings[0] 27 | except IndexError: 28 | kwargs[string] = True 29 | break 30 | 31 | if not _is_kwarg(value): 32 | kwargs[string] = strings.pop(0) 33 | else: 34 | kwargs[string] = True 35 | continue 36 | 37 | # Check for list by checking if the next value has `--` or `-` 38 | try: 39 | value = strings[0] 40 | except IndexError: 41 | continue 42 | 43 | while not _is_kwarg(value): 44 | # if we make it here, than we're in a list loop 45 | value = strings.pop(0) 46 | 47 | # try to append the value onto the reference for the string 48 | try: 49 | kwargs[string].append(value) 50 | # if we fail, then it's probably a string and needs to be converted 51 | # to a list 52 | except AttributeError: 53 | kwargs[string] = [kwargs[string], value] 54 | 55 | # Move on to the next one, checking to see if we're at the end 56 | try: 57 | value = strings[0] 58 | except IndexError: 59 | break 60 | 61 | for k, v in dict(kwargs).items(): 62 | if v in ('true', 'True'): 63 | kwargs[k] = True 64 | continue 65 | if v in ('False', 'false'): 66 | kwargs[k] = False 67 | continue 68 | try: 69 | v = int(v) 70 | kwargs[k] = v 71 | continue 72 | except ValueError: 73 | pass 74 | try: 75 | v = float(v) 76 | kwargs[k] = v 77 | # NOTE: End of the line 78 | except ValueError: 79 | continue 80 | 81 | return args, kwargs 82 | -------------------------------------------------------------------------------- /vexbot/adapters/shell/shell.py: -------------------------------------------------------------------------------- 1 | import re 2 | import time 3 | from os import path 4 | import shlex as _shlex 5 | import pprint as _pprint 6 | import logging 7 | from threading import Thread as _Thread 8 | 9 | from prompt_toolkit.shortcuts import PromptSession 10 | from prompt_toolkit.history import FileHistory 11 | from prompt_toolkit.patch_stdout import patch_stdout 12 | from zmq.eventloop.ioloop import PeriodicCallback 13 | 14 | from vexbot.adapters.shell.parser import parse as _parse 15 | from vexbot.adapters.messaging import Messaging as _Messaging 16 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath 17 | from vexbot.adapters.shell.interfaces import (AuthorInterface, 18 | ServiceInterface, 19 | EntityInterface) 20 | 21 | from vexbot.adapters.shell.observers import (PrintObserver, 22 | CommandObserver, 23 | AuthorObserver, 24 | ServiceObserver, 25 | LogObserver, 26 | ServicesObserver) 27 | 28 | from vexbot.adapters.shell.completers import ServiceCompleter, WordCompleter 29 | 30 | 31 | # add in ! and # to our word completion regex 32 | _WORD = re.compile(r'([!#a-zA-Z0-9_]+|[^!#a-zA-Z0-9_\s]+)') 33 | 34 | 35 | def _get_cmd_args_kwargs(text: str) -> (str, tuple, dict): 36 | # consume shebang 37 | text = text[1:] 38 | args = _shlex.split(text) 39 | try: 40 | command = args.pop(0) 41 | except IndexError: 42 | return '', (), {} 43 | args, kwargs = _parse(args) 44 | return command, args, kwargs 45 | 46 | 47 | def _get_default_history_filepath(): 48 | vexdir = get_vexdir_filepath() 49 | return path.join(vexdir, 'vexshell_history') 50 | 51 | 52 | class Shell(PromptSession): 53 | def __init__(self, history_filepath=None): 54 | self._logger = logging.getLogger(__name__) 55 | if history_filepath is None: 56 | self._logger.info(' getting default history filepath.') 57 | history_filepath = _get_default_history_filepath() 58 | 59 | self._logger.info(' history filepath %s', history_filepath) 60 | self.history = FileHistory(history_filepath) 61 | self.messaging = _Messaging('shell', run_control_loop=True) 62 | 63 | # TODO: Cleanup this API access 64 | self.messaging._heartbeat_reciever.identity_callback = self._identity_callback 65 | self._thread = _Thread(target=self.messaging.start, 66 | daemon=True) 67 | 68 | self._bot_status_monitor = PeriodicCallback(self._monitor_bot_state, 69 | 1000) 70 | 71 | self.shebangs = ['!', ] 72 | self.command_observer = CommandObserver(self.messaging, prompt=self) 73 | commands = self.command_observer._get_commands() 74 | self._word_completer = WordCompleter(commands, WORD=_WORD) 75 | self._services_completer = ServiceCompleter(self._word_completer) 76 | self.author_interface = AuthorInterface(self._word_completer, 77 | self.messaging) 78 | 79 | self.service_interface = ServiceInterface(self._word_completer, 80 | self.messaging) 81 | 82 | self.entity_interface = EntityInterface(self.author_interface, 83 | self.service_interface) 84 | 85 | self._bot_status = '' 86 | 87 | super().__init__(message='vexbot: ', 88 | history=self.history, 89 | completer=self._services_completer, 90 | enable_system_prompt=True, 91 | enable_suspend=True, 92 | enable_open_in_editor=True, 93 | complete_while_typing=False) 94 | 95 | 96 | self.print_observer = PrintObserver(self.app) 97 | 98 | self._print_subscription = self.messaging.chatter.subscribe(self.print_observer) 99 | self.messaging.chatter.subscribe(LogObserver()) 100 | self.messaging.command.subscribe(self.command_observer) 101 | self.messaging.command.subscribe(ServicesObserver(self._identity_setter, 102 | self._set_service_completion)) 103 | 104 | def _identity_setter(self, services: list) -> None: 105 | try: 106 | services.remove(self.messaging._service_name) 107 | except ValueError: 108 | pass 109 | 110 | for service in services: 111 | self.service_interface.add_service(service) 112 | self.messaging.send_command('REMOTE', 113 | remote_command='commands', 114 | target=service, 115 | suppress=True) 116 | 117 | def _set_service_completion(self, service: str, commands: list) -> None: 118 | shebang = self.shebangs[0] 119 | commands = [shebang + command for command in commands] 120 | completer = WordCompleter(commands, WORD=_WORD) 121 | self._services_completer.set_service_completer(service, completer) 122 | 123 | # FIXME: `!commands` does not update these as it 124 | if service == 'vexbot': 125 | for command in commands: 126 | self._word_completer.words.add(command) 127 | 128 | def _identity_callback(self): 129 | self.messaging.send_command('services', suppress=True) 130 | self.messaging.send_command('get_commands', suppress=True) 131 | 132 | def _monitor_bot_state(self): 133 | time_now = time.time() 134 | last_message = self.messaging._heartbeat_reciever._last_message_time 135 | # TODO: put in a countdown since last contact? 136 | delta_time = time_now - last_message 137 | # NOTE: Bot is set to send a heartbeat every 1.5 seconds 138 | if time_now - last_message > 3.4: 139 | self._bot_status = '' 140 | else: 141 | if self._bot_status == '': 142 | return 143 | self._bot_status = '' 144 | 145 | self.app.invalidate() 146 | 147 | def is_command(self, text: str) -> bool: 148 | """ 149 | checks for presence of shebang in the first character of the text 150 | """ 151 | if text[0] in self.shebangs: 152 | return True 153 | 154 | return False 155 | 156 | def run(self): 157 | self._thread.start() 158 | self._bot_status_monitor.start() 159 | with patch_stdout(raw=True): 160 | while True: 161 | # Get our text 162 | try: 163 | text = self.prompt(rprompt=self._get_rprompt_tokens) 164 | # KeyboardInterrupt continues 165 | except KeyboardInterrupt: 166 | continue 167 | # End of file returns 168 | except EOFError: 169 | return 170 | 171 | # Clean text 172 | text = text.lstrip() 173 | # Handle empty text 174 | if text == '': 175 | self._logger.debug(' empty string found') 176 | continue 177 | # Program specific handeling. Currently either first word 178 | # or second word can be commands 179 | for string in text.split('&&'): 180 | self._handle_text(string) 181 | 182 | def _get_rprompt_tokens(self): 183 | return self._bot_status 184 | 185 | def _handle_text(self, text: str): 186 | """ 187 | Check to see if text is a command. Otherwise, check to see if the 188 | second word is a command. 189 | 190 | Commands get handeled by the `_handle_command` method 191 | 192 | If not command, check to see if first word is a service or an author. 193 | Default program is to send message replies. However, we will also 194 | check to see if the second word is a command and handle approparitly 195 | 196 | This method does simple string parsing and high level program control 197 | """ 198 | # If first word is command 199 | if self.is_command(text): 200 | self._logger.debug(' first word is a command') 201 | # get the command, args, and kwargs out using `shlex` 202 | command, args, kwargs = _get_cmd_args_kwargs(text) 203 | self._logger.info(' command: %s, %s %s', command, args, kwargs) 204 | # hand off to the `handle_command` method 205 | result = self._handle_command(command, args, kwargs) 206 | 207 | if result: 208 | if isinstance(result, str): 209 | print(result) 210 | else: 211 | _pprint.pprint(result) 212 | # Exit the method here if in this block 213 | return 214 | # Else first word is not a command 215 | else: 216 | self._logger.debug(' first word is not a command') 217 | # get the first word and then the rest of the text. 218 | try: 219 | first_word, second_word = text.split(' ', 1) 220 | self._logger.debug(' first word: %s', first_word) 221 | except ValueError: 222 | self._logger.debug('No second word in chain!') 223 | return self._handle_NLP(text) 224 | # check if second word/string is a command 225 | if self.is_command(second_word): 226 | self._logger.info(' second word is a command') 227 | # get the command, args, and kwargs out using `shlex` 228 | command, args, kwargs = _get_cmd_args_kwargs(second_word) 229 | self._logger.debug(' second word: %s', command) 230 | self._logger.debug(' command %s', command) 231 | self._logger.debug('args %s ', args) 232 | self._logger.debug('kwargs %s', kwargs) 233 | return self._first_word_not_cmd(first_word, command, args, kwargs) 234 | # if second word is not a command, default to NLP 235 | else: 236 | self._logger.info(' defaulting to message since second word ' 237 | 'isn\'t a command') 238 | 239 | return self._handle_NLP(text) 240 | 241 | def _handle_NLP(self, text: str): 242 | entities = self.entity_interface.get_entities(text) 243 | self.messaging.send_command('NLP', text=text, entities=entities) 244 | 245 | def _handle_command(self, command: str, args: tuple, kwargs: dict): 246 | if kwargs.get('remote', False): 247 | self._logger.debug(' `remote` in kwargs, sending command') 248 | self.messaging.send_command(command, *args, **kwargs) 249 | return 250 | if self.command_observer.is_command(command): 251 | self._logger.debug(' `%s` is a command', command) 252 | # try: 253 | return self.command_observer.handle_command(command, *args, **kwargs) 254 | """ 255 | except Exception as e: 256 | self.command_observer.on_error(e, command, args, kwargs) 257 | return 258 | """ 259 | else: 260 | self._logger.debug('command not found! Sending to bot') 261 | self.messaging.send_command(command, *args, **kwargs) 262 | 263 | def _first_word_not_cmd(self, 264 | first_word: str, 265 | command: str, 266 | args: tuple, 267 | kwargs: dict) -> None: 268 | """ 269 | check to see if this is an author or service. 270 | This method does high level control handling 271 | """ 272 | if self.service_interface.is_service(first_word): 273 | self._logger.debug(' first word is a service') 274 | kwargs = self.service_interface.get_metadata(first_word, kwargs) 275 | self._logger.debug(' service transform kwargs: %s', kwargs) 276 | elif self.author_interface.is_author(first_word): 277 | self._logger.debug(' first word is an author') 278 | kwargs = self.author_interface.get_metadata(first_word, kwargs) 279 | self._logger.debug(' author transform kwargs: %s', kwargs) 280 | if not kwargs.get('remote'): 281 | kwargs['remote_command'] = command 282 | command= 'REMOTE' 283 | self.messaging.send_command(command, *args, **kwargs) 284 | return 285 | else: 286 | self.messaging.send_command(command, *args, **kwargs) 287 | -------------------------------------------------------------------------------- /vexbot/adapters/socket_io/__init__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import html 4 | import logging 5 | import pkg_resources 6 | import atexit 7 | from time import sleep 8 | from threading import Thread 9 | 10 | from vexmessage import decode_vex_message 11 | 12 | from vexbot.adapters.messaging import Messaging 13 | from vexbot.adapters.socket_io.observer import SocketObserver 14 | try: 15 | from websocket import WebSocketApp 16 | except ImportError: 17 | logging.error('Socket IO needs `websocket` installed. Please run `pip ' 18 | 'install websocket-client`') 19 | try: 20 | import requests 21 | except ImportError as e: 22 | logging.error('Socket IO needs `requests` installed. Please run `pip ' 23 | 'install requests`') 24 | 25 | 26 | class WebSocket(WebSocketApp): 27 | def __init__(self, 28 | streamer_name: str, 29 | namespace: str, 30 | website_url: str, 31 | service_name: str, 32 | connection: dict=None): 33 | 34 | self.log = logging.getLogger(__name__) 35 | self.log.setLevel(0) 36 | if connection is None: 37 | connection = {} 38 | 39 | self.messaging = Messaging(service_name, run_control_loop=True, **connection) 40 | self._scheduler_thread = Thread(target=self.messaging.start, 41 | daemon=True) 42 | 43 | self._scheduler_thread.start() 44 | self.observer = SocketObserver(self, self.messaging) 45 | self.messaging.command.subscribe(self.observer) 46 | 47 | self._streamer_name = streamer_name 48 | self.namespace = namespace 49 | self._website_url = website_url 50 | self.log.info('Getting Socket IO key!') 51 | self.key, heartbeat = self._connect_to_server_helper() 52 | self.log.info('Socket IO key got!') 53 | # self.command_manager = AdapterCommandManager(self.messaging) 54 | 55 | # alters URL to be more websocket...ie 56 | self._website_socket = self._website_url.replace('http', 'ws') 57 | self._website_socket += 'websocket/' 58 | self.nick = None 59 | super().__init__(self._website_socket + self.key, 60 | on_open=self.on_open, 61 | on_close=self.on_close, 62 | on_message=self.on_message, 63 | on_error=self.on_error) 64 | 65 | def _connect_to_server_helper(self): 66 | r = requests.post(self._website_url) 67 | params = r.text 68 | 69 | # unused variables are connection_timeout and supported_formats 70 | key, heartbeat_timeout, _, _ = params.split(':') 71 | heartbeat_timeout = int(heartbeat_timeout) 72 | return key, heartbeat_timeout 73 | 74 | def on_open(self, *args): 75 | logging.info('Websocket open!') 76 | 77 | def on_close(self, *args): 78 | logging.info('Websocket closed :(') 79 | 80 | def on_message(self, *args): 81 | message = args[1].split(':', 3) 82 | key = int(message[0]) 83 | # namespace = message[2] 84 | 85 | if len(message) >= 4: 86 | data = message[3] 87 | else: 88 | data = '' 89 | if key == 1 and args[1] == '1::': 90 | self.send_packet_helper(1) 91 | elif key == 1 and args[1] == '1::{}'.format(self.namespace): 92 | self.send_packet_helper(5, data={'name': 'initialize'}) 93 | data = {'name': 'join', 94 | 'args': ['{}'.format(self._streamer_name)]} 95 | 96 | self.send_packet_helper(5, data=data) 97 | self.log.info('Connected to channel with socket io!') 98 | # self.messaging.send_status('CONNECTED') 99 | elif key == 2: 100 | self.send_packet_helper(2) 101 | elif key == 5: 102 | data = json.loads(data, ) 103 | if data['name'] == 'message': 104 | message = data['args'][0] 105 | sender = html.unescape(message['sender']) 106 | if sender == self.nick: 107 | return 108 | message = html.unescape(message['text']) 109 | self.messaging.send_chatter(author=sender, message=message) 110 | elif data['name'] == 'join': 111 | self.nick = data['args'][1] 112 | 113 | def on_error(self, *args): 114 | logging.error(args[1]) 115 | 116 | def disconnect(self): 117 | callback = '' 118 | data = '' 119 | # '1::namespace' 120 | self.send(':'.join([str(self.TYPE_KEYS['DISCONNECT']), 121 | callback, self.namespace, data])) 122 | 123 | def send_packet_helper(self, 124 | type_key, 125 | data=None): 126 | 127 | if data is None: 128 | data = '' 129 | else: 130 | data = json.dumps(data) 131 | 132 | # NOTE: callbacks currently not implemented 133 | callback = '' 134 | message = ':'.join([str(type_key), callback, self.namespace, data]) 135 | self.send(message) 136 | -------------------------------------------------------------------------------- /vexbot/adapters/socket_io/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import configparser 3 | 4 | from vexbot.adapters.socket_io import WebSocket 5 | 6 | 7 | def _get_args(): 8 | # FIXME: need to ensure that these values are passed in 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('configuration_file') 11 | return parser.parse_args() 12 | 13 | 14 | def main(): 15 | args = _get_args() 16 | config = configparser.ConfigParser() 17 | config.read(args.configuration_file) 18 | kwargs = config['socket_io'] 19 | client = WebSocket(**kwargs) 20 | client.run_forever() 21 | 22 | 23 | if __name__ == '__main__': 24 | main() 25 | -------------------------------------------------------------------------------- /vexbot/adapters/socket_io/observer.py: -------------------------------------------------------------------------------- 1 | import inspect as _inspect 2 | 3 | from vexmessage import Request 4 | from vexbot.observer import Observer 5 | from vexbot.extensions import develop 6 | from vexbot.extensions import help as vexhelp 7 | 8 | 9 | class SocketObserver(Observer): 10 | extensions = (develop.get_code, develop.get_commands, vexhelp.help) 11 | def __init__(self, bot, messaging): 12 | super().__init__() 13 | self.bot = bot 14 | self.messaging = messaging 15 | self._commands = self._get_commands() 16 | 17 | def _get_commands(self) -> dict: 18 | result = {} 19 | for name, method in _inspect.getmembers(self): 20 | if name.startswith('do_'): 21 | result[name[3:]] = method 22 | 23 | return result 24 | 25 | def do_MSG(self, message, *args, **kwargs): 26 | msg_target = kwargs.get('msg_target') 27 | if msg_target: 28 | message = msg_target + ', ' + message 29 | 30 | data = {} 31 | data['name'] = 'message' 32 | data['args'] = [message, self.bot._streamer_name] 33 | 34 | # FIXME: this API is awful. 35 | self.bot.send_packet_helper(5, data) 36 | 37 | def on_next(self, item: Request): 38 | command = item.command 39 | args = item.args 40 | kwargs = item.kwargs 41 | try: 42 | callback = self._commands[command] 43 | except KeyError: 44 | return 45 | 46 | try: 47 | result = callback(*args, **kwargs) 48 | except Exception as e: 49 | self.on_error(e, command, args) 50 | return 51 | 52 | if result is None: 53 | return 54 | 55 | source = item.source 56 | self.messaging.send_command_response(source, command, result=result, *args, **kwargs) 57 | 58 | def on_error(self, *args, **kwargs): 59 | pass 60 | 61 | def on_completed(self, *args, **kwargs): 62 | pass 63 | -------------------------------------------------------------------------------- /vexbot/adapters/stackoverflow.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from time import sleep 3 | 4 | import selenium 5 | from selenium import webdriver 6 | from selenium.webdriver.chrome.options import Options 7 | from vexbot.adapters.messaging import Messaging 8 | 9 | 10 | class StackOverflow: 11 | def __init__(self, room='https://chat.stackoverflow.com/rooms/6/python'): 12 | options = Options() 13 | options.add_argument('--headless') 14 | self.driver = webdriver.Chrome(chrome_options=options) 15 | self.driver.get(room) 16 | self.messaging = Messaging('stack') 17 | 18 | def run(self): 19 | # start messaging 20 | self.messaging._setup() 21 | # Get the chat block 22 | comment_block = self.driver.find_element_by_id('chat') 23 | # Get the comments from the chat block 24 | comments = comment_block.find_elements_by_xpath('//*[@id="chat"]') 25 | # calculate the number of comments 26 | num_comments = len(comments) 27 | # get the last comment 28 | last_comment = comments[-1] 29 | last_message_length = len(last_comment.find_elements_by_class_name('message')) 30 | while True: 31 | comments = comment_block.find_elements_by_xpath('//*[@id="chat"]') 32 | # Check last one here 33 | messages = last_comment.find_elements_by_class_name('message') 34 | message_length = len(messages) 35 | if message_length > last_message_length: 36 | username = last_comment.find_elements_by_class_name('username')[-1].text 37 | messages = messages[last_message_length - message_length:] 38 | texts = [] 39 | for message in messages: 40 | texts.append(message.find_element_by_class_name('content').text) 41 | text = '\n'.join(texts) 42 | self.messaging.send_chatter('stack', author=username, message=text, channel='stackoverflow') 43 | last_message_length = message_length 44 | 45 | new_num = len(comments) 46 | if new_num > num_comments: 47 | delta = num_comments - new_num 48 | for comment in comments[delta:]: 49 | username = comment.find_element_by_class_name('username').text 50 | messages = comment.find_elements_by_class_name('messages') 51 | texts = [] 52 | for message in comment.find_elements_by_class_name('message'): 53 | texts.append(message.find_element_by_class_name('content').text) 54 | text = '\n'.join(texts) 55 | self.messaging.send_chatter('stack', author=username, message=text, channel='stackoverflow') 56 | # loop maintainence 57 | num_comments = new_num 58 | last_comment = comments[-1] 59 | last_message_length = len(last_comment.find_elements_by_class_name('message')) 60 | else: 61 | sleep(1) 62 | 63 | def __del__(self): 64 | try: 65 | self.driver.close() 66 | self.driver.quit() 67 | except Exception: 68 | pass 69 | 70 | 71 | def main(): 72 | s = StackOverflow() 73 | s.run() 74 | 75 | 76 | if __name__ == '__main__': 77 | main() 78 | -------------------------------------------------------------------------------- /vexbot/adapters/xmpp/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import argparse 4 | # import signal 5 | import atexit 6 | import pkg_resources 7 | from threading import Thread 8 | 9 | import zmq 10 | from vexmessage import decode_vex_message 11 | 12 | from vexbot.adapters.messaging import Messaging # flake8: noqa 13 | 14 | from sleekxmpp import ClientXMPP 15 | from sleekxmpp.exceptions import IqError, IqTimeout 16 | 17 | 18 | class XMPPBot(ClientXMPP): 19 | def __init__(self, 20 | jid: str, 21 | password: str, 22 | room: str, 23 | service_name: str, 24 | bot_nick: str='EchoBot', 25 | connection: dict=None, 26 | **kwargs): 27 | 28 | # Initialize the parent class 29 | super().__init__(jid, password) 30 | if connection is None: 31 | connection = {} 32 | self.messaging = Messaging(service_name, 33 | run_control_loop=True, 34 | **connection) 35 | 36 | self.room = room 37 | self.nick = bot_nick 38 | # self.log = logging.getLogger(__file__) 39 | # self['feature_mechanisms'].unencrypted_plain = True 40 | 41 | # One-shot helper method used to register all the plugins 42 | self._register_plugin_helper() 43 | 44 | self.add_event_handler("session_start", self.session_start) 45 | self.add_event_handler("groupchat_message", self.muc_message) 46 | self._thread = Thread(target=self.messaging.start, daemon=True) 47 | self._thread.start() 48 | 49 | def _register_plugin_helper(self): 50 | """ 51 | One-shot helper method used to register all the plugins 52 | """ 53 | # Service Discovery 54 | self.register_plugin('xep_0030') 55 | # XMPP Ping 56 | self.register_plugin('xep_0199') 57 | # Multiple User Chatroom 58 | self.register_plugin('xep_0045') 59 | 60 | def session_start(self, event): 61 | self.log.info('starting xmpp') 62 | self.send_presence() 63 | self.plugin['xep_0045'].joinMUC(self.room, 64 | self.nick, 65 | wait=True) 66 | 67 | self.get_roster() 68 | 69 | def muc_message(self, msg): 70 | self.messaging.send_chatter(author=msg['mucnick'], 71 | message=msg['body'], 72 | channel=msg['from'].bare) 73 | -------------------------------------------------------------------------------- /vexbot/adapters/xmpp/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import configparser 3 | from vexbot.adapters.xmpp import XMPPBot 4 | import logging 5 | 6 | 7 | def _get_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('configuration_file') 10 | return parser.parse_args() 11 | 12 | 13 | def main(**kwargs) 14 | args = _get_args() 15 | config = configparser.ConfigParser() 16 | config.read(args.configuration_file) 17 | kwargs = config['xmpp'] 18 | # TODO: validate this works like I think it does 19 | connection = config.get('connection', fallback={}) 20 | 21 | jid = '{}@{}/{}'.format(kwargs['local'], kwargs['domain'], kwargs['resource']) 22 | xmpp_bot = XMPPBot(jid, connection=connection, **kwargs) 23 | # logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s %(message)s') 24 | xmpp_bot.connect() 25 | try: 26 | xmpp_bot.process(block=True) 27 | finally: 28 | xmpp_bot.disconnect() 29 | 30 | 31 | if __name__ == '__main__': 32 | main() 33 | -------------------------------------------------------------------------------- /vexbot/adapters/youtube/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import asyncio 4 | import argparse 5 | from argparse import Namespace 6 | import tempfile 7 | import pkg_resources 8 | from time import sleep 9 | 10 | from vexbot.adapters.messaging import Messaging 11 | from vexbot.adapters.scheduler import Scheduler 12 | 13 | import httplib2 14 | 15 | from apiclient.discovery import build 16 | from oauth2client.client import flow_from_clientsecrets 17 | from oauth2client.file import Storage 18 | from oauth2client.tools import run_flow, argparser 19 | 20 | 21 | class Youtube: 22 | def __init__(self, connection: dict=None, **kwargs): 23 | if connection is None: 24 | connection = {} 25 | self.messaging = Messaging('youtube', **connection) 26 | scope = ['https://www.googleapis.com/auth/youtube', 27 | 'https://www.googleapis.com/auth/youtube.force-ssl', 28 | 'https://www.googleapis.com/auth/youtube.readonly'] 29 | self.api = _youtube_authentication(client_secret_filepath, scope) 30 | 31 | def run(self): 32 | self.messaging._setup() 33 | parts = 'snippet' 34 | livestream_response = self.api.liveBroadcasts().list(mine=True, 35 | part=parts, 36 | maxResults=1).execute() 37 | 38 | 39 | live_chat_id = livestream_response.get('items')[0]['snippet']['liveChatId'] 40 | 41 | livechat_response = youtube_api.liveChatMessages().list(liveChatId=live_chat_id, part='snippet').execute() 42 | 43 | next_token = livechat_response.get('nextPageToken') 44 | polling_interval = livechat_response.get('pollingIntervalMillis') 45 | polling_interval = _convert_to_seconds(polling_interval) 46 | while True: 47 | await asyncio.sleep(polling_interval) 48 | part = 'snippet, authorDetails' 49 | livechat_response = self.api.liveChatMessages().list(liveChatId=live_chat_id, part=part, pageToken=next_token).execute() 50 | 51 | next_token = livechat_response.get('nextPageToken') 52 | polling_interval = livechat_response.get('pollingIntervalMillis') 53 | polling_interval = _convert_to_seconds(polling_interval) 54 | 55 | for live_chat_message in livechat_response.get('items'): 56 | snippet = live_chat_message['snippet'] 57 | if not bool(snippet['hasDisplayContent']): 58 | continue 59 | message = snippet['displayMessage'] 60 | author = live_chat_message['authorDetails']['displayName'] 61 | self.messaging.send_chatter(author=author, message=message) 62 | 63 | 64 | async def _run(messaging, live_chat_messages, live_chat_id, ): 65 | command_manager = AdapterCommandManager(messaging) 66 | frame = None 67 | for message in messaging.run(250): 68 | if message.type == 'CMD': 69 | command_manager.parse_commands(message) 70 | elif message.type == 'RSP': 71 | message = message.contents.get('response') 72 | body={'snippet':{'type': 'textmessageEvent', 73 | 'liveChatId': live_chat_id, 74 | 'textMessageDetails': {'messageText': message}}} 75 | 76 | live_chat_messages.insert(part='snippet', 77 | body=body).execute() 78 | 79 | await asyncio.sleep(1) 80 | 81 | 82 | 83 | def _convert_to_seconds(milliseconds: str): 84 | return float(milliseconds)/1000.0 85 | 86 | 87 | 88 | 89 | _YOUTUBE_API_SERVICE_NAME = 'youtube' 90 | _YOUTUBE_API_VERSION = 'v3' 91 | _ALL = "https://www.googleapis.com/auth/youtube" 92 | 93 | 94 | def _youtube_authentication(client_filepath, youtube_scope=_READ_ONLY): 95 | missing_client_message = "You need to populate the client_secrets.json!" 96 | flow = flow_from_clientsecrets(client_filepath, 97 | scope=youtube_scope, 98 | message=missing_client_message) 99 | 100 | dir = os.path.abspath(__file__) 101 | filepath = "{}-oauth2.json".format(dir) 102 | # TODO: Determine if removing file is needed 103 | # remove old oauth files to be safe 104 | if os.path.isfile(filepath): 105 | os.remove(filepath) 106 | 107 | storage = Storage(filepath) 108 | credentials = storage.get() 109 | if credentials is None or credentials.invalid: 110 | args = Namespace(auth_host_name='localhost', 111 | auth_host_port=[8080, 8090], 112 | noauth_local_webserver=False, 113 | logging_level='ERROR') 114 | 115 | credentials = run_flow(flow, storage, args) 116 | return build(_YOUTUBE_API_SERVICE_NAME, 117 | _YOUTUBE_API_VERSION, 118 | http=credentials.authorize(httplib2.Http())) 119 | 120 | 121 | def _get_youtube_link(client_secrets_filepath): 122 | youtube_api = youtube_authentication(client_secrets_filepath) 123 | parts = 'id, snippet, status' 124 | livestream_requests = youtube_api.liveBroadcasts().list(mine=True, 125 | part=parts, 126 | maxResults=5) 127 | 128 | while livestream_requests: 129 | response = livestream_requests.execute() 130 | # TODO: add better parsing here 131 | youtube_id = response.get('items', [])[0]['id'] 132 | return 'http://youtube.com/watch?v={}'.format(youtube_id) 133 | -------------------------------------------------------------------------------- /vexbot/adapters/youtube/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from configparser import ConfigParser 3 | 4 | from vexbot.adapters.youtube import Youtube 5 | 6 | 7 | def _get_kwargs(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('client_secret_filepath') 10 | args = parser.parse_args() 11 | kwargs = vars(args) 12 | 13 | return kwargs 14 | 15 | def main(client_secret_filepath: str): 16 | config_parser = ConfigParser.read(client_secret_filepath) 17 | youtube = Youtube() 18 | youtube.run() 19 | 20 | 21 | if __name__ == '__main__': 22 | kwargs = _get_kwargs() 23 | main(**kwargs) 24 | -------------------------------------------------------------------------------- /vexbot/command.py: -------------------------------------------------------------------------------- 1 | import functools as _functools 2 | 3 | 4 | def command(function=None, 5 | alias: list=None, 6 | hidden: bool=False, 7 | roles: list=None, 8 | utterances: list=None, 9 | short: str=None): 10 | 11 | if function is None: 12 | return _functools.partial(command, 13 | alias=alias, 14 | hidden=hidden, 15 | roles=roles, 16 | utterances=utterances, 17 | short=short) 18 | 19 | # https://stackoverflow.com/questions/10176226/how-to-pass-extra-arguments-to-python-decorator 20 | @_functools.wraps(function) 21 | def wrapper(*args, **kwargs): 22 | return function(*args, **kwargs) 23 | # TODO: check for string and convert to list 24 | if alias is not None: 25 | wrapper.alias = alias 26 | wrapper.hidden = hidden 27 | wrapper.command = True 28 | wrapper.roles = roles 29 | wrapper.utterances = utterances 30 | wrapper.short = short 31 | return wrapper 32 | -------------------------------------------------------------------------------- /vexbot/command_observer.py: -------------------------------------------------------------------------------- 1 | import sys as _sys 2 | import shelve 3 | import logging 4 | import inspect as _inspect 5 | import typing 6 | import importlib 7 | from os import path 8 | 9 | from tblib import Traceback 10 | 11 | from vexmessage import Request 12 | from vexbot.observer import Observer 13 | from vexbot.messaging import Messaging 14 | from vexbot.intents import intent 15 | from vexbot.command import command 16 | from vexbot.extensions import help as vexhelp 17 | from vexbot.extensions import (log, 18 | admin, 19 | extensions) 20 | 21 | from vexbot.util.get_cache_filepath import get_cache 22 | from vexbot.util.get_cache_filepath import get_cache_filepath as get_cache_dir 23 | from vexbot.util.create_cache_filepath import create_cache_directory 24 | 25 | 26 | class CommandObserver(Observer): 27 | extensions = (admin.get_commands, 28 | admin.get_disabled, 29 | admin.disable, 30 | vexhelp.help, 31 | extensions.add_extensions, 32 | extensions.get_extensions, 33 | extensions.remove_extension, 34 | extensions.get_installed_extensions) 35 | 36 | def __init__(self, 37 | bot, 38 | messaging: Messaging, 39 | subprocess_manager: 'vexbot.subprocess_manager.SubprocessManager', 40 | language): 41 | 42 | super().__init__() 43 | self.bot = bot 44 | self.messaging = messaging 45 | self.subprocess_manager = subprocess_manager 46 | self.language = language 47 | 48 | cache_dir = get_cache_dir() 49 | mkdir = not path.isdir(cache_dir) 50 | if mkdir: 51 | os.makedirs(cache_dir, exist_ok=True) 52 | 53 | filepath = get_cache(__name__ + '.pickle') 54 | init = not path.isfile(filepath) 55 | 56 | self._config = shelve.open(filepath, writeback=True) 57 | 58 | if init: 59 | self._config['extensions'] = {} 60 | self._config['disabled'] = {} 61 | self._config['modules'] = {} 62 | 63 | self._commands = {} 64 | self.init_commands() 65 | self._intents = self._get_intents() 66 | for key, values in self._config['extensions'].items(): 67 | self.add_extensions(key, **values) 68 | self.logger = logging.getLogger(self.messaging._service_name + '.observers.command') 69 | 70 | self.root_logger = logging.getLogger() 71 | # self.root_logger.setLevel(logging.DEBUG) 72 | # logging.basicConfig() 73 | self.root_logger.addHandler(self.messaging.pub_handler) 74 | 75 | def _get_intents(self) -> dict: 76 | result = {} 77 | # FIXME: define a `None` callback 78 | # result[None] = self.do_commands 79 | for name, method in _inspect.getmembers(self): 80 | is_intent = getattr(method, '_vex_intent', False) 81 | if name.startswith('do_') or is_intent: 82 | try: 83 | name = method._vex_intent_name 84 | except AttributeError: 85 | name = name[3:] 86 | result[name] = method 87 | 88 | return result 89 | 90 | def init_commands(self): 91 | for key, value in self._config['extensions'].items(): 92 | self.add_extensions(key, **value, update=False) 93 | self.update_commands() 94 | # FIXME: not handeling the conflict 95 | 96 | def update_commands(self): 97 | for name, method in _inspect.getmembers(self): 98 | if name.startswith('do_'): 99 | self._commands[name[3:]] = method 100 | elif getattr(method, 'command', False): 101 | self._commands[name] = method 102 | else: 103 | continue 104 | 105 | if getattr(method, 'alias', False): 106 | for alias in method.alias: 107 | self._commands[alias] = method 108 | 109 | def _get_commands(self) -> dict: 110 | result = {} 111 | for name, method in _inspect.getmembers(self): 112 | if name.startswith('do_'): 113 | result[name[3:]] = method 114 | elif getattr(method, 'command', False): 115 | result[name] = method 116 | else: 117 | continue 118 | 119 | if getattr(method, 'alias', False): 120 | for alias in method.alias: 121 | result[alias] = method 122 | return result 123 | 124 | @command(alias=['MSG',], roles=['admin',]) 125 | def do_REMOTE(self, 126 | target: str, 127 | remote_command: str, 128 | source: list, 129 | *args, 130 | **kwargs) -> None: 131 | """ 132 | Send a remote command to a service. Used 133 | 134 | Args: 135 | target: The service that the command gets set to 136 | remote_command: The command to do remotely. 137 | source: the binary source of the zmq_socket. Packed to send to the 138 | """ 139 | if target == self.messaging._service_name: 140 | info = 'target for remote command is the bot itself! Returning the function' 141 | self.logger.info(info) 142 | return self._handle_command(remote_command, source, *args, **kwargs) 143 | 144 | try: 145 | target = self.messaging._address_map[target] 146 | except KeyError: 147 | warn = ' Target %s, not found in addresses. Are you sure that %s sent an IDENT message?' 148 | self.logger.warn(warn, target, target) 149 | # TODO: raise an error instead of returning? 150 | # NOTE: Bail here since there's no point in going forward 151 | return 152 | 153 | self.logger.info(' REMOTE %s, target: %s | %s, %s', 154 | remote_command, target, args, kwargs) 155 | 156 | # package the binary together 157 | source = target + source 158 | self.messaging.send_command_response(source, 159 | remote_command, 160 | *args, 161 | **kwargs) 162 | 163 | @command(roles=['admin',]) 164 | def do_NLP(self, *args, **kwargs): 165 | # FIXME: Do entity extraction here 166 | intent, confidence, entities = self.language.get_intent(*args, **kwargs) 167 | self.logger.debug('intent from do_NLP: %s', intent) 168 | 169 | # FIXME: `None` returns `do_command` 170 | try: 171 | callback = self._intents[intent] 172 | except NameError: 173 | self.logger.debug('intent not found! intent: %s', intent) 174 | # TODO: send intent back to source 175 | return 176 | 177 | # FIXME:Pass entites in as kwargs? 178 | result = callback(*args, entities=entities, **kwargs) 179 | return result 180 | 181 | @command(roles=['admin',]) 182 | def do_TRAIN_INTENT(self, *args, **kwargs): 183 | self.logger.debug('starting training!') 184 | intents = self.bot.intents.get_intents() 185 | for k, v in intents.items(): 186 | intents[k] = v() 187 | # self.logger.debug('intents: %s', intents) 188 | self.language.train_classifier(intents) 189 | self.logger.debug('training finished') 190 | 191 | @command(roles=['admin',]) 192 | def do_IDENT(self, service_name: str, source: list, *args, **kwargs) -> None: 193 | """ 194 | Perform identification of a service to a binary representation. 195 | 196 | Args: 197 | service_name: human readable name for service 198 | source: zmq representation for the socket source 199 | """ 200 | self.logger.info(' IDENT %s as %s', service_name, source) 201 | self.messaging._address_map[service_name] = source 202 | 203 | @command(roles=['admin',]) 204 | def do_AUTH_FOR_SUBSCRIPTION_SOCKET(self): 205 | self.messaging.subscription_socket.close() 206 | self.messaging._setup_subscription_socket(True) 207 | 208 | @command(roles=['admin',]) 209 | def do_NO_AUTH_FOR_SUBSCRIPTION_SOCKET(self): 210 | self.messaging.subscription_socket.close() 211 | self.messaging._setup_subscription_socket(False) 212 | 213 | @command(roles=['admin',]) 214 | @intent(name='get_services') 215 | def do_services(self, *args, **kwargs) -> tuple: 216 | return tuple(self.messaging._address_map.keys()) 217 | 218 | @command(alias=['get_last_error',], roles=['admin']) 219 | @intent(name='get_last_error') 220 | def do_show_last_error(self, *args, **kwargs): 221 | exc_info = _sys.exc_info() 222 | if exc_info[2] is None: 223 | return 'No problems here boss!' 224 | self.logger.debug('exc_info: %s', exc_info) 225 | return Traceback(exc_info[2]).to_dict() 226 | 227 | def _handle_result(self, 228 | command: str, 229 | source: list, 230 | result, 231 | *args, 232 | **kwargs) -> None: 233 | 234 | self.logger.info('send response %s %s %s', source, command, result) 235 | self.messaging.send_command_response(source, command, result=result, *args, **kwargs) 236 | 237 | def _handle_command(self, 238 | command: str, 239 | source: list, 240 | *args, 241 | **kwargs) -> None: 242 | 243 | try: 244 | callback = self._commands[command] 245 | except KeyError: 246 | self.logger.info(' command not found! %s', command) 247 | return 248 | 249 | self.logger.debug(' handle_command kwargs: %s', kwargs) 250 | kwargs['source'] = source 251 | try: 252 | result = callback(*args, **kwargs) 253 | except Exception as e: 254 | self.on_error(e, command, *args, **kwargs) 255 | return 256 | kwargs.pop('source') 257 | 258 | if result is None: 259 | self.logger.debug(' No result from callback on command: %s', command) 260 | else: 261 | service = self.messaging._service_name 262 | self._handle_result(command, source, result, service=service, *args, **kwargs) 263 | 264 | def on_next(self, item: Request) -> None: 265 | command = item.command 266 | args = item.args 267 | kwargs = item.kwargs 268 | source = item.source 269 | self.logger.info(' Request recieved, %s %s %s %s', 270 | command, source, args, kwargs) 271 | 272 | if not kwargs.get('result'): 273 | self._handle_command(command, source, *args, **kwargs) 274 | else: 275 | # TODO: Verify that this makes sense 276 | # NOTE: Stripping the first address on here as it should be the bot address 277 | source = source[1:] 278 | self._handle_result(command, 279 | source, 280 | kwargs.pop('result'), 281 | *args, 282 | **kwargs) 283 | 284 | def on_error(self, error: Exception, command, *args, **kwargs): 285 | self.logger.warn('on_error called for %s %s %s', command, args, kwargs) 286 | self.logger.warn('on_error called %s:%s', error.__class__.__name__, error.args) 287 | self.logger.exception(' on_next error for command {}'.format(command)) 288 | 289 | def on_completed(self, *args, **kwargs): 290 | self.logger.info(' command observer completed!') 291 | 292 | @command(alias=['quit', 'exit'], roles=['admin',]) 293 | @intent(name='exit') 294 | def do_kill_bot(self, *args, **kwargs): 295 | """ 296 | Kills the instance of vexbot 297 | """ 298 | self.logger.warn(' Exiting bot!') 299 | _sys.exit() 300 | -------------------------------------------------------------------------------- /vexbot/dynamic_entities.py: -------------------------------------------------------------------------------- 1 | # NOTE: Unimplemented 2 | 3 | 4 | def pass(): 5 | levels = ('debug', 'info', 'warning', 'warn', 'error', 'critical', 6 | 'DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL') 7 | examples = ('set logging to {}', 8 | 'set log to {}', 9 | 'logging to {}') 10 | 11 | class BotIntents: 12 | def __init__(self): 13 | self._intents = self.get_intents() 14 | 15 | def get_intents(self) -> dict: 16 | result = {} 17 | for name, method in _inspect.getmembers(self): 18 | isintent = getattr(method, '_vex_intent', False) 19 | 20 | if name.startswith('do_') or isintent: 21 | if isintent: 22 | name = method._vex_intent_name 23 | else: 24 | name = name[3:] 25 | 26 | result[name] = method 27 | 28 | return result 29 | 30 | def get_intent_names(self, *args, **kwargs) -> tuple: 31 | if not args: 32 | return tuple(self._intents.keys()) 33 | results = {} 34 | for arg in args: 35 | # FIXME: I hate this API 36 | intent_values = self._intents.get(arg) 37 | if intent_values is not None: 38 | intent_values = intent_values() 39 | if intent_values is None: 40 | intent_values = () 41 | results[arg] = intent_values 42 | return results 43 | 44 | def do_set_log(self) -> tuple: 45 | levels = ('debug', 'info', 'warning', 'warn', 'error', 'critical', 46 | 'DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL') 47 | examples = ('set logging to {}', 48 | 'set log to {}', 49 | 'logging to {}') 50 | 51 | # FIXME: finish 52 | values = () 53 | return values 54 | 55 | def do_get_code(self) -> tuple: 56 | values = ('get code for that commands', 57 | 'get code for last', 58 | 'get code for {}', 59 | 'get source for {}', 60 | 'show me the code', 61 | 'show code', 62 | 'display code') 63 | 64 | def do_get_commands(self) -> tuple: 65 | pass 66 | 67 | def do_restart_program(self) -> tuple: 68 | pass 69 | 70 | def do_start_program(self) -> tuple: 71 | pass 72 | 73 | def do_stop(self) -> tuple: 74 | pass 75 | 76 | def do_status(self) -> tuple: 77 | pass 78 | 79 | 80 | def do_trace(self) -> tuple: 81 | values = ('run that back for me', 82 | 'trace last command', 83 | 'trace command', 84 | 'trace') 85 | 86 | return values 87 | 88 | def do_lint(self) -> tuple: 89 | values = ('lint file', 90 | 'flake file') 91 | 92 | return values 93 | 94 | 95 | -------------------------------------------------------------------------------- /vexbot/entity_extraction.py: -------------------------------------------------------------------------------- 1 | import sklearn_crfsuite 2 | import spacy 3 | 4 | 5 | class EntityExtraction: 6 | def __init__(self, language_model=None): 7 | values = {'algorithm': 'lbfgs', 8 | # coefficient for L1 penalty 9 | 'c1': 1, 10 | # coefficient for L2 penalty 11 | 'c2': 1e-3, 12 | 'max_iterations': 50, 13 | # include transitions that are possible, but not observed 14 | 'all_possible_transitions': True} 15 | 16 | self.ent_tagger = sklearn_crfsuite.CRF(**values) 17 | self._language_model = language_model 18 | 19 | def train(self, data): 20 | # _sentence_to_features 21 | x_train = [] 22 | # _sentence_to_labels 23 | y_train = [] 24 | 25 | self.ent_tagger.fit(x_train, y_train) 26 | 27 | def get_entites(self, text: str, entities: list=None, *args, **kwargs): 28 | spacy_entities = self.get_spacy_entites(text) 29 | # TODO: Implement 30 | duckling_entities = self.get_duckling_entities(text) 31 | custom_entities = self.get_custom_entities(text) 32 | 33 | # FIXME 34 | return spacy_entities 35 | 36 | def get_custom_entities(self, text: str) -> list: 37 | """ 38 | Custom, Domain-specific entities 39 | Uses conditional random field 40 | Needs to be trained 41 | """ 42 | # NOTE: MITIE uses structured SVM 43 | pass 44 | 45 | def get_duckling_entities(self, text: str) -> list: 46 | """ 47 | Pretrained (duckling) 48 | Uses context-free grammer 49 | Dates, Amounts of Money, Druations, Distances, Ordinals 50 | """ 51 | pass 52 | 53 | def get_spacy_entites(self, text: str) -> list: 54 | """ 55 | Pretrained (spaCy) 56 | Uses averaged preceptron 57 | Places, dates, people, organizations 58 | """ 59 | # NOTE: spaCy doc 60 | doc = self._language_model(text) 61 | 62 | entities = [{"entity": ent.label_, 63 | "value": ent.text, 64 | "start": ent.start_char, 65 | "end": ent.end_char} for ent in doc.ents] 66 | 67 | return entities 68 | 69 | # NOTE: used for conditional random field training 70 | def _sentence_to_features(self, sentence: list): 71 | # type: (List[Tuple[Text, Text, Text, Text]]) -> List[Dict[Text, Any]] 72 | """Convert a word into discrete features in self.crf_features, 73 | including word before and word after.""" 74 | 75 | sentence_features = [] 76 | prefixes = ('-1', '0', '+1') 77 | 78 | for word_idx in range(len(sentence)): 79 | # word before(-1), current word(0), next word(+1) 80 | word_features = {} 81 | for i in range(3): 82 | if word_idx == len(sentence) - 1 and i == 2: 83 | word_features['EOS'] = True 84 | # End Of Sentence 85 | elif word_idx == 0 and i == 0: 86 | word_features['BOS'] = True 87 | # Beginning Of Sentence 88 | else: 89 | word = sentence[word_idx - 1 + i] 90 | prefix = prefixes[i] 91 | features = self.crf_features[i] 92 | for feature in features: 93 | # append each feature to a feature vector 94 | value = self.function_dict[feature](word) 95 | word_features[prefix + ":" + feature] = value 96 | sentence_features.append(word_features) 97 | return sentence_features 98 | 99 | # NOTE: used for conditional random field training 100 | def _sentence_to_labels(self, sentence): 101 | # type: (List[Tuple[Text, Text, Text, Text]]) -> List[Text] 102 | 103 | return [label for _, _, label, _ in sentence] 104 | -------------------------------------------------------------------------------- /vexbot/extension_metadata.py: -------------------------------------------------------------------------------- 1 | extensions = {'pip_install': {'path': 'vexbot.extensions.admin:install', 2 | 'short': 'Install packages using pip',}, 3 | 'pip_uninstall': {'path': 'vexbot.extensions.admin:uninstall', 4 | 'short': 'Uninstall packages using pip'}, 5 | 'pip_update': {'path': 'vexbot.extensions.admin:update', 6 | 'short': 'Update packages using pip'}, 7 | 'get_commands': {'path': 'vexbot.extensions.admin:get_commands', 8 | 'short': 'Get commands from command observer'}, 9 | 'get_command_modules': {'path': 'vexbot.extensions.admin:get_command_modules', 10 | 'short': 'Get all the module names for the commands that are available'}, 11 | 'get_disabled': {'path': 'vexbot.extensions.admin:get_disabled', 12 | 'short': 'Get all disabled commands from command observer'}, 13 | 'disable': {'path': 'vexbot.extensions.admin:disable', 14 | 'short': 'Remove a command from the commands'}, 15 | 'get_cache': {'path': 'vexbot.extensions.admin:get_cache', 16 | 'short': 'Get the values of the configuration cache'}, 17 | 'delete_cache': {'path': 'vexbot.extensions.admin:delete_cache', 18 | 'short': 'Remove the values from the configuration cache'}, 19 | 'get_code': {'path': 'vexbot.extensions.develop:get_code', 20 | 'short': 'Get the source code from a method using the inspect module'}, 21 | 'get_method_names': {'path': 'vexbot.extensions.develop:get_members', 22 | 'short': 'Get all the method names from a class using inspect module'}, 23 | 'get_size': {'path': 'vexbot.extensions.digitalocean:get_size', 24 | 'short': 'Get the size of a digital ocean droplet', 25 | 'extras': ['digitalocean',]}, 26 | 'power_off': {'path': 'vexbot.extensions.digitalocean:power_off', 27 | 'short': 'Power off a digital ocean droplet', 28 | 'extras': ['digitalocean',]}, 29 | 'power_on': {'path': 'vexbot.extensions.digitalocean:power_on', 30 | 'short': 'Power on a digital ocean droplet', 31 | 'extras': ['digitalocean',]}, 32 | 'log_level': {'path': 'vexbot.extensions.log:log_level', 33 | 'short': 'Set or get the root logger to a level'}, 34 | 'set_debug': {'path': 'vexbot.extensions.log:set_debug', 35 | 'short': 'Set the root logger to `logging.DEBUG'}, 36 | 'set_info': {'path': 'vexbot.extensions.log:set_info', 37 | 'short': 'Set the root logger to `logging.INFO'}, 38 | 'set_default': {'path': 'vexbot.extensions.log:set_default', 39 | 'short': 'Set the root logger to `logging.WARNING'}, 40 | 'filter_logs': {'path': 'vexbot.extensions.log:filter_logs', 41 | 'short': 'Filter logs'}, 42 | 'anti_filter': {'path': 'vexbot.extensions.log:anti_filter', 43 | 'short': 'Filter anything that is not this'}, 44 | 'get_all_droplets': {'path': 'vexbot.extensions.digitalocean:get_all_droplets', 45 | 'short': 'Get all droplets from digital ocean', 46 | 'extras': ['digitalocean',]}, 47 | 'add_extensions': {'path': 'vexbot.extensions.extensions:add_extensions', 48 | 'short': 'Add an extension to a command observer'}, 49 | 'get_extensions': {'path': 'vexbot.extensions.extensions:get_extensions', 50 | 'short': 'Get all the extensions from an observer instance'}, 51 | 'remove_extension': {'path': 'vexbot.extensions.extensions:remove_extension', 52 | 'short': 'Remove an extension from an observer instance'}, 53 | 'get_installed_extensions': {'path': 'vexbot.extensions.extensions:get_installed_extensions', 54 | 'short': 'See all installed extensions for Vexbot'}, 55 | 'get_installed_modules': {'path': 'vexbot.extensions.extensions:get_installed_modules', 56 | 'short': 'See all installed modules for Vexbot'}, 57 | 'add_extensions_from_dict': {'path': 'vexbot.extensions.extensions:add_extensions_from_dict', 58 | 'short': 'Used to intialize command observers efficient'}, 59 | 'help': {'path': 'vexbot.extensions.help:help', 60 | 'short': 'Get the help from a method'}, 61 | 'hidden': {'path': 'vexbot.extensions.hidden:hidden', 62 | 'short': 'Get all hidden method methods'}, 63 | 'bot_intents': {'path': 'vexbot.extensions.intents:get_intents', 64 | 'short': 'Get the intents from the bot'}, 65 | 'bot_intent': {'path': 'vexbot.extensions.intents:get_intent'}, 66 | 'get_google_trends': {'path': 'vexbot.extensions.news:get_hot_trends', 67 | 'short': 'Get the google trends', 68 | 'extras': ['summarization']}, 69 | 'get_news_urls': {'path': 'vexbot.extensions.news:get_popular_urls', 70 | 'extras': ['summarization']}, 71 | 'summarize_news_url': {'path': 'vexbot.extensions.news:summarize_article', 72 | 'extras': ['summarization']}, 73 | 'start_process': {'path': 'vexbot.extensions.subprocess:start', 74 | 'extras': ['process_manager'], 75 | 'short': 'Start a process'}, 76 | 'stop_process': {'path': 'vexbot.extensions.subprocess:stop', 77 | 'extras': ['process_manager'], 78 | 'short': "Stop a process"}, 79 | 'restart_process': {'path': 'vexbot.extensions.subprocess:restart', 80 | 'extras': ['process_manager'], 81 | 'short': "Restart a process"}, 82 | 'status_process': {'path': 'vexbot.extensions.subprocess:status', 83 | 'extras': ['process_manager'], 84 | 'short': 'Get the status of a process'}, 85 | 'process_uptime': {'path': 'vexbot.extensions.subprocess:uptime', 86 | 'extras': ['process_manager'], 87 | 'short': 'Get the uptime of a process'}, 88 | 'cpu_count': {'path': 'vexbot.extensions.system:cpu_count', 89 | 'short': 'Show the total number of CPU\'s', 90 | 'extras': ['system']}, 91 | 'cpu_frequency': {'path': 'vexbot.extensions.system:cpu_frequency', 92 | 'extras': ['system']}, 93 | 'virtual_memory_percent': {'path': 'vexbot.extensions.system:virtual_memory_percent', 94 | 'extras': ['system']}, 95 | 'virtual_memory_total': {'path': 'vexbot.extensions.system:virtual_memory_total', 96 | 'short': 'Show the total amoun of virtual memory available in Mb\'s', 97 | 'extras': ['system']}, 98 | 'virtual_memory_used': {'path': 'vexbot.extensions.system:virtual_memory_used', 99 | 'short': 'Show the total amount of virtual memory in use in MBs', 100 | 'extras': ['system']}, 101 | 'swap': {'path': 'vexbot.extensions.system:swap', 102 | 'short': 'Show the total amount of swap used in MB', 103 | 'extras': ['system']}} 104 | -------------------------------------------------------------------------------- /vexbot/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | from vexbot.extension_metadata import extensions as _ext_meta 2 | 3 | def extension(base, 4 | alias: list=None, 5 | name: str=None, 6 | hidden: bool=False, 7 | instancemethod: bool=False, 8 | roles: list=None, 9 | short: str=None): 10 | 11 | def wrapper(function): 12 | # Try to use the meta first 13 | if short is None and hasattr(function, '_meta'): 14 | new_short = _ext_meta.get(function._meta, {}).get('short') 15 | elif short: 16 | new_short = short 17 | else: 18 | new_short = None 19 | # Then try to use the function name 20 | if short is None and new_short is None: 21 | new_short = _ext_meta.get(function.__name__, {}).get('short') 22 | 23 | new_name = name or function.__name__ 24 | function.command = True 25 | function.hidden = hidden 26 | function.roles = roles 27 | function.alias = alias 28 | function.short = new_short 29 | 30 | 31 | if instancemethod: 32 | # FIXME: implement 33 | if hasattr(base, '_commands'): 34 | pass 35 | else: 36 | pass 37 | else: 38 | setattr(base, new_name, function) 39 | 40 | return function 41 | 42 | return wrapper 43 | 44 | 45 | def extend(base, 46 | function, 47 | alias: list=None, 48 | name: str=None, 49 | hidden: bool=False, 50 | instancemethod: bool=False, 51 | roles: list=None, 52 | short: str=None): 53 | 54 | wrapper = extension(base, alias, name, hidden, instancemethod, roles, short) 55 | wrapper(function) 56 | 57 | 58 | def extendmany(base, 59 | *functions, 60 | hidden: bool=None, 61 | instancemethod: bool=False, 62 | roles: list=None): 63 | for function in functions: 64 | wrapper = extension(base, 65 | hidden=hidden, 66 | instancemethod=instancemethod, 67 | roles=roles) 68 | 69 | wrapper(function) 70 | -------------------------------------------------------------------------------- /vexbot/extensions/admin.py: -------------------------------------------------------------------------------- 1 | import pip 2 | import inspect 3 | 4 | 5 | def install(self, *args, **kwargs): 6 | pip.main(['install', *args]) 7 | 8 | 9 | def uninstall(self, *args, **kwargs): 10 | pip.main(['uninstall', *args]) 11 | 12 | 13 | def update(self, *args, **kwargs): 14 | pip.main(['install', '--upgrade', *args]) 15 | 16 | 17 | def get_commands(self, *args, **kwargs): 18 | # TODO: add in a short summary 19 | # TODO: also end in some entity parsing? or getting of the args and 20 | # kwargs 21 | commands = self._commands.keys() 22 | return sorted(commands, key=str.lower) 23 | 24 | 25 | def get_disabled(self, *args, **kwargs): 26 | commands = self._config['disabled'].keys() 27 | return sorted(commands, key=str.lower) 28 | 29 | 30 | def disable(self, name: str, *args, **kwargs): 31 | self._commands.pop(name) 32 | 33 | 34 | def get_command_modules(self, *args, **kwargs): 35 | result = set() 36 | for cb in self._commands.values(): 37 | module = inspect.getmodule(cb) 38 | result.add(module.__name__) 39 | return result 40 | 41 | 42 | def get_cache(self, *args, **kwargs): 43 | return dict(self._config) 44 | 45 | 46 | def delete_cache(self, *args, **kwargs): 47 | for key in list(self._config.keys()): 48 | self._config.pop(key) 49 | self._config.sync() 50 | 51 | return ('Cache cleared. Please restart service to remove traces of old ' 52 | 'cache.') 53 | -------------------------------------------------------------------------------- /vexbot/extensions/develop.py: -------------------------------------------------------------------------------- 1 | import inspect as _inspect 2 | 3 | 4 | def get_code(self, *args, **kwargs): 5 | """ 6 | get the python source code from callback 7 | """ 8 | # FIXME: Honestly should allow multiple commands 9 | callback = self._commands[args[0]] 10 | # TODO: syntax color would be nice 11 | source = _inspect.getsourcelines(callback)[0] 12 | """ 13 | source_len = len(source) 14 | source = PygmentsLexer(CythonLexer).lex_document(source)() 15 | """ 16 | # FIXME: formatting sucks 17 | return "\n" + "".join(source) 18 | 19 | 20 | def get_members(self, *args, **kwargs): 21 | return [x[0] for x in _inspect.getmembers(self)] 22 | -------------------------------------------------------------------------------- /vexbot/extensions/digitalocean.py: -------------------------------------------------------------------------------- 1 | import os 2 | import digitalocean 3 | 4 | 5 | _MANAGER = False 6 | _DEFAULT_DROPLET = os.getenv('DIGITAL_OCEAN_DROPLET_ID') 7 | _DIGITAL_OCEAN_KEY = os.getenv('DIGITAL_OCEAN_KEY') 8 | if _DIGITAL_OCEAN_KEY: 9 | _MANAGER = digitalocean.Manager(token=_DIGITAL_OCEAN_KEY) 10 | 11 | 12 | def get_size(*args, **kwargs): 13 | id_ = kwargs.get('id', _DEFAULT_DROPLET) 14 | if id_ is None: 15 | # FIXME: implement a way to pass this in as a config 16 | raise RuntimeError('Must pass in an `id_` or set the `DITIAL_OCEAN_DROPLET_ID` in your envirnoment') 17 | droplet = _MANAGER.get_droplet(id_) 18 | return droplet.size 19 | 20 | 21 | def power_off(*args, **kwargs): 22 | id_ = kwargs.get('id', _DEFAULT_DROPLET) 23 | if id_ is None: 24 | # FIXME: implement a way to pass this in as a config 25 | raise RuntimeError('Must pass in an `id_` or set the `DITIAL_OCEAN_DROPLET_ID` in your envirnoment') 26 | droplet = _MANAGER.get_droplet(id_) 27 | droplet.power_off() 28 | 29 | 30 | def power_on(*args, **kwargs): 31 | id_ = kwargs.get('id', _DEFAULT_DROPLET) 32 | if id_ is None: 33 | # FIXME: implement a way to pass this in as a config 34 | raise RuntimeError('Must pass in an `id_` or set the `DITIAL_OCEAN_DROPLET_ID` in your envirnoment') 35 | droplet = _MANAGER.get_droplet(id_) 36 | droplet.power_on() 37 | 38 | 39 | def get_all_droplets(*args, **kwargs): 40 | droplets = _MANAGER.get_all_droplets() 41 | return ['{}: {}'.format(x.name, x.id) for x in droplets] 42 | -------------------------------------------------------------------------------- /vexbot/extensions/dynamic_loading.py: -------------------------------------------------------------------------------- 1 | def get_dynamic_extensions(self, *args, **kwargs): 2 | pass 3 | -------------------------------------------------------------------------------- /vexbot/extensions/extensions.py: -------------------------------------------------------------------------------- 1 | import pip 2 | import pkg_resources 3 | from vexbot.extensions import extend as _extend 4 | from vexbot.extension_metadata import extensions as _meta_data 5 | 6 | 7 | def get_installed_extensions(self, *args, **kwargs): 8 | verify_requirements = True 9 | name = 'vexbot_extensions' 10 | if not args: 11 | extensions = ['{}: {}'.format(x.name, _meta_data[x.name].get('short', 'No Documentation')) for x in pkg_resources.iter_entry_points(name)] 12 | else: 13 | extensions = ['{}: {}'.format(x.name, _meta_data[x.name].get('short', 'No Documentation')) for x in pkg_resources.iter_entry_points(name) if x.module_name in args] 14 | return sorted(extensions, key=str.lower) 15 | 16 | 17 | def get_installed_modules(self, *args, **kwargs): 18 | result = set() 19 | [result.add(x.module_name) for x in pkg_resources.iter_entry_points('vexbot_extensions')] 20 | return result 21 | 22 | 23 | def _install(package) -> pkg_resources.Distribution: 24 | pip.main(['install', package.project_name]) 25 | pkg_resources._initialize_master_working_set() 26 | return pkg_resources.get_distribution(package.project_name) 27 | 28 | 29 | def add_extensions(self, 30 | *args, 31 | alias: list=None, 32 | call_name=None, 33 | hidden: bool=False, 34 | update: bool=True, 35 | **kwargs): 36 | 37 | # FIXME: implement 38 | not_found = set() 39 | # NOTE: The dist should be used to figure out which name we want, not by grabbing blindly 40 | entry_points = [x for x in pkg_resources.iter_entry_points('vexbot_extensions') if x.name in args] 41 | for entry in entry_points: 42 | entry.require(installer=_install) 43 | function = entry.resolve() 44 | values = {'alias': alias, 'call_name': call_name, 'hidden': hidden, 'kwargs': kwargs} 45 | self._config['extensions'][entry.name] = values 46 | self.extend(function, alias=alias, name=call_name, hidden=hidden, update=False) 47 | self._config.sync() 48 | update_method = getattr(self, 'update_commands') 49 | if update and update_method: 50 | update_method() 51 | if not_found: 52 | raise RuntimeError('Packages not found/installed: {}'.format(not_found)) 53 | 54 | 55 | def add_extensions_from_dict(self, extensions: dict): 56 | keys = tuple(extensions.keys()) 57 | entry_points = [x for x in pkg_resources.iter_entry_points('vexbot_extensions') if x.name in keys] 58 | for entry in entry_points: 59 | entry.require(installer=_install) 60 | function = entry.resolve() 61 | values = dict(extensions[entry.name]) 62 | kwargs = values.pop('kwargs', {}) 63 | values.update(kwargs) 64 | if values.get('short') is None: 65 | meta = _meta_data.get(entry.name) 66 | if meta is None: 67 | values['short'] = _meta_data.get(getattr(function, '_meta', 'No Documentation')) 68 | else: 69 | values['short'] = meta.get('short', 'No Documentation') 70 | 71 | name = values.pop('call_name', function.__name__) 72 | _extend(self.__class__, function, **values) 73 | 74 | 75 | def get_extensions(self, *args, values: bool=False, **kwargs): 76 | if values: 77 | return self._config['extensions'] 78 | return tuple(self._config['extensions'].keys()) 79 | 80 | 81 | def remove_extension(self, *args, **kwargs) -> str: 82 | for arg in args: 83 | popped = self._config['extensions'].pop(arg, None) 84 | -------------------------------------------------------------------------------- /vexbot/extensions/help.py: -------------------------------------------------------------------------------- 1 | def help(self, *arg, **kwargs): 2 | """ 3 | Help helps you figure out what commands do. 4 | Example usage: !help code 5 | To see all commands: !commands 6 | """ 7 | name = arg[0] 8 | try: 9 | callback = self._commands[name] 10 | except KeyError: 11 | self._logger.info(' !help not found for: %s', name) 12 | return self.do_help.__doc__ 13 | 14 | return callback.__doc__ 15 | -------------------------------------------------------------------------------- /vexbot/extensions/hidden.py: -------------------------------------------------------------------------------- 1 | def hidden(self, *args, **kwargs): 2 | results = [] 3 | for k, v in self._commands.items(): 4 | if hasattr(v, 'hidden') and v.hidden: 5 | results.append('!' + k) 6 | else: 7 | continue 8 | return results 9 | -------------------------------------------------------------------------------- /vexbot/extensions/intents.py: -------------------------------------------------------------------------------- 1 | def get_intents(self, *args, **kwargs): 2 | return self.bot.intents.get_intent_names() 3 | 4 | 5 | def get_intent(self, *args, **kwargs): 6 | return self.bot.intents.get_intent_names(*args, **kwargs) 7 | -------------------------------------------------------------------------------- /vexbot/extensions/log.py: -------------------------------------------------------------------------------- 1 | import typing 2 | import logging 3 | 4 | 5 | def log_level(self, 6 | level: typing.Union[str, int]=None, 7 | *args, 8 | **kwargs) -> typing.Union[None, str]: 9 | """ 10 | Args: 11 | level: 12 | 13 | Returns: 14 | The log level if a `level` is passed in 15 | """ 16 | if level is None: 17 | return self.root_logger.getEffectiveLevel() 18 | # NOTE: `setLevel` takes both string and integers. Try to cast to an integer first 19 | try: 20 | value = int(level) 21 | # if we can't cast to an int, it's probably a string 22 | except ValueError: 23 | pass 24 | 25 | self.root_logger.setLevel(value) 26 | 27 | 28 | def set_debug(self, *args, **kwargs) -> None: 29 | self.root_logger.setLevel(logging.DEBUG) 30 | try: 31 | self.messaging.pub_handler.setLevel(logging.DEBUG) 32 | except Exception: 33 | pass 34 | 35 | 36 | def set_info(self, *args, **kwargs) -> None: 37 | """ 38 | Sets the log level to `INFO` 39 | """ 40 | self.root_logger.setLevel(logging.INFO) 41 | try: 42 | self.messaging.pub_handler.setLevel(logging.INFO) 43 | except Exception: 44 | pass 45 | 46 | def set_default(self, *args, **kwargs): 47 | self.root_logger.setLevel(logging.WARN) 48 | 49 | 50 | def filter_logs(self, *args, **kwargs): 51 | if not args: 52 | raise ValueError('Must supply something to filter against!') 53 | for name in args: 54 | for handler in self.root_logger.handlers: 55 | handler.addFilter(logging.Filter(name)) 56 | 57 | 58 | def anti_filter(self, *args, **kwargs): 59 | def filter_(record: logging.LogRecord): 60 | if record.name in args: 61 | return False 62 | return True 63 | for handler in self.root_logger.handlers: 64 | handler.addFilter(filter_) 65 | -------------------------------------------------------------------------------- /vexbot/extensions/modules.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | 4 | def add_module(self, function_name: str, alias: list=None, call_name: str=None, hidden: bool=False, *args, **kwargs) -> None: 5 | # split off the last dot as the function name, the rest is the module 6 | module, function = function_name.rsplit('.', maxsplit=1) 7 | # strip off any whitespace 8 | function = function.rstrip() 9 | # import the module 10 | module = importlib.import_module(module, package=None) 11 | # get the function from the module 12 | function = getattr(module, function) 13 | self.extend(function, alias=alias, name=call_name, hidden=hidden) 14 | 15 | values = {'function_name': function_name, 'alias': alias, 'call_name': call_name, 'hidden': hidden} 16 | 17 | self._config['modules'][function_name] = values 18 | self._config.sync() 19 | -------------------------------------------------------------------------------- /vexbot/extensions/news.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import gensim 4 | import newspaper 5 | 6 | 7 | def summarize_article(self, url: str, *args, **kwargs): 8 | article = newspaper.Article(url) 9 | article.download() 10 | article.parse() 11 | summarization = gensim.summarization.summarize(article.text) 12 | return summarization 13 | 14 | 15 | def get_hot_trends(self, *args, **kwargs): 16 | return newspaper.hot() 17 | 18 | 19 | def get_popular_urls(self, *args, **kwargs): 20 | return newspaper.popular_urls() 21 | -------------------------------------------------------------------------------- /vexbot/extensions/subprocess.py: -------------------------------------------------------------------------------- 1 | # from vexbot.intents import intent 2 | 3 | 4 | # @intent(name='restart_program') 5 | def restart(self, *args, mode: str='replace', **kwargs) -> None: 6 | self.logger.info(' restart service %s in mode %s', args, mode) 7 | for target in args: 8 | self.subprocess_manager.restart(target, mode) 9 | 10 | 11 | # @intent(CommandObserver, name='stop_program') 12 | def stop(self, *args, mode: str='replace', **kwargs) -> None: 13 | self.logger.info(' stop service %s in mode %s', args, mode) 14 | for target in args: 15 | self.subprocess_manager.stop(target, mode) 16 | 17 | 18 | # @intent(CommandObserver, name='get_status') 19 | def status(self, name: str, *args, **kwargs) -> str: 20 | self.logger.debug(' get status for %s', name) 21 | return self.subprocess_manager.status(name) 22 | 23 | 24 | # @intent(name='start_program') 25 | def start(self, *args, mode: str='replace', **kwargs) -> None: 26 | """ 27 | Start a program. 28 | 29 | Args: 30 | name: The name of the serivce. Will often end with `.service` or 31 | `.target`. If no argument is provided, will default to 32 | `.service` 33 | mode: 34 | 35 | Raises: 36 | GError: if the service is not found 37 | """ 38 | for target in args: 39 | self.logger.info(' start service %s in mode %s', target, mode) 40 | self.subprocess_manager.start(target, mode) 41 | 42 | 43 | def uptime(self, *args, name: str='vexbot.service', **kwargs): 44 | return self.subprocess_manager.uptime(name) 45 | 46 | 47 | uptime._meta = 'process_uptime' 48 | restart._meta = 'restart_process' 49 | start._meta = 'start_process' 50 | status._meta = 'status_process' 51 | stop._meta = 'stop_process' 52 | -------------------------------------------------------------------------------- /vexbot/extensions/system.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | 3 | _mb_conversion = 1024 * 1024 4 | 5 | def cpu_times(*args, **kwargs): 6 | return psutil.cpu_times() 7 | 8 | 9 | def cpu_count(logical=True, *args, **kwargs): 10 | cores = psutil.cpu_count(logical) 11 | if cores == 1: 12 | word = 'Core' 13 | else: 14 | word = 'Cores' 15 | return '{} CPU {}'.format(cores, word) 16 | 17 | 18 | def cpu_frequency(*args, **kwargs): 19 | freq = psutil.cpu_freq() 20 | if freq is None: 21 | return ('CPU frequency file moved or not present. See: ' 22 | 'https://stackoverflow.com/questions/42979943/python3-psutils') 23 | 24 | return [x.max for x in freq] 25 | 26 | 27 | def virtual_memory_percent(*arg, **kwargs): 28 | percent = psutil.virtual_memory().percent 29 | return '{}%'.format(percent) 30 | 31 | 32 | def virtual_memory_total(*args, **kwargs): 33 | total = int(psutil.virtual_memory().total / _mb_conversion) 34 | return '{} Mb'.format(total) 35 | 36 | 37 | def virtual_memory_used(*args, **kwargs): 38 | used = int(psutil.virtual_memory().used / _mb_conversion) 39 | return '{} Mb'.format(used) 40 | 41 | 42 | def swap(*args, **kwargs): 43 | swap = psutil.swap_memory() 44 | used = swap.used 45 | total = swap.total 46 | used = int(used/_mb_conversion) 47 | total = int(total/_mb_conversion) 48 | return 'Used: {} | Total: {}'.format(used, total) 49 | -------------------------------------------------------------------------------- /vexbot/intents.py: -------------------------------------------------------------------------------- 1 | import inspect as _inspect 2 | import functools as _functools 3 | 4 | 5 | # http://rasa-nlu.readthedocs.io/en/latest/tutorial.html#preparing-the-training-data 6 | def intent(function=None, 7 | name: str=None): 8 | 9 | if function is None: 10 | return _functools.partial(intent, 11 | name=name) 12 | 13 | function._vex_intent = True 14 | function._vex_intent_name = name 15 | return function 16 | 17 | 18 | class Entity: 19 | def __init__(self, text: str, start: int, end: int, name: str, type_: str, value: str=None): 20 | """ 21 | Args: 22 | text: Example utterance 23 | start: Starting position of the value 24 | end: Ending position of the value 25 | name: Name of the entity 26 | type_: Entity Type 27 | """ 28 | self.text = text 29 | self.start = start 30 | self.end = end 31 | self.name = name 32 | self.type = type_ 33 | if value is None: 34 | value = name 35 | self.value = value 36 | 37 | 38 | class FindEntity: 39 | def __init__(text, value, name): 40 | pass 41 | -------------------------------------------------------------------------------- /vexbot/language.py: -------------------------------------------------------------------------------- 1 | import code 2 | import pickle 3 | import logging 4 | import multiprocessing 5 | 6 | import numpy as np 7 | import spacy 8 | 9 | from sklearn.svm import SVC 10 | from sklearn.preprocessing import LabelEncoder 11 | from sklearn.model_selection import GridSearchCV 12 | 13 | from vexbot.entity_extraction import EntityExtraction 14 | from vexbot.util.get_classifier_filepath import (get_classifier_filepath, 15 | get_entity_filepath) 16 | 17 | 18 | MAX_CV_FOLDS = 5 19 | 20 | 21 | class Language: 22 | def __init__(self, classifier_filepath: str=None, language: str='en'): 23 | if classifier_filepath is None: 24 | classifier_filepath = get_classifier_filepath() 25 | 26 | self._classifier_filepath = classifier_filepath 27 | self.label_encoder = LabelEncoder() 28 | self.feature_extractor = None 29 | self.logger = logging.getLogger(__name__) 30 | self.language_model = None 31 | self._language_model_loaded = False 32 | self.classifier = None 33 | self.synonyms = {} 34 | self.entity_extractor = EntityExtraction() 35 | 36 | def predict(self, X): 37 | pred_result = self.classifier.predict_proba(X) 38 | sorted_indicies = np.fliplr(np.argsort(pred_result, axis=1)) 39 | return sorted_indicies, pred_result[:, sorted_indicies] 40 | 41 | def get_intent(self, text: str, entities: list=None, *args, **kwargs): 42 | # Text classification 43 | # Entity extraction 44 | if self.classifier is None: 45 | self.load_classifier() 46 | if not self._language_model_loaded: 47 | self._load_models() 48 | 49 | langague_features = self.language_model(text).vector 50 | langague_features = langague_features.reshape(1, -1) 51 | intents, probabilities = self.predict(langague_features) 52 | if intents.size > 0 and probabilities.size > 0: 53 | ranking = list(zip(list(intents), list(probabilities)))[:10] 54 | intent_ranking = [{"name": self.label_encoder.inverse_transform(intent_name), "confidence": score} for intent_name, score in ranking] 55 | first_name = intent_ranking[0]['name'][0] 56 | first_confidence = intent_ranking[0]['confidence'][0] 57 | else: 58 | first_name = None 59 | first_confidence = 0 60 | self.logger.debug('name: %s', first_name) 61 | self.logger.debug('confidence: %s', first_confidence) 62 | extracted = self.entity_extractor.get_entites(text, entities) 63 | # TODO: clean or merge entities here? 64 | 65 | return first_name, first_confidence, entities 66 | 67 | # text -> tokens 68 | # text -> entities 69 | # tokens + entites -> intent 70 | 71 | # total_word_feature_extractor 72 | 73 | # FIXME: currently unused 74 | def train(self, examples: dict): 75 | self.train_classifier(examples) 76 | # FIXME: Finish method 77 | # self.train_entity_extractor(examples) 78 | 79 | def _load_models(self): 80 | language = 'en' 81 | self.language_model = spacy.load(language, parser=False) 82 | self.entity_extractor._language_model = self.language_model 83 | self._language_model_loaded = True 84 | 85 | def load_classifier(self, filename: str=None): 86 | if filename is None: 87 | filename = self._classifier_filepath 88 | 89 | # FIXME: this code is super brittle for various reasons 90 | with open(filename, 'rb') as f: 91 | self.classifier = pickle.load(f) 92 | with open(filename + 'label', 'rb') as f: 93 | self.label_encoder = pickle.load(f) 94 | 95 | def get_spacy_text_features(self, text: str): 96 | # type: np.ndarray 97 | doc = self.language_model(text) 98 | vector = doc.vector 99 | return vector 100 | 101 | def train_classifier(self, examples: dict, filename: str=None, persist=True): 102 | # tokenizer_spacy -> Tokenizes -> Eh. 103 | # NOTE: this should keep track of a numpy list that is [#, 0/1] showing 104 | # which known regex pattern and if it was found (1) or not found (0) 105 | # AKA a feature matrix 106 | 107 | # intent_entity_featurizer_regex |entity| -> regex 108 | 109 | # NOTE: Start here 110 | # nlp_spacy -> spacy language initializer -> Done 111 | # intent_featurizer_spacy -> adds text features -> Eh. 112 | 113 | # ner_crf |entity| -> conditional random field entity extraction -> crf_entity_extractor 114 | # ner_synonyms |entity| -> dict replacing the entity values bascially 115 | 116 | # intent_classifier_sklearn -> end of pipeline 117 | if not self._language_model_loaded: 118 | self._load_models() 119 | 120 | if filename is None: 121 | filename = self._classifier_filepath 122 | 123 | self.logger.debug('train classifier filepath: %s', filename) 124 | # - Y - 125 | numbers = [] 126 | # text_features 127 | # - X - 128 | values = [] 129 | for k, v in examples.items(): 130 | self.logger.debug('intent: %s', k) 131 | # FIXME: I don't think I want this to be a callback. 132 | if v is None: 133 | continue 134 | for value in v: 135 | numbers.append(k) 136 | language_values = self.language_model(value) 137 | # get the vector from spacy 138 | values.append(language_values.vector) 139 | 140 | values = np.stack(values) 141 | # Y 142 | numbers = self.label_encoder.fit_transform(numbers) 143 | # aim for 5 examples in each fold 144 | cv_splits = max(2, 145 | min(MAX_CV_FOLDS, 146 | np.min(np.bincount(numbers)) // 5)) 147 | 148 | estimator = SVC(C=1, probability=True, class_weight='balanced') 149 | 150 | number_threads = multiprocessing.cpu_count() 151 | kernel = 'linear' 152 | # NOTE: the `kernel` parameter needs to be a list for sklearn 153 | tuned_parameters = [{'C': [1, 2, 5, 10, 20, 100], 154 | 'kernel': [kernel]}] 155 | # create the classifier 156 | self.classifier = GridSearchCV(estimator, 157 | param_grid=tuned_parameters, 158 | n_jobs=number_threads, 159 | cv=cv_splits, 160 | scoring='f1_weighted', 161 | verbose=1) 162 | 163 | # self.classifier.fit(X, y) 164 | self.classifier.fit(values, numbers) 165 | if persist: 166 | with open(filename, 'wb') as f: 167 | pickle.dump(self.classifier, f) 168 | with open(filename + 'label', 'wb') as f: 169 | pickle.dump(self.label_encoder, f) 170 | 171 | def get_entities(self, text: str, *args, **kwargs): 172 | pass 173 | 174 | # FIXME: Finish 175 | # https://github.com/RasaHQ/rasa_nlu/blob/master/rasa_nlu/extractors/mitie_entity_extractor.py 176 | def train_entity_extractor(self, examples: list, filename: str=None): 177 | if filename is None: 178 | filename = get_entity_filepath() 179 | 180 | trainer = mitie.ner_trainer(filename) 181 | trainer.num_threads = multiprocessing.cpu_count() 182 | for example in examples: 183 | tokens = self.tokenize(example['text']) 184 | for entity in example['entities']: 185 | pass 186 | 187 | 188 | def tokenize(self, text: str): 189 | return text.split() 190 | -------------------------------------------------------------------------------- /vexbot/observer.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta as _ABCMeta 2 | from abc import abstractmethod as _abstractmethod 3 | from vexbot.extensions import extendmany as _extendmany 4 | from vexbot.extensions import extend as _extend 5 | 6 | 7 | class Observer(metaclass=_ABCMeta): 8 | extensions = () 9 | def __init__(self, *args, **kwargs): 10 | extensions = list(self.extensions) 11 | dicts = [x for x in extensions if isinstance(x, dict)] 12 | for d in dicts: 13 | extensions.remove(d) 14 | self.extend(**d, update=False) 15 | _extendmany(self.__class__, *extensions) 16 | 17 | @_abstractmethod 18 | def on_next(self, value): 19 | return NotImplemented 20 | 21 | @_abstractmethod 22 | def on_error(self, error): 23 | return NotImplemented 24 | 25 | @_abstractmethod 26 | def on_completed(self): 27 | return NotImplemented 28 | 29 | def extend(self, 30 | method, 31 | alias: list=None, 32 | name: str=None, 33 | hidden: bool=False, 34 | instancemethod: bool=False, 35 | roles: list=None, 36 | update: bool=True, 37 | short: str=None): 38 | 39 | _extend(self.__class__, method, alias, name, hidden, instancemethod, roles, short) 40 | update_method = getattr(self, 'update_commands') 41 | if update and update_method: 42 | update_method() 43 | 44 | def extendmany(self, *methods): 45 | _extendmany(self.__class__, *methods) 46 | update = getattr(self, 'update_commands') 47 | if update: 48 | update() 49 | -------------------------------------------------------------------------------- /vexbot/robot.py: -------------------------------------------------------------------------------- 1 | import sys as _sys 2 | import logging as _logging 3 | 4 | from vexbot import _get_default_port_config 5 | # from vexbot.intents import BotIntents 6 | from vexbot.messaging import Messaging as _Messaging 7 | from vexbot.adapters.shell.observers import LogObserver 8 | from vexbot.command_observer import CommandObserver as _CommandObserver 9 | 10 | try: 11 | from vexbot.subprocess_manager import SubprocessManager 12 | except ImportError: 13 | SubprocessManager = False 14 | _logging.exception('Import Error for subprocess manager!') 15 | 16 | try: 17 | from vexbot.language import Language 18 | except ImportError: 19 | Language = False 20 | _logging.exception('Import Error for language!') 21 | 22 | class Robot: 23 | def __init__(self, 24 | bot_name: str, 25 | connection: dict=None, 26 | subprocess_manager=None): 27 | 28 | if connection is None: 29 | connection = _get_default_port_config() 30 | 31 | self.messaging = _Messaging(bot_name, **connection) 32 | log_name = __name__ if __name__ != '__main__' else 'vexbot.robot' 33 | self._logger = _logging.getLogger(log_name) 34 | if subprocess_manager is None and SubprocessManager: 35 | subprocess_manager = SubprocessManager() 36 | else: 37 | err = ('If you would like to use the subporcess manager, please ' 38 | 'run `pip install -e .[process_manager]` from the ' 39 | 'directory') 40 | 41 | self._logger.warn(err) 42 | 43 | # NOTE: probably need some kind of config here 44 | if Language: 45 | self.language = Language() 46 | else: 47 | err = ('If you would like to use natural language processing,' 48 | ' please run `pip install -e.[nlp]` from the directory') 49 | self._logger.warn(err) 50 | self.language = False 51 | 52 | # self.intents = BotIntents() 53 | self.subprocess_manager = subprocess_manager 54 | self.command_observer = _CommandObserver(self, 55 | self.messaging, 56 | self.subprocess_manager, 57 | self.language) 58 | 59 | self.messaging.command.subscribe(self.command_observer) 60 | self.messaging.chatter.subscribe(LogObserver(pass_through=True)) 61 | 62 | def run(self): 63 | if self.messaging is None: 64 | e = ' No `messaging` provided to `Robot` on initialization' 65 | self._logger.error(e) 66 | _sys.exit(1) 67 | 68 | self._logger.info(' Start Messaging') 69 | # NOTE: blocking call 70 | try: 71 | self.messaging.start() 72 | except KeyboardInterrupt: 73 | _sys.exit(0) 74 | -------------------------------------------------------------------------------- /vexbot/scheduler.py: -------------------------------------------------------------------------------- 1 | from zmq.eventloop.zmqstream import ZMQStream 2 | 3 | 4 | class Scheduler: 5 | def __init__(self): 6 | # FIXME: Move into the SocketFactory 7 | self._streams = [] 8 | 9 | def register_socket(self, socket, loop, on_recv): 10 | stream = ZMQStream(socket, loop) 11 | stream.on_recv(on_recv) 12 | self._streams.append(stream) 13 | -------------------------------------------------------------------------------- /vexbot/subprocess_manager.py: -------------------------------------------------------------------------------- 1 | import time 2 | # import atexit 3 | # import signal 4 | 5 | 6 | # TODO: try/catch around this. If pydbus isn't installed, probably an 7 | # incomplete installation 8 | from pydbus import SessionBus as _SessionBus 9 | from pydbus import SystemBus as _SystemBus 10 | 11 | 12 | def _name_helper(name: str): 13 | """ 14 | default to returning a name with `.service` 15 | """ 16 | name = name.rstrip() 17 | if name.endswith(('.service', '.socket', '.target')): 18 | return name 19 | return name + '.service' 20 | 21 | 22 | def _pretty_time_delta(seconds): 23 | seconds = abs(int(seconds)) 24 | days, seconds = divmod(seconds, 86400) 25 | hours, seconds = divmod(seconds, 3600) 26 | minutes, seconds = divmod(seconds, 60) 27 | if days > 0: 28 | return '%dday %dh' % (days, hours) 29 | elif hours > 0: 30 | return '%dh %dmin' % (hours, minutes) 31 | elif minutes > 0: 32 | return '%dmin %dsec' % (minutes, seconds) 33 | else: 34 | return '{}s'.format(seconds) 35 | 36 | 37 | # TODO: think of a better name 38 | class SubprocessManager: 39 | def __init__(self): 40 | self.session_bus_available = True 41 | try: 42 | self.bus = _SessionBus() 43 | # NOTE: GError from package `gi` 44 | except Exception: 45 | # No session bus if we're here. Depending on linux distro, that's 46 | # not surprising 47 | self.session_bus_available = False 48 | 49 | # TODO: It's possible that the user is on a system that is not using 50 | # systemd, which means that this next call will fail. Should probably 51 | # have a try/catch and then just default to not having a subprocess 52 | # manager if that happens. 53 | self.system_bus = _SystemBus() 54 | 55 | # TODO: Verify that we can start services as the system bus w/o root 56 | # permissions 57 | if self.session_bus_available: 58 | self.systemd = self.bus.get('.systemd1') 59 | else: 60 | self.systemd = self.system_bus.get('.systemd1') 61 | 62 | # atexit.register(self._close_subprocesses) 63 | # signal.signal(signal.SIGINT, self._handle_close_signal) 64 | # signal.signal(signal.SIGTERM, self._handle_close_signal) 65 | 66 | def start(self, name: str, mode: str='replace') -> None: 67 | # TODO: add in some parsing if fail 68 | # https://github.com/LEW21/pydbus/issues/35 69 | name = _name_helper(name) 70 | self.systemd.StartUnit(name, mode) 71 | 72 | def restart(self, name: str, mode: str='replace') -> None: 73 | name = _name_helper(name) 74 | self.systemd.ReloadOrRestartUnit(name, mode) 75 | 76 | def stop(self, name: str, mode: str='replace') -> None: 77 | name = _name_helper(name) 78 | self.systemd.StopUnit(name, mode) 79 | 80 | def _uptime(self, unit): 81 | time_start = unit.ConditionTimestamp/1000000 82 | delta = time.time() - time_start 83 | delta = _pretty_time_delta(delta) 84 | return delta 85 | 86 | def uptime(self, name: str) -> str: 87 | unit = self.bus.get('.systemd1', self.systemd.GetUnit(name)) 88 | return self._uptime(unit) 89 | 90 | def status(self, name: str) -> str: 91 | name = _name_helper(name) 92 | unit = self.bus.get('.systemd1', self.systemd.GetUnit(name)) 93 | # NOTE: what systemctl status shows 94 | # freenode.service - IRC Client 95 | # active (running) since Tue 2017-10-17 10:36:03 UTC; 19s ago 96 | delta = self._uptime(unit) 97 | fo = '%a %b %d %H:%M:%S %Y' 98 | time_start = time_start = unit.ConditionTimestamp/1000000 99 | time_stamp = time.strftime(fo, time.gmtime(time_start)) 100 | return '{}: {} ({}) since {}; {} ago'.format(unit.Id, 101 | unit.ActiveState, 102 | unit.SubState, 103 | time_stamp, 104 | delta) 105 | 106 | def get_units(self): 107 | return self.systemd.ListUnits() 108 | 109 | """ 110 | def mask(self, name: str): 111 | pass 112 | 113 | def unmask(self, name: str): 114 | pass 115 | 116 | def killall(self): 117 | pass 118 | """ 119 | -------------------------------------------------------------------------------- /vexbot/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/vexbot/util/__init__.py -------------------------------------------------------------------------------- /vexbot/util/create_cache_filepath.py: -------------------------------------------------------------------------------- 1 | from os import path, mkdir 2 | from vexbot.util.get_cache_filepath import get_cache_filepath 3 | 4 | 5 | def create_cache_directory(): 6 | home_direct = path.expanduser("~") 7 | cache = path.abspath(path.join(home_direct, 8 | '.cache')) 9 | 10 | if not path.isdir(cache): 11 | mkdir(cache) 12 | cache = path.join(cache, 'vexbot') 13 | if not path.isdir(cache): 14 | mkdir(cache) 15 | 16 | -------------------------------------------------------------------------------- /vexbot/util/create_config_file.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from configparser import ConfigParser as _ConfigParser 3 | 4 | from vexbot import _get_default_port_config 5 | from vexbot.util.get_config_filepath import get_config_filepath as _get_config_filepath 6 | 7 | 8 | def create_config_file(settings=None): 9 | filepath = _get_config_filepath() 10 | if path.isfile(filepath): 11 | return 12 | 13 | config = _ConfigParser() 14 | config['vexbot'] = {'kill_on_exit': False, 15 | 'profile': 'default'} 16 | 17 | config['vexbot_ports'] = _get_default_port_config() 18 | 19 | with open(filepath, 'w') as f: 20 | config.write(f) 21 | -------------------------------------------------------------------------------- /vexbot/util/create_vexdir.py: -------------------------------------------------------------------------------- 1 | from os import path, mkdir 2 | 3 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath as _get_vexdir_filepath 4 | from vexbot.util.get_vexdir_filepath import get_config_dir 5 | 6 | 7 | def create_vexdir(): 8 | config_dir = get_config_dir() 9 | vexdir = _get_vexdir_filepath() 10 | 11 | if not path.isdir(config_dir): 12 | mkdir(config_dir) 13 | if not path.isdir(vexdir): 14 | mkdir(vexdir) 15 | 16 | return vexdir 17 | -------------------------------------------------------------------------------- /vexbot/util/generate_certificates.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Generate client and server CURVE certificate files then move them into the 4 | appropriate store directory, private_keys or public_keys. 5 | 6 | Adapted from: 7 | https://github.com/zeromq/pyzmq/blob/master/examples/security/generate_certificates.py 8 | 9 | Original Author: Chris Laws 10 | """ 11 | 12 | import os 13 | import sys 14 | import shutil 15 | import logging 16 | import zmq.auth 17 | 18 | from prompt_toolkit import prompt 19 | 20 | from vexbot.util.create_vexdir import create_vexdir 21 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath 22 | from vexbot.util.get_certificate_filepath import get_certificate_filepath 23 | 24 | 25 | def generate_certificates(base_dir: str, remove_certificates: bool=False): 26 | ''' Generate client and server CURVE certificate files''' 27 | public_keys_dir = os.path.join(base_dir, 'public_keys') 28 | secret_keys_dir = os.path.join(base_dir, 'private_keys') 29 | 30 | # Make the public and private key directories 31 | for path in (public_keys_dir, secret_keys_dir): 32 | if not os.path.exists(path): 33 | os.mkdir(path) 34 | 35 | # create new keys in certificates dir 36 | server_public_file, server_secret_file = zmq.auth.create_certificates(base_dir, "vexbot") 37 | client_public_file, client_secret_file = zmq.auth.create_certificates(base_dir, "client") 38 | 39 | vexbot_public = os.path.join(public_keys_dir, 'vexbot.key') 40 | client_public = os.path.join(public_keys_dir, 'client.key') 41 | 42 | vexbot_secret = os.path.join(secret_keys_dir, 'vexbot.key_secret') 43 | client_secret = os.path.join(secret_keys_dir, 'client.key_secret') 44 | 45 | if os.path.exists(vexbot_public) and remove_certificates: 46 | os.unlink(vexbot_public) 47 | try: 48 | os.unlink(vexbot_secret) 49 | except OSError: 50 | pass 51 | elif os.path.exists(vexbot_public) and not remove_certificates: 52 | os.unlink(server_public_file) 53 | os.unlink(server_secret_file) 54 | 55 | if os.path.exists(client_public) and remove_certificates: 56 | os.unlink(client_public) 57 | try: 58 | os.unlink(client_secret) 59 | except OSError: 60 | pass 61 | elif os.path.exists(client_public) and not remove_certificates: 62 | os.unlink(client_public_file) 63 | os.unlink(client_secret_file) 64 | 65 | # move public keys to appropriate directory 66 | for key_file in os.listdir(base_dir): 67 | if key_file.endswith(".key"): 68 | shutil.move(os.path.join(base_dir, key_file), 69 | os.path.join(public_keys_dir, '.')) 70 | 71 | # move secret keys to appropriate directory 72 | for key_file in os.listdir(base_dir): 73 | if key_file.endswith(".key_secret"): 74 | shutil.move(os.path.join(base_dir, key_file), 75 | os.path.join(secret_keys_dir, '.')) 76 | 77 | 78 | def _check_vexbot_filepath(create_if_missing=True): 79 | filepath = get_vexdir_filepath() 80 | is_missing = os.path.isdir(filepath) 81 | 82 | if create_if_missing and is_missing: 83 | create_vexdir() 84 | elif is_missing and not create_if_missing: 85 | logging.error('Vexbot filepath not found! Please create') 86 | 87 | 88 | def main(): 89 | _check_vexbot_filepath(create_if_missing=True) 90 | cert_path = get_certificate_filepath() 91 | if not os.path.exists(cert_path): 92 | os.makedirs(cert_path, exist_ok=True) 93 | 94 | remove_certificates = input('Remove certificates if present? Y/n: ') 95 | remove_certificates = remove_certificates.lower() 96 | if remove_certificates == 'y': 97 | remove_certificates = True 98 | else: 99 | remove_certificates = False 100 | generate_certificates(cert_path, remove_certificates) 101 | 102 | 103 | if __name__ == '__main__': 104 | main() 105 | -------------------------------------------------------------------------------- /vexbot/util/generate_config_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import inspect 4 | import configparser 5 | import pkg_resources 6 | from os import path 7 | 8 | from vexbot.util.get_vexdir_filepath import get_config_dir 9 | 10 | 11 | def _systemd_user_filepath() -> str: 12 | config = get_config_dir() 13 | config = path.join(config, 'systemd', 'user') 14 | os.makedirs(config, exist_ok=True) 15 | return path.join(config, 'vexbot.service') 16 | 17 | 18 | def _get_vexbot_robot() -> str: 19 | for entry_point in pkg_resources.iter_entry_points('console_scripts'): 20 | if entry_point.name == 'vexbot_robot' and entry_point.dist.project_name == 'vexbot': 21 | # TODO: call the `require` function first and provide an installer 22 | # so that we can get the matching `Distribution` instance? 23 | return inspect.getsourcefile(entry_point.resolve()) 24 | 25 | raise RuntimeError('Entry point for `vexbot_robot` not found! Are you ' 26 | 'sure it\'s installed?') 27 | 28 | 29 | def config(filepath=None, remove_config=False): 30 | if filepath is None: 31 | filepath = _systemd_user_filepath() 32 | if path.exists(filepath) and remove_config: 33 | os.unlink(filepath) 34 | elif path.exists(filepath) and not remove_config: 35 | logging.error(' Configuration filepath already exsists at: %s', 36 | filepath) 37 | return 38 | 39 | parser = configparser.SafeConfigParser() 40 | parser.optionxform = str 41 | parser['Unit'] = {'Description': 'Helper Bot'} 42 | parser['Service'] = {'Type': 'simple', 43 | 'ExecStart': _get_vexbot_robot(), 44 | 'StandardOutput': 'syslog', 45 | 'StandardError': 'syslog'} 46 | 47 | with open(filepath, 'w') as f: 48 | parser.write(f) 49 | 50 | print('Config file created at: {}'.format(filepath)) 51 | 52 | 53 | def main(): 54 | remove_config = input('Remove config if present? [Y]/n: ') 55 | remove_config = remove_config.lower() 56 | if remove_config in ('y', 'yes', 'ye', ''): 57 | remove_config = True 58 | else: 59 | remove_config = False 60 | 61 | config(remove_config=remove_config) 62 | 63 | 64 | if __name__ == '__main__': 65 | main() 66 | -------------------------------------------------------------------------------- /vexbot/util/get_cache_filepath.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | 3 | 4 | def get_cache_filepath() -> str: 5 | home_direct = path.expanduser("~") 6 | cache = path.abspath(path.join(home_direct, 7 | '.cache')) 8 | cache = path.join(cache, 'vexbot') 9 | return cache 10 | 11 | 12 | def get_cache(name: str) -> str: 13 | filepath = get_cache_filepath() 14 | filepath = path.join(filepath, name) 15 | return filepath 16 | -------------------------------------------------------------------------------- /vexbot/util/get_certificate_filepath.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | 3 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath 4 | 5 | 6 | def get_certificate_filepath(): 7 | vexdir = get_vexdir_filepath() 8 | certs = path.join(vexdir, 'certificates') 9 | return certs 10 | 11 | def get_certificate_directories() -> (str, str): 12 | root = get_certificate_filepath() 13 | public_dir = 'public_keys' 14 | private_dir = 'private_keys' 15 | public_dir = path.join(root, public_dir) 16 | private_dir = path.join(root, private_dir) 17 | return public_dir, private_dir 18 | 19 | def _certificate_helper(public_filepath: str, 20 | secret_filepath: str) -> (str, str): 21 | 22 | public_dir, private_dir = get_certificate_directories() 23 | secret_filepath = path.join(private_dir, secret_filepath) 24 | public_filepath = path.join(public_dir, public_filepath) 25 | return public_filepath, secret_filepath 26 | 27 | 28 | def get_vexbot_certificate_filepath() -> (str, bool): 29 | """ 30 | Returns the vexbot certificate filepath and the whether it is the private 31 | filepath or not 32 | """ 33 | public_filepath, secret_filepath = _certificate_helper('vexbot.key','vexbot.key_secret') 34 | if path.isfile(secret_filepath): 35 | return secret_filepath, True 36 | if not path.isfile(public_filepath): 37 | err = ('certificates not found. Generate certificates from the ' 38 | 'command line using `vexbot_generate_certificates`') 39 | raise FileNotFoundError(err) 40 | return public_filepath, False 41 | 42 | 43 | def get_client_certificate_filepath() -> (str, bool): 44 | _, secret_filepath = _certificate_helper('client.key','client.key_secret') 45 | if not path.isfile(secret_filepath): 46 | err = 'certificates not found. Where they generated?' 47 | raise FileNotFoundError(err) 48 | return secret_filepath 49 | -------------------------------------------------------------------------------- /vexbot/util/get_classifier_filepath.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath 3 | 4 | 5 | def get_classifier_filepath(name: str=None) -> str: 6 | if name is None: 7 | name = 'classifier.vex' 8 | directory = get_vexdir_filepath() 9 | return path.join(directory, name) 10 | 11 | 12 | def get_entity_filepath(name: str=None) -> str: 13 | if name is None: 14 | name = 'entity_extractor.vex' 15 | directory = get_vexdir_filepath() 16 | return path.join(directory, name) 17 | -------------------------------------------------------------------------------- /vexbot/util/get_config.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser as _ConfigParser 2 | 3 | from vexbot.util.get_config_filepath import get_config_filepath as _get_config_filepath 4 | 5 | 6 | def get_config(filepath=None): 7 | if filepath is None: 8 | filepath = _get_config_filepath() 9 | config = _ConfigParser() 10 | config.read(filepath) 11 | config_dict = {s: dict(config.items(s)) 12 | for s 13 | in config.sections()} 14 | 15 | return config_dict 16 | -------------------------------------------------------------------------------- /vexbot/util/get_config_filepath.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | 3 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath as _get_vexdir_filepath 4 | 5 | 6 | def get_config_filepath(): 7 | vexdir = _get_vexdir_filepath() 8 | config_filepath = path.join(vexdir, 'vexbot.ini') 9 | return config_filepath 10 | -------------------------------------------------------------------------------- /vexbot/util/get_kwargs.py: -------------------------------------------------------------------------------- 1 | import argparse as _argparse 2 | 3 | 4 | def get_kwargs(): 5 | parser = _argparse.ArgumentParser() 6 | parser.add_argument('--configuration_filepath', default=None) 7 | args = parser.parse_args() 8 | return vars(args) 9 | -------------------------------------------------------------------------------- /vexbot/util/get_vexdir_filepath.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | 3 | 4 | def get_config_dir(): 5 | home_direct = path.expanduser("~") 6 | config = path.abspath(path.join(home_direct, 7 | '.config')) 8 | 9 | return config 10 | 11 | 12 | def get_vexdir_filepath(): 13 | vexdir = path.join(get_config_dir(), 14 | 'vexbot') 15 | 16 | return vexdir 17 | -------------------------------------------------------------------------------- /vexbot/util/lru_cache.py: -------------------------------------------------------------------------------- 1 | # Class for the node objects. 2 | class _dlnode(object): 3 | def __init__(self): 4 | self.empty = True 5 | 6 | 7 | # https://github.com/jlhutch/pylru 8 | class LRUCache(object): 9 | 10 | def __init__(self, size, add_callback=None, delete_callback=None): 11 | 12 | self.add_callback = add_callback 13 | self.delete_callback = delete_callback 14 | 15 | # Create an empty hash table. 16 | self.table = {} 17 | 18 | # Initialize the doubly linked list with one empty node. This is an 19 | # invariant. The cache size must always be greater than zero. Each 20 | # node has a 'prev' and 'next' variable to hold the node that comes 21 | # before it and after it respectively. Initially the two variables 22 | # each point to the head node itself, creating a circular doubly 23 | # linked list of size one. Then the size() method is used to adjust 24 | # the list to the desired size. 25 | 26 | self.head = _dlnode() 27 | self.head.next = self.head 28 | self.head.prev = self.head 29 | 30 | self.listSize = 1 31 | 32 | # Adjust the size 33 | self.size(size) 34 | 35 | def __len__(self): 36 | return len(self.table) 37 | 38 | def clear(self): 39 | for node in self.dli(): 40 | node.empty = True 41 | node.key = None 42 | node.value = None 43 | 44 | self.table.clear() 45 | 46 | def __contains__(self, key): 47 | return key in self.table 48 | 49 | # Looks up a value in the cache without affecting cache order. 50 | def peek(self, key): 51 | # Look up the node 52 | node = self.table[key] 53 | return node.value 54 | 55 | def __getitem__(self, key): 56 | # Look up the node 57 | node = self.table[key] 58 | 59 | # Update the list ordering. Move this node so that is directly 60 | # proceeds the head node. Then set the 'head' variable to it. This 61 | # makes it the new head of the list. 62 | self.mtf(node) 63 | self.head = node 64 | 65 | # Return the value. 66 | return node.value 67 | 68 | def get(self, key, default=None): 69 | """Get an item - return default (None) if not present""" 70 | if key not in self.table: 71 | return default 72 | 73 | return self[key] 74 | 75 | def __setitem__(self, key, value): 76 | # First, see if any value is stored under 'key' in the cache already. 77 | # If so we are going to replace that value with the new one. 78 | if key in self.table: 79 | 80 | # Lookup the node 81 | node = self.table[key] 82 | 83 | # Replace the value. 84 | node.value = value 85 | 86 | # Update the list ordering. 87 | self.mtf(node) 88 | self.head = node 89 | 90 | return 91 | 92 | # TODO First add the value here 93 | if self.add_callback is not None: 94 | self.add_callback(key) 95 | 96 | # Ok, no value is currently stored under 'key' in the cache. We need 97 | # to choose a node to place the new item in. There are two cases. If 98 | # the cache is full some item will have to be pushed out of the 99 | # cache. We want to choose the node with the least recently used 100 | # item. This is the node at the tail of the list. If the cache is not 101 | # full we want to choose a node that is empty. Because of the way the 102 | # list is managed, the empty nodes are always together at the tail 103 | # end of the list. Thus, in either case, by chooseing the node at the 104 | # tail of the list our conditions are satisfied. 105 | 106 | # Since the list is circular, the tail node directly preceeds the 107 | # 'head' node. 108 | node = self.head.prev 109 | 110 | # If the node already contains something we need to remove the old 111 | # key from the dictionary. 112 | if not node.empty: 113 | if self.delete_callback is not None: 114 | self.delete_callback(node.key) 115 | del self.table[node.key] 116 | 117 | # Place the new key and value in the node 118 | node.empty = False 119 | node.key = key 120 | node.value = value 121 | 122 | # Add the node to the dictionary under the new key. 123 | self.table[key] = node 124 | 125 | # We need to move the node to the head of the list. The node is the 126 | # tail node, so it directly preceeds the head node due to the list 127 | # being circular. Therefore, the ordering is already correct, we just 128 | # need to adjust the 'head' variable. 129 | self.head = node 130 | 131 | def update(self, items): 132 | 133 | # Add multiple items to the cache. 134 | for n, v in items.items(): 135 | self[n] = v 136 | 137 | def __delitem__(self, key): 138 | 139 | # Lookup the node, then remove it from the hash table. 140 | node = self.table[key] 141 | del self.table[key] 142 | 143 | node.empty = True 144 | 145 | # Not strictly necessary. 146 | node.key = None 147 | node.value = None 148 | 149 | # Because this node is now empty we want to reuse it before any 150 | # non-empty node. To do that we want to move it to the tail of the 151 | # list. We move it so that it directly preceeds the 'head' node. This 152 | # makes it the tail node. The 'head' is then adjusted. This 153 | # adjustment ensures correctness even for the case where the 'node' 154 | # is the 'head' node. 155 | self.mtf(node) 156 | self.head = node.next 157 | 158 | def __iter__(self): 159 | 160 | # Return an iterator that returns the keys in the cache in order from 161 | # the most recently to least recently used. Does not modify the cache 162 | # order. 163 | for node in self.dli(): 164 | yield node.key 165 | 166 | def items(self): 167 | 168 | # Return an iterator that returns the (key, value) pairs in the cache 169 | # in order from the most recently to least recently used. Does not 170 | # modify the cache order. 171 | for node in self.dli(): 172 | yield (node.key, node.value) 173 | 174 | def keys(self): 175 | 176 | # Return an iterator that returns the keys in the cache in order from 177 | # the most recently to least recently used. Does not modify the cache 178 | # order. 179 | for node in self.dli(): 180 | yield node.key 181 | 182 | def values(self): 183 | 184 | # Return an iterator that returns the values in the cache in order 185 | # from the most recently to least recently used. Does not modify the 186 | # cache order. 187 | for node in self.dli(): 188 | yield node.value 189 | 190 | def size(self, size=None): 191 | 192 | if size is not None: 193 | assert size > 0 194 | if size > self.listSize: 195 | self.addTailNode(size - self.listSize) 196 | elif size < self.listSize: 197 | self.removeTailNode(self.listSize - size) 198 | 199 | return self.listSize 200 | 201 | # Increases the size of the cache by inserting n empty nodes at the tail 202 | # of the list. 203 | def addTailNode(self, n): 204 | for i in range(n): 205 | node = _dlnode() 206 | node.next = self.head 207 | node.prev = self.head.prev 208 | 209 | self.head.prev.next = node 210 | self.head.prev = node 211 | 212 | self.listSize += n 213 | 214 | # Decreases the size of the list by removing n nodes from the tail of the 215 | # list. 216 | def removeTailNode(self, n): 217 | assert self.listSize > n 218 | for i in range(n): 219 | node = self.head.prev 220 | if not node.empty: 221 | if self.callback is not None: 222 | self.callback(node.key, node.value) 223 | del self.table[node.key] 224 | 225 | # Splice the tail node out of the list 226 | self.head.prev = node.prev 227 | node.prev.next = self.head 228 | 229 | # The next four lines are not strictly necessary. 230 | node.prev = None 231 | node.next = None 232 | 233 | node.key = None 234 | node.value = None 235 | 236 | self.listSize -= n 237 | 238 | # This method adjusts the ordering of the doubly linked list so that 239 | # 'node' directly precedes the 'head' node. Because of the order of 240 | # operations, if 'node' already directly precedes the 'head' node or if 241 | # 'node' is the 'head' node the order of the list will be unchanged. 242 | def mtf(self, node): 243 | node.prev.next = node.next 244 | node.next.prev = node.prev 245 | 246 | node.prev = self.head.prev 247 | node.next = self.head.prev.next 248 | 249 | node.next.prev = node 250 | node.prev.next = node 251 | 252 | # This method returns an iterator that iterates over the non-empty nodes 253 | # in the doubly linked list in order from the most recently to the least 254 | # recently used. 255 | def dli(self): 256 | node = self.head 257 | for i in range(len(self.table)): 258 | yield node 259 | node = node.next 260 | -------------------------------------------------------------------------------- /vexbot/util/messaging.py: -------------------------------------------------------------------------------- 1 | from zmq.utils.interop import cast_int_addr 2 | 3 | def get_addresses(message: list) -> list: 4 | """ 5 | parses a raw list from zmq to get back the components that are the address 6 | messages are broken by the addresses in the beginning and then the 7 | message, like this: ```addresses | '' | message ``` 8 | """ 9 | # Need the address so that we know who to send the message back to 10 | addresses = [] 11 | for address in message: 12 | # if we hit a blank string, then we've got all the addresses 13 | if address == b'': 14 | break 15 | addresses.append(address) 16 | 17 | return addresses 18 | -------------------------------------------------------------------------------- /vexbot/util/socket_factory.py: -------------------------------------------------------------------------------- 1 | import sys as _sys 2 | import zmq as _zmq 3 | import logging 4 | from os import path 5 | 6 | from zmq.auth.thread import ThreadAuthenticator 7 | from zmq.auth.ioloop import IOLoopAuthenticator 8 | 9 | from vexbot.util.get_certificate_filepath import (get_certificate_filepath, 10 | get_vexbot_certificate_filepath, 11 | get_client_certificate_filepath) 12 | 13 | class SocketFactory: 14 | """ 15 | Abstracts out the socket creation, specifically the issues with 16 | transforming ports into zmq addresses. 17 | Also sets up some default protocol and address handeling. 18 | For example, if everything is being run locally on one machine, and the 19 | default is to bind all addresses `*` and the transport protocol 20 | `tcp` means tcp communication 21 | """ 22 | def __init__(self, 23 | address: str, 24 | protocol: str='tcp', 25 | context: 'zmq.Context'=None, 26 | logger: 'logging.Logger'=None, 27 | loop=None, 28 | auth_whitelist: list=None, 29 | using_auth: bool=True): 30 | 31 | self.address = address 32 | self.protocol = protocol 33 | self.context = context or _zmq.Context.instance() 34 | if logger is None: 35 | logger = logging.getLogger(__name__) 36 | self.logger = logger 37 | self._server_certs = (None, None) 38 | self.using_auth = using_auth 39 | 40 | if self.using_auth: 41 | self._setup_auth(auth_whitelist, loop) 42 | 43 | def _setup_auth(self, auth_whitelist: list, loop): 44 | if loop is None: 45 | self.auth = ThreadAuthenticator(self.context) 46 | else: 47 | self.auth = IOLoopAuthenticator(self.context, io_loop=loop) 48 | 49 | # allow all local host 50 | self.logger.debug('Auth whitelist %s', auth_whitelist) 51 | if auth_whitelist is not None: 52 | self.auth.allow(auth_whitelist) 53 | self._base_filepath = get_certificate_filepath() 54 | public_key_location = path.join(self._base_filepath, 'public_keys') 55 | self.auth.configure_curve(domain='*', location=public_key_location) 56 | self.auth.start() 57 | 58 | def _set_server_certs(self, any_=True): 59 | secret_filepath, secret = get_vexbot_certificate_filepath() 60 | if not secret and not any_: 61 | err = ('Could not find the vexbot certificates. Generate them ' 62 | 'using `vexbot_generate_certificates` from command line') 63 | raise FileNotFoundError(err) 64 | 65 | self._server_certs = _zmq.auth.load_certificate(secret_filepath) 66 | 67 | def _create_using_auth(self, socket, address, bind, on_error, socket_name): 68 | if not any(self._server_certs): 69 | self._set_server_certs(bind) 70 | if bind: 71 | if self._server_certs[1] is None: 72 | err = ('Server Secret File Not Found! Generate using ' 73 | '`vexbot_generate_certificates` from the command line') 74 | raise FileNotFoundError(err) 75 | 76 | public, private = self._server_certs 77 | else: 78 | # NOTE: This raises a `FileNotFoundError` if not found 79 | secret_filepath = get_client_certificate_filepath() 80 | public, private = _zmq.auth.load_certificate(secret_filepath) 81 | 82 | socket.curve_secretkey = private 83 | socket.curve_publickey = public 84 | 85 | if bind: 86 | socket.curve_server = True 87 | try: 88 | socket.bind(address) 89 | except _zmq.error.ZMQError: 90 | self._handle_error(on_error, address, socket_name) 91 | socket = None 92 | else: # connect the socket 93 | socket.curve_serverkey = self._server_certs[0] 94 | socket.connect(address) 95 | 96 | def _create_no_auth(self, socket, address, bind, on_error, socket_name): 97 | if bind: 98 | try: 99 | socket.bind(address) 100 | except _zmq.error.ZMQError: 101 | self._handle_error(on_error, address, socket_name) 102 | socket = None 103 | else: # connect the socket 104 | socket.connect(address) 105 | 106 | def create_n_connect(self, 107 | socket_type, 108 | address: str, 109 | bind=False, 110 | on_error='log', 111 | socket_name=''): 112 | """ 113 | Creates and connects or binds the sockets 114 | on_error: 115 | 'log': will log error 116 | 'exit': will exit the program 117 | socket_name: 118 | used for troubleshooting/logging 119 | """ 120 | self.logger.debug('create and connect: %s %s %s', 121 | socket_type, socket_name, address) 122 | 123 | socket = self.context.socket(socket_type) 124 | if self.using_auth: 125 | self._create_using_auth(socket, address, bind, on_error, socket_name) 126 | else: 127 | self._create_no_auth(socket, address, bind, on_error, socket_name) 128 | 129 | return socket 130 | 131 | def multiple_create_n_connect(self, 132 | socket_type, 133 | addresses: tuple, 134 | bind=False, 135 | on_error='log', 136 | socket_name=''): 137 | 138 | # TODO: Refactor this to remove code duplication with create_n_connect 139 | # Or at least make better sense 140 | socket = self.context.socket(socket_type) 141 | for address in addresses: 142 | if self.auth: 143 | self._create_using_auth(socket, address, bind, on_error, socket_name) 144 | else: 145 | self._create_no_auth(socket, address, bind, on_error, socket_name) 146 | return socket 147 | 148 | def to_address(self, port: str): 149 | """ 150 | transforms the ports into addresses. 151 | Will fall through if the port looks like an address 152 | """ 153 | # check to see if user passed in a string 154 | # if they did, they want to use that instead 155 | if isinstance(port, str) and len(port) > 6: 156 | return port 157 | 158 | zmq_address = '{}://{}:{}' 159 | return zmq_address.format(self.protocol, 160 | self.address, 161 | port) 162 | 163 | def iterate_multiple_addresses(self, ports: (str, list, tuple)): 164 | """ 165 | transforms an iterable, or the expectation of an iterable 166 | into zmq addresses 167 | """ 168 | if isinstance(ports, (str, int)): 169 | # TODO: verify this works. 170 | ports = tuple(ports,) 171 | 172 | result = [] 173 | for port in ports: 174 | result.append(self.to_address(port)) 175 | 176 | return result 177 | 178 | def _handle_error(self, how_to: str, address: str, socket_name: str): 179 | if socket_name == '': 180 | socket_name = 'unknown type' 181 | 182 | if how_to == 'exit': 183 | self._handle_bind_error_by_exit(address, socket_name) 184 | else: 185 | self._handle_bind_error_by_log(address, socket_name) 186 | 187 | def _handle_bind_error_by_log(self, address: str, socket_name: str): 188 | if self.logger is None: 189 | return 190 | s = 'Address bind attempt fail. Address tried: {}' 191 | s = s.format(address) 192 | self.logger.error(s) 193 | self.logger.error('socket type: {}'.format(socket_name)) 194 | 195 | def _handle_bind_error_by_exit(self, address: str, socket_type: str): 196 | if self.logger is not None: 197 | s = 'Address bind attempt fail. Alredy in use? Address tried: {}' 198 | s = s.format(address) 199 | self.logger.error(s) 200 | self.logger.error('socket type: {}'.format(socket_type)) 201 | 202 | _sys.exit(1) 203 | --------------------------------------------------------------------------------