├── .codeclimate.yml
├── .coveragerc
├── .gitignore
├── .travis.yml
├── LICENSE.md
├── MANIFEST.in
├── README.rst
├── config
├── irc.ini
├── irc.service
├── socket_io.ini
├── socket_io.service
├── vexbot.ini
├── vexbot.service
├── xmpp.ini
├── xmpp.service
├── youtube.ini
└── youtube.service
├── data
├── add_extensions
├── cpu_count
├── get_help
├── get_installed_extensions
├── get_last_error
├── get_log
├── get_services
├── restart_process
├── start_process
├── status_process
├── stop_process
├── swap
├── virtual_memory_percent
└── virtual_memory_used
├── docs
├── Makefile
├── adapter_configuration.rst
├── adapter_development.rst
├── conf.py
├── extension_development.rst
├── extension_discovery.rst
├── extension_management.rst
├── index.rst
└── packages.rst
├── setup.py
├── tests
├── __init__.py
├── test_command_managers.py
├── test_messaging.py
└── test_subprocess_manager.py
└── vexbot
├── __init__.py
├── __main__.py
├── _logging.py
├── _version.py
├── adapters
├── __init__.py
├── irc
│ ├── __init__.py
│ ├── __main__.py
│ ├── echo_to_message.py
│ └── observer.py
├── messaging.py
├── shell
│ ├── __init__.py
│ ├── __main__.py
│ ├── completers.py
│ ├── intents.py
│ ├── interfaces.py
│ ├── observers.py
│ ├── parser.py
│ └── shell.py
├── socket_io
│ ├── __init__.py
│ ├── __main__.py
│ └── observer.py
├── stackoverflow.py
├── xmpp
│ ├── __init__.py
│ └── __main__.py
└── youtube
│ ├── __init__.py
│ └── __main__.py
├── command.py
├── command_observer.py
├── dynamic_entities.py
├── entity_extraction.py
├── extension_metadata.py
├── extensions
├── __init__.py
├── admin.py
├── develop.py
├── digitalocean.py
├── dynamic_loading.py
├── extensions.py
├── help.py
├── hidden.py
├── intents.py
├── log.py
├── modules.py
├── news.py
├── subprocess.py
└── system.py
├── intents.py
├── language.py
├── messaging.py
├── observer.py
├── robot.py
├── scheduler.py
├── subprocess_manager.py
└── util
├── __init__.py
├── create_cache_filepath.py
├── create_config_file.py
├── create_vexdir.py
├── generate_certificates.py
├── generate_config_file.py
├── get_cache_filepath.py
├── get_certificate_filepath.py
├── get_classifier_filepath.py
├── get_config.py
├── get_config_filepath.py
├── get_kwargs.py
├── get_vexdir_filepath.py
├── lru_cache.py
├── messaging.py
└── socket_factory.py
/.codeclimate.yml:
--------------------------------------------------------------------------------
1 | languages:
2 | Python: true
3 |
4 | exclude_paths:
5 | - 'tests/*'
6 |
--------------------------------------------------------------------------------
/.coveragerc:
--------------------------------------------------------------------------------
1 | [run]
2 | omit = */tests/*, */distutils/*, /tmp/*, */site-packages/*
3 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Generic files to ignore
2 | *~
3 | *.lock
4 | *.DS_Store
5 | *.swp
6 | *.out
7 | ENV*
8 |
9 | # Byte-compiled / optimized / DLL files
10 | __pycache__/
11 | *.py[cod]
12 |
13 | # C extensions
14 | *.so
15 |
16 | # Distribution / packaging
17 | .Python
18 | env/
19 | venv/
20 | build/
21 | develop-eggs/
22 | dist/
23 | downloads/
24 | eggs/
25 | lib/
26 | lib64/
27 | parts/
28 | sdist/
29 | var/
30 | *.egg-info/
31 | .installed.cfg
32 | *.egg
33 |
34 | # PyInstaller
35 | # Usually these files are written by a python script from a template
36 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
37 | *.manifest
38 | *.spec
39 |
40 | # Installer logs
41 | pip-log.txt
42 | pip-delete-this-directory.txt
43 |
44 | # Unit test / coverage reports
45 | htmlcov/
46 | .tox/
47 | .coverage
48 | .cache
49 | nosetests.xml
50 | coverage.xml
51 |
52 | # Translations
53 | *.mo
54 | *.pot
55 |
56 | # Django stuff/Logfiles
57 | *.log
58 |
59 | # Sphinx documentation
60 | docs/_build/
61 |
62 | # PyBuilder
63 | target/
64 |
65 | # Coverage reports for a specific version
66 | .coverage
67 |
68 | # SublimeLinter config file
69 | .sublimelinterrc
70 |
71 | settings.yml
72 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: python
2 |
3 | notifications:
4 | email: false
5 |
6 | python:
7 | - "3.5"
8 |
9 | # command to install dependencies
10 | install:
11 | - pip install flake8
12 | - pip install coverage
13 | - pip install python-coveralls
14 | - pip install -e .
15 |
16 | # command to run tests
17 | script:
18 | - coverage run -m unittest discover
19 | - flake8 vexbot
20 | - flake8 tests
21 |
22 | after_success:
23 | - coverage report
24 | - coveralls
25 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include data/*
2 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | ======
2 | vexbot
3 | ======
4 |
5 | |readthedocs| |codeclimate|
6 |
7 | .. |readthedocs| image:: https://readthedocs.org/projects/vexbot/badge/?version=latest
8 | :target: http://vexbot.readthedocs.io/en/latest/?badge=latest
9 | :alt: Documentation Status
10 |
11 | .. |codeclimate| image:: https://api.codeclimate.com/v1/badges/c17129f65b11dfef34c0/maintainability.svg
12 | :target: https://codeclimate.com/github/benhoff/vexbot/badges/gpa.svg
13 | :alt: Climate Status
14 |
15 |
16 | Pluggable bot
17 |
18 | Under development. Very useable but currently not feature complete.
19 |
20 | Requirements
21 | ------------
22 |
23 | Requires python 3.5 for asyncio and only runs on linux.
24 |
25 | If you're a python developer, you can probably get this to run on not linux.
26 |
27 | Installation
28 | ------------
29 |
30 | You will need an active DBus user session bus. Depending on your distro, you might already have one (Arch linux, for example).
31 |
32 | For Ubuntu:
33 |
34 | .. code-block:: bash
35 |
36 | $ apt-get install dbus-user-session python3-gi python3-dev python3-pip build-essential
37 |
38 | For everyone:
39 |
40 | .. code-block:: bash
41 |
42 | $ python3 -m venv
43 |
44 | .. code-block:: bash
45 |
46 | $ source /bin/activate
47 |
48 | .. code-block:: bash
49 |
50 | $ ln -s /usr/lib/python3/dist-packages/gi /lib/python3.5/site-packages/
51 |
52 | .. code-block:: bash
53 |
54 | $ pip install vexbot[process_manager]
55 |
56 | Configuring
57 | -----------
58 |
59 | Make sure your virtual environment is activated. Then run:
60 |
61 | .. code-block:: bash
62 |
63 | $ vexbot_generate_certificates
64 |
65 | .. code-block:: bash
66 |
67 | $ vexbot_generate_unit_file
68 |
69 | .. code-block:: bash
70 |
71 | $ systemctl --user daemon-reload
72 |
73 | Your bot is ready to run!
74 |
75 | Running
76 | -------
77 |
78 | .. code-block:: bash
79 |
80 | $ systemctl --user start vexbot
81 |
82 | Or
83 |
84 | .. code-block:: bash
85 |
86 | $ vexbot_robot
87 |
88 | Please note that vexbot has a client/server architecture. The above commands will launch the server. To launch the command line client:
89 |
90 | .. code-block:: bash
91 |
92 | $ vexbot
93 |
94 | Exit the command line client by typing `!exit` or using `ctl+D`.
95 |
--------------------------------------------------------------------------------
/config/irc.ini:
--------------------------------------------------------------------------------
1 | [bot]
2 | nick=
3 | password=
4 | host=
5 | port =
6 | # ssl = True
7 | # level = 30
8 | inlcudes =
9 | irc3.plugins.core
10 | irc3.plugins.autojoins
11 | autojoins =
12 | cmd = !
13 | service_name=
14 |
15 | #[connection]
16 | #address=
17 | #publish_address=
18 | #subscribe_address=
19 | #command_address=
20 | #control_address=
21 |
--------------------------------------------------------------------------------
/config/irc.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=IRC Client for Vexbot
3 |
4 | [Service]
5 | Type=simple
6 | ExecStart=/path/to/virtual/env/binary /path/to/config
7 |
8 | StandardOutput=syslog
9 | StandardError=syslog
10 |
11 | [Install]
12 | WantedBy=multi-user.target
13 |
--------------------------------------------------------------------------------
/config/socket_io.ini:
--------------------------------------------------------------------------------
1 | [socket_io]
2 | streamer_name=
3 | namespace=
4 | website_url=
5 | service_name=
6 |
7 | #[connection]
8 | #address=
9 | #publish_address=
10 | #subscribe_address=
11 | #command_address=
12 | #control_address=
13 |
--------------------------------------------------------------------------------
/config/socket_io.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=Socket IO Adpater for Vexbot
3 |
4 | [Service]
5 | Type=simple
6 | ExecStart=/path/to/socket_io/binary /path/to/config
7 | StandardOutput=syslog
8 | StandardError=syslog
9 |
10 | [Install]
11 | WantedBy=multi-user.target
12 |
--------------------------------------------------------------------------------
/config/vexbot.ini:
--------------------------------------------------------------------------------
1 | # version: "0.0.4"
2 | [vexbot]
3 | bot_name='vexbot'
4 |
5 | #[connection]
6 | #address = *
7 | #publish_address = 4000
8 | #subscribe_address = [4001,]
9 | #command_address = 4002
10 | #control_address = 4003
11 |
--------------------------------------------------------------------------------
/config/vexbot.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=Helper robot
3 |
4 | [Service]
5 | Type=simple
6 | ExecStart=/path/to/vexbot/script
7 | StandardOutput=syslog
8 | StandardError=syslog
9 |
10 | [Install]
11 | WantedBy=multi-user.target
12 |
--------------------------------------------------------------------------------
/config/xmpp.ini:
--------------------------------------------------------------------------------
1 | [xmpp]
2 | # local is username
3 | local=
4 | domain=
5 | room=
6 | resource=
7 | nick=
8 | password=
9 | service_name=
10 |
11 | #[connection]
12 | #address=
13 | #publish_address=
14 | #subscribe_address=
15 | #command_address=
16 | #control_address=
17 |
--------------------------------------------------------------------------------
/config/xmpp.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=XMPP Adapter for vexbot
3 |
4 | [Service]
5 | Type=simple
6 | ExecStart=/path/to/xmpp/script /path/to/config/file
7 | StandardOutput=syslog
8 | StandardError=syslog
9 |
10 | [Install]
11 | WantedBy=multi-user.target
12 |
--------------------------------------------------------------------------------
/config/youtube.ini:
--------------------------------------------------------------------------------
1 | [youtube]
2 | client_secret_filepath=
3 | service_name=
4 |
5 | #[connection]
6 | #address =
7 | #publish_address=
8 | #subscribe_address=
9 | #command_address=
10 | #control_address=
11 |
--------------------------------------------------------------------------------
/config/youtube.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=Youtube Adapter for Vexbot
3 |
4 | [Service]
5 | Type=simple
6 | ExecStart=/path/to/youtube/script /path/to/config/file
7 | StandardOutput=syslog
8 | StandardError=syslog
9 |
10 | [Install]
11 | WantedBy=multi-user.target
12 |
--------------------------------------------------------------------------------
/data/add_extensions:
--------------------------------------------------------------------------------
1 | load commands [x,y,z]
2 |
--------------------------------------------------------------------------------
/data/cpu_count:
--------------------------------------------------------------------------------
1 | how many cores do we have
2 | get cpu cores
3 |
--------------------------------------------------------------------------------
/data/get_help:
--------------------------------------------------------------------------------
1 | help me please
2 | what does this do
3 | what can I do
4 | what can you do
5 |
--------------------------------------------------------------------------------
/data/get_installed_extensions:
--------------------------------------------------------------------------------
1 | show me what we've got installed
2 |
--------------------------------------------------------------------------------
/data/get_last_error:
--------------------------------------------------------------------------------
1 | show me the error
2 | what's wrong with you
3 | tell me how you're broken
4 | what's wrong
5 | what happened
6 |
--------------------------------------------------------------------------------
/data/get_log:
--------------------------------------------------------------------------------
1 | get log
2 | get logging
3 | get logging value
4 | get logs
5 | what is the logging
6 | what is the logging?
7 | show me the log
8 | show me the logging
9 | what is the logging value?
10 |
--------------------------------------------------------------------------------
/data/get_services:
--------------------------------------------------------------------------------
1 | what services are up
2 | services up
3 | services available
4 | what is running
5 | running
6 |
--------------------------------------------------------------------------------
/data/restart_process:
--------------------------------------------------------------------------------
1 | restart adapter [x]
2 | restart process [x]
3 | restart program [x]
4 |
--------------------------------------------------------------------------------
/data/start_process:
--------------------------------------------------------------------------------
1 | start adapter [x]
2 | start process [x]
3 | start program [x]
4 |
--------------------------------------------------------------------------------
/data/status_process:
--------------------------------------------------------------------------------
1 | status adapter
2 | status process
3 | status program
4 |
--------------------------------------------------------------------------------
/data/stop_process:
--------------------------------------------------------------------------------
1 | stop adapter [x]
2 | stop process [x]
3 | stop program [x]
4 |
--------------------------------------------------------------------------------
/data/swap:
--------------------------------------------------------------------------------
1 | show me the swap
2 |
--------------------------------------------------------------------------------
/data/virtual_memory_percent:
--------------------------------------------------------------------------------
1 | how much memory are we using
2 | what's our memory utilization
3 |
--------------------------------------------------------------------------------
/data/virtual_memory_used:
--------------------------------------------------------------------------------
1 | how much memory do we have
2 | how much memory
3 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Minimal makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line.
5 | SPHINXOPTS =
6 | SPHINXBUILD = sphinx-build
7 | SPHINXPROJ = vexbot
8 | SOURCEDIR = .
9 | BUILDDIR = _build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
--------------------------------------------------------------------------------
/docs/adapter_configuration.rst:
--------------------------------------------------------------------------------
1 | =====================
2 | Adapter Configuration
3 | =====================
4 |
5 | There are two things that need to be configured for most adapters. The `.service` file which systemd uses to launch the service, and an option configuration file, which can be used to pass in configurations that need to be persisted.
6 |
7 | See `this link`_ for example configurations for the packaged adapters. And the below for a primer on ZMQ addresses, if you desire to change the configuration of anything from running locally on the loopback address.
8 |
9 | .. _`this link`: https://github.com/benhoff/vexbot/tree/master/config
10 |
11 | Please note that the `.service` files should be put in `~/.config/systemd/user/*`. The `.ini` files may be placed almost anywhere, as long as they are referred to properly in the `.service` file, but recommend they be placed in `!/.config/vexbot/*` for consistency.
12 |
13 |
14 | Configuring ZMQ Addresses
15 | -------------------------
16 |
17 | Addresses can be configured for the adapters and the bot itself in the .ini files. This is a bit more advanced and probably not recommended.
18 |
19 | The address expected is in the format of `tcp://[ADDRESS]:[PORT_NUMBER]`.
20 |
21 | For example `tcp://127.0.0.1:5617` is a valid address. 127.0.0.1 is the ADDRESS and 5617 is the PORT_NUMBER.
22 |
23 | 127.0.0.1 was chosen specifially as an example because for IPV4 it is the "localhost". Localhost is the computer the program is being run on. So if you want the program to connect to a socket on your local computer (you probably do), use 127.0.0.1.
24 |
25 | Port numbers range from 0-65536, and can be mostly aribratry chosen. For linux ports 0-1024 are reserved, so best to stay away from those. Port 5555 is usually used as an example port for coding examples, so probably best to stay away from that as well.
26 |
27 | The value of the `publish_address` and `subscribe_address` at the top of the settings file are likely what you want to copy for the `publish_address` and `subscribe_address` under shell, irc, xmpp, youtube, and socket_io if you're running everything locally on one computer. But you don't have to. You could run all the services on one computer and the main robot on a different computer. You would just need to configure the address and ports correctly, as well as work through any networking/port issues going across the local area network (LAN).
28 |
--------------------------------------------------------------------------------
/docs/adapter_development.rst:
--------------------------------------------------------------------------------
1 | ===================
2 | Adapter Development
3 | ===================
4 |
5 | Create A New Adapter
6 | --------------------
7 |
8 | Create a messaging instance and pass in a service name that will uniquely identify it.
9 |
10 | .. code-block:: python
11 |
12 | from vexbot.adapters.messaging import Messaging
13 |
14 | messaging = Messaging('unique_service_name', run_control_loop=True)
15 | messaging.run(blocking=False)
16 | # Your messaging code here
17 |
18 | # Some sort of loop, using a `while` loop for illustration purposes
19 | while True:
20 | author = ''
21 | message = ''
22 | # optional
23 | # channel = ''
24 |
25 | messaging.send_chatter(author=author,
26 | message=message)
27 |
28 | # NOTE: Alternate implementation
29 | """
30 | messaging.send_chatter(author=author,
31 | message=message,
32 | channel=channel)
33 | """
34 |
35 | Dope. But what about something that sends commands to the robot?
36 |
37 | .. code-block:: python
38 |
39 | from vexbot.adapters.messaging import Messaging
40 |
41 | messaging = Messaging('unique_service_name', run_control_loop=True)
42 | messaging.run(blocking=False)
43 |
44 | # Your code here. You would probably want this in a loop as well.
45 | command = ''
46 | args = []
47 | kwargs = {}
48 |
49 | messaging.send_command(command, *args, **kwargs)
50 |
51 | You probably want a response back out of that command, huh?
52 |
53 | .. code-block:: python
54 |
55 | from vexbot.observer import Observer
56 | from vexbot.adapters.messaging import Messaging
57 |
58 | class MyObserver(Observer):
59 | def on_next(self, request):
60 | result = request.kwargs.get('result')
61 | # your code here
62 |
63 | def on_error(self, *args, **kwargs):
64 | pass
65 |
66 | def on_completed(*args, **kwargs):
67 | pass
68 |
69 | messaging = Messaging('unique_service_name', run_control_loop=True)
70 | messaging.run(blocking=False)
71 |
72 | my_observer = MyObserver()
73 |
74 | messaging.command.subscribe(my_observer)
75 | # You can also pass in methods to the `subscribe` method
76 | messaging.command.subscribe(your_custom_method_here)
77 |
78 | Actually you probably want the ability to dynamically load commands, persist your dynamic commands, and see all the installed commands available.
79 |
80 | .. code-block:: python
81 |
82 | import shelve
83 | from os import path
84 | from vexbot.observer import Observer
85 | from vexbot.extensions import extensions
86 |
87 | from vexbot.util.get_cache_filepath import get_cache
88 | from vexbot.util.get_cache_filepath import get_cache_filepath as get_cache_dir
89 |
90 | class MyObserver(Observer):
91 | extensions = (extensions.add_extensions,
92 | extensions.remove_extension,
93 | # NOTE: you can pass in dict's here to change the behavior
94 | {'method': your_method_here,
95 | 'hidden': True,
96 | 'name': 'some_alternate_method_name',
97 | 'alias': ['method_name2',
98 | 'method_name3']},
99 |
100 | extensions.get_extensions,
101 | extensions.get_installed_extensions)
102 |
103 | def __init__(self):
104 | super().__init__()
105 | self._commands = {}
106 | cache_dir = get_cache_dir()
107 | mkdir = not path.isdir(cache_dir)
108 | if mkdir:
109 | os.makedirs(cache_dir, exist_ok=True)
110 |
111 | filepath = get_cache(__name__ + '.pickle')
112 | init = not path.isfile(filepath)
113 |
114 | self._config = shelve.open(filepath, flag='c', writeback=True)
115 |
116 | if init:
117 | self._config['extensions'] = {}
118 | self._config['disabled'] = {}
119 | self._config['modules'] = {}
120 |
121 | # NOTE: Here's our command handeling
122 | def handle_command(self, command: str, *args, **kwargs):
123 | callback = self._commands.get(command)
124 | if callback is None:
125 | return
126 |
127 | # Wrap our callback to catch errors
128 | try:
129 | result = callback(*args, **kwargs)
130 | except Exception as e:
131 | self.on_error(command, e, args, kwargs)
132 |
133 | print(result)
134 |
135 | def on_next(self, request):
136 | # NOTE: Here's our responses back from the bot
137 | result = request.kwargs.get('result')
138 | # your code here
139 |
140 | def on_error(self, *args, **kwargs):
141 | pass
142 |
143 | def on_completed(*args, **kwargs):
144 | pass
145 |
146 | >> observer = MyObserver()
147 | >> observer.handle_command('get_extensions')
148 | >> []
149 | >> observer.handle_command('add_extensions', 'log_level')
150 | >> observer.handle_command('get_extensions')
151 | >> ['log_level']
152 |
--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 | #
4 | # vexbot documentation build configuration file, created by
5 | # sphinx-quickstart on Tue Dec 26 19:40:36 2017.
6 | #
7 | # This file is execfile()d with the current directory set to its
8 | # containing dir.
9 | #
10 | # Note that not all possible configuration values are present in this
11 | # autogenerated file.
12 | #
13 | # All configuration values have a default; values that are commented out
14 | # serve to show the default.
15 |
16 | # If extensions (or modules to document with autodoc) are in another directory,
17 | # add these directories to sys.path here. If the directory is relative to the
18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
19 | #
20 | # import os
21 | # import sys
22 | # sys.path.insert(0, os.path.abspath('.'))
23 |
24 |
25 | # -- General configuration ------------------------------------------------
26 |
27 | # If your documentation needs a minimal Sphinx version, state it here.
28 | #
29 | # needs_sphinx = '1.0'
30 |
31 | # Add any Sphinx extension module names here, as strings. They can be
32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
33 | # ones.
34 | extensions = ['sphinx.ext.autodoc',
35 | 'sphinx.ext.viewcode']
36 |
37 | # Add any paths that contain templates here, relative to this directory.
38 | templates_path = ['_templates']
39 |
40 | # The suffix(es) of source filenames.
41 | # You can specify multiple suffix as a list of string:
42 | #
43 | # source_suffix = ['.rst', '.md']
44 | source_suffix = '.rst'
45 |
46 | # The master toctree document.
47 | master_doc = 'index'
48 |
49 | # General information about the project.
50 | project = 'vexbot'
51 | copyright = '2017, Ben Hoff'
52 | author = 'Ben Hoff'
53 |
54 | # The version info for the project you're documenting, acts as replacement for
55 | # |version| and |release|, also used in various other places throughout the
56 | # built documents.
57 | #
58 | # The short X.Y version.
59 | version = '0.4.0'
60 | # The full version, including alpha/beta/rc tags.
61 | release = '0.4.0'
62 |
63 | # The language for content autogenerated by Sphinx. Refer to documentation
64 | # for a list of supported languages.
65 | #
66 | # This is also used if you do content translation via gettext catalogs.
67 | # Usually you set "language" from the command line for these cases.
68 | language = None
69 |
70 | # List of patterns, relative to source directory, that match files and
71 | # directories to ignore when looking for source files.
72 | # This patterns also effect to html_static_path and html_extra_path
73 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
74 |
75 | # The name of the Pygments (syntax highlighting) style to use.
76 | pygments_style = 'sphinx'
77 |
78 | # If true, `todo` and `todoList` produce output, else they produce nothing.
79 | todo_include_todos = False
80 |
81 |
82 | # -- Options for HTML output ----------------------------------------------
83 |
84 | # The theme to use for HTML and HTML Help pages. See the documentation for
85 | # a list of builtin themes.
86 | #
87 | html_theme = 'alabaster'
88 |
89 | # Theme options are theme-specific and customize the look and feel of a theme
90 | # further. For a list of options available for each theme, see the
91 | # documentation.
92 | #
93 | # html_theme_options = {}
94 |
95 | # Add any paths that contain custom static files (such as style sheets) here,
96 | # relative to this directory. They are copied after the builtin static files,
97 | # so a file named "default.css" will overwrite the builtin "default.css".
98 | html_static_path = ['_static']
99 |
100 | # Custom sidebar templates, must be a dictionary that maps document names
101 | # to template names.
102 | #
103 | # This is required for the alabaster theme
104 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
105 | html_sidebars = {
106 | '**': [
107 | 'relations.html', # needs 'show_related': True theme option to display
108 | 'searchbox.html',
109 | ]
110 | }
111 |
112 |
113 | # -- Options for HTMLHelp output ------------------------------------------
114 |
115 | # Output file base name for HTML help builder.
116 | htmlhelp_basename = 'vexbotdoc'
117 |
118 |
119 | # -- Options for LaTeX output ---------------------------------------------
120 |
121 | latex_elements = {
122 | # The paper size ('letterpaper' or 'a4paper').
123 | #
124 | # 'papersize': 'letterpaper',
125 |
126 | # The font size ('10pt', '11pt' or '12pt').
127 | #
128 | # 'pointsize': '10pt',
129 |
130 | # Additional stuff for the LaTeX preamble.
131 | #
132 | # 'preamble': '',
133 |
134 | # Latex figure (float) alignment
135 | #
136 | # 'figure_align': 'htbp',
137 | }
138 |
139 | # Grouping the document tree into LaTeX files. List of tuples
140 | # (source start file, target name, title,
141 | # author, documentclass [howto, manual, or own class]).
142 | latex_documents = [
143 | (master_doc, 'vexbot.tex', 'vexbot Documentation',
144 | 'Ben Hoff', 'manual'),
145 | ]
146 |
147 |
148 | # -- Options for manual page output ---------------------------------------
149 |
150 | # One entry per manual page. List of tuples
151 | # (source start file, name, description, authors, manual section).
152 | man_pages = [
153 | (master_doc, 'vexbot', 'vexbot Documentation',
154 | [author], 1)
155 | ]
156 |
157 |
158 | # -- Options for Texinfo output -------------------------------------------
159 |
160 | # Grouping the document tree into Texinfo files. List of tuples
161 | # (source start file, target name, title, author,
162 | # dir menu entry, description, category)
163 | texinfo_documents = [
164 | (master_doc, 'vexbot', 'vexbot Documentation',
165 | author, 'vexbot', 'One line description of project.',
166 | 'Miscellaneous'),
167 | ]
168 |
169 |
170 |
171 |
--------------------------------------------------------------------------------
/docs/extension_development.rst:
--------------------------------------------------------------------------------
1 | =====================
2 | Extension Development
3 | =====================
4 |
5 | Foreword
6 | --------
7 |
8 | Currently extensions are tied very closely to python's packaging ecosystem. This is unlikely to change, honestly.
9 |
10 |
11 | Setup a Packging Environment
12 | ----------------------------
13 |
14 | .. code-block:: bash
15 |
16 | $ # Note that the directory these commands are run from is `~/my_vexbot_extensions`!
17 |
18 | $ mkdir my_vexbot_extenstions
19 | $ cd my_vexbot_extensions
20 |
21 | Create a `setup.py` file.
22 |
23 | .. code-block:: python
24 |
25 | # filepath: my_vexbot_extensions/setup.py
26 | from setuptools import setup
27 |
28 | setup(name='my_vexbot_extensions')
29 |
30 | Now take a breather. Python packaging is hard, and we're almost done!
31 |
32 | Extensions Development
33 | ----------------------
34 |
35 | Done with your breather? Good. Now let's say we've got a `hello_world` function that we want to use such as the below.
36 |
37 | .. code-block:: python
38 |
39 | # filepath: my_vexbot_extensions/hello_world.py
40 |
41 | def hello(*args, **kwargs):
42 | print('Hello World!')
43 |
44 | Dope. Let's add this as an extension in our `setup.py` file.
45 |
46 | .. code-block:: python
47 |
48 | # filepath: my_vexbot_extensions/setup.py
49 | from setuptools import setup
50 |
51 | setup(name='my_vexbot_extensions',
52 | entry_points={'vexbot_extensions':['hello=hello_world:hello'])
53 |
54 | That's it. Let's make sure the package is installed on our path. Make sure you're in the directory `my_vexbot_extensions` (or whatever you named your folder).
55 |
56 | .. code-block:: bash
57 |
58 | $ # Note that the directory these commands are run from is `~/my_vexbot_extensions`!
59 |
60 | $ ls
61 | setup.py hello_world.py
62 |
63 | $ python setup.py develop
64 |
65 | You might say, wait, that's not a normal python packaging structure! And you'd be right, so let's look at how it'll change for that. We'll make a directory called `my_src` and create a file in there known as `goodbye_world.py`
66 |
67 | .. code-block:: python
68 |
69 | # filepath: my_vexbot_extensions/my_src/goodbye_world.py
70 |
71 | def angsty(*args, **kwargs):
72 | print('Goodbye, cruel world!')
73 |
74 | Note how I'm taking an arbitrary amount of arguments and key word arguments using `*args and **kwargs`_? You should do that for every extension, or your program will error out at somepoint when it gets called with metadata that it's not ready for.
75 |
76 | .. code-block:: python
77 |
78 | # filepath: my_vexbot_extensions/my_src/goodbye_world.py
79 |
80 | def angsty(*args, **kwargs):
81 | print('Goodbye, cruel world!')
82 |
83 | # Note: Do NOT make extensions without using `*args, **kwargs`
84 | def function_that_will_fail_on_metadata():
85 | print('Please notice the lack of flexibility in this function for taking arguments')
86 | print('This type of extension will inevitabley throw `TypeError` exceptions if put in a codebase')
87 |
88 |
89 | But back to registering our extension in our `setup.py` file. Remember that the filepath for this is `my_vexbot_extensions/my_src/goodbye_world.py`.
90 |
91 | .. code-block:: python
92 |
93 | # filepath: my_vexbot_extensions/setup.py
94 | from setuptools import setup
95 |
96 | setup(name='my_vexbot_extensions',
97 | entry_points={'vexbot_extensions':['hello=hello_world:hello',
98 | 'angsty=my_src.goodbye_world:angsty'])
99 |
100 | Notice how we use the `.` operator to represent the folder directory, and the `:` to specify the method name? That's important.
101 |
102 | We can have multiple methods in a file that way.
103 |
104 | .. code-block:: python
105 |
106 | # filepath: my_vexbot_extensions/my_src/goodbye_world.py
107 |
108 | def angsty(*args, **kwargs):
109 | print('Goodbye, cruel world!')
110 |
111 | def crocodile(*args, **kwargs):
112 | print('Goodbye, crocodile!')
113 |
114 | .. code-block:: python
115 |
116 | # filepath: my_vexbot_extensions/setup.py
117 | from setuptools import setup
118 |
119 | setup(name='my_vexbot_extensions',
120 | entry_points={'vexbot_extensions':['hello=hello_world:hello',
121 | 'angsty=my_src.goodbye_world:angsty',
122 | 'crocodile=my_src.goodbye_world:crocodile'])
123 |
124 | If you have a deeply python nested file, such as one in `my_vexbot_extensions/my_src/how/deep/we/go.py`...
125 | .. code-block:: python
126 |
127 | # filepath: my_vexbot_extensions/setup.py
128 | from setuptools import setup
129 |
130 | setup(name='my_vexbot_extensions',
131 | entry_points={'vexbot_extensions':['hello=hello_world:hello',
132 | 'angsty=my_src.goodbye_world:angsty',
133 | 'crocodile=my_src.goodbye_world:crocodile',
134 | 'nested_func=my_src.how.deep.we.go:waay_to_much'])
135 |
136 | Note that each folder is separated by the `.` operator, and the function name in the above example is `waay_to_much`, which is how deeply I feel that function is nested for a simple example such as this.
137 |
138 | Remember, remember the 5th of November.
139 | And also to re-run `python setup.py develop` once you've added an entry point/extension to your `setup.py` file.
140 |
141 | .. code-block:: bash
142 |
143 | $ # Note that the directory these commands are run from is `~/my_vexbot_extensions`!
144 |
145 | $ ls
146 | setup.py hello_world.py my_src
147 |
148 | $ python setup.py develop
149 |
150 | The string used before the path decleration, I.e the `nested_func` in the string `nested_func=my_src.how.deep.we.go:waay_to_much` is the name you will use in vexbot itself or the `hello` in `hello=hellow_world:hello`.
151 |
152 | Let's add our hello world greeting to our command line interface.
153 |
154 |
155 | .. code-block:: bash
156 |
157 | $ vexbot
158 |
159 | vexbot: !add_extensions hello
160 | vexbot: !hello
161 | Hello World!
162 |
163 | You can also add the hello world to the robot instance as well.
164 |
165 | .. code-block:: bash
166 |
167 | $ vexbot
168 |
169 | vexbot: !add_extensions hello --remote
170 |
171 |
172 | Last, but not least, you can specify command name alias's, specify if the command should be hidden, and give a short description for what the command does by using the `command` decorator.
173 |
174 | .. code-block:: python
175 |
176 | # filepath: my_vexbot_extensions/hello_world.py
177 | from vexbot.commands import command
178 |
179 |
180 | @command(alias=['hello_world', 'world_hello'],
181 | hidden=True, # default is `False`
182 | short='Prints `Hello World`!')
183 | def hello(*args, **kwargs):
184 | print('Hello World!')
185 |
186 | .. _`*args and **kwargs`: https://stackoverflow.com/questions/3394835/args-and-kwargs
187 |
--------------------------------------------------------------------------------
/docs/extension_discovery.rst:
--------------------------------------------------------------------------------
1 | ===================
2 | Extension Discovery
3 | ===================
4 |
5 | See Installed Extensions
6 | ------------------------
7 |
8 | Installed extensions are on your path, but not neccesairly attached to anything running currently
9 |
10 | .. code-block:: bash
11 |
12 | vexbot: !get_installed_extensions
13 |
14 | This command displays a list of every extension installed with a short doc string on what it does.
15 |
16 | For example, the command `power_off` will be displayed as
17 |
18 | 'power_off: Power off a digital ocean droplet'
19 |
20 | You can also use a module name to see all the extensions that are provided by that module. `vexbot.extensions.subprocess` is an installed module for vexbot. To see all the extensions that are provided by that module:
21 |
22 | .. code-block:: bash
23 |
24 | vexbot: !get_installed_modules vexbot.extensions.subprocess
25 |
26 |
27 |
28 | See Installed Modules
29 | ---------------------
30 |
31 | There are a lot of installed extensions and it's hard to figure out what each one does.
32 | You can break them up into modules
33 |
34 | .. code-block:: bash
35 |
36 | vexbot: !get_installed_modules
37 |
38 | This is helpful because the installed modules can be used with the `get_installed_extensions` to narrow down what is shown. For example, the every extensions in the module `vexbot.extensions.digitalocean` can be shown by using the following command:
39 |
40 | .. code-block:: bash
41 |
42 | vexbot: !get_installed_modules vexbot.extensions.digitalocean
43 |
44 | .. TODO See extension code would be helpful. Also implementing typing information. Also seeing the documentation. Also seeing some sort of use documentation
45 |
--------------------------------------------------------------------------------
/docs/extension_management.rst:
--------------------------------------------------------------------------------
1 | ====================
2 | Extension Management
3 | ====================
4 |
5 |
6 | For the most part it's assumed that you are running in the command line program for this documentation.
7 |
8 | .. code-block:: bash
9 |
10 | $ vexbot
11 |
12 | vexbot:
13 |
14 |
15 | See Current Commands
16 | --------------------
17 | .. code-block:: bash
18 |
19 | vexbot: !commands
20 |
21 | See bot commands
22 |
23 | .. code-block:: bash
24 |
25 | vexbot: !commands --remote
26 |
27 |
28 | .. TODO I'm not sure how you get commands for a service right now? Think I might have regressed that functionality during development
29 |
30 | See Extensions In Use
31 | ---------------------
32 |
33 | .. code-block:: bash
34 |
35 | vexbot: !extensions
36 |
37 | From bot:
38 |
39 | .. code-block:: bash
40 |
41 | vexbot: !extensions --remote
42 |
43 | Remove Extensions In Use
44 | ------------------------
45 | Let's say you've the `get_cache` (shows you your configuration cache) and the `cpu_count` extensions in use and you'd like to remove them.
46 |
47 |
48 | .. code-block:: bash
49 |
50 | vexbot: !remove_extension get_cache cpu_count
51 |
52 | Alternatively just removing one:
53 |
54 | .. code-block:: bash
55 |
56 | vexbot: !remove_extension get_cache
57 |
58 | See Installed Extensions
59 | ------------------------
60 |
61 | Installed extensions are on your path, but not neccesairly attached to anything running currently
62 |
63 | .. code-block:: bash
64 |
65 | vexbot: !get_installed_extensions
66 |
67 | This command displays a list of every extension installed with a short doc string on what it does.
68 |
69 | For example, the command `power_off` will be displayed as
70 |
71 | 'power_off: Power off a digital ocean droplet'
72 |
73 | You can also use a module name to see all the extensions that are provided by that module. `vexbot.extensions.subprocess` is an installed module for vexbot. To see all the extensions that are provided by that module:
74 |
75 | .. code-block:: bash
76 |
77 | vexbot: !get_installed_modules vexbot.extensions.subprocess
78 |
79 |
80 |
81 | See Installed Modules
82 | ---------------------
83 |
84 | There are a lot of installed extensions and it's hard to figure out what each one does.
85 | You can break them up into modules
86 |
87 | .. code-block:: bash
88 |
89 | vexbot: !get_installed_modules
90 |
91 | This is helpful because the installed modules can be used with the `get_installed_extensions` to narrow down what is shown. For example, the every extensions in the module `vexbot.extensions.digitalocean` can be shown by using the following command:
92 |
93 | .. code-block:: bash
94 |
95 | vexbot: !get_installed_modules vexbot.extensions.digitalocean
96 |
97 |
98 | Add Extensions
99 | --------------
100 |
101 | .. code-block:: bash
102 |
103 | vexbot: !add_extensions get_code delete_cache
104 |
105 | To add commands to the robot instance:
106 |
107 | .. code-block:: bash
108 |
109 | vexbot: !add_extensions get_code delete_cache --remote
110 |
--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
1 | Welcome to vexbot's documentation!
2 | ==================================
3 |
4 | .. toctree::
5 | :maxdepth: 2
6 | :caption: Contents:
7 |
8 | extension_development
9 | extension_management
10 | extension_discovery
11 | adapter_development
12 | adapter_configuration
13 | packages
14 |
15 |
16 | .. TODO role_management
17 | .. Should commands be called extensions?
18 |
19 |
20 | Installation
21 | ------------
22 |
23 | You will need an active DBus user session bus. Depending on your distro, you might already have one (Arch linux, for example).
24 |
25 | For Ubuntu:
26 |
27 | .. code-block:: bash
28 |
29 | $ apt-get install dbus-user-session python3-gi python3-dev python3-pip build-essential
30 |
31 | For everyone:
32 |
33 | .. code-block:: bash
34 |
35 | $ python3 -m venv
36 |
37 | .. code-block:: bash
38 |
39 | $ source /bin/activate
40 |
41 | .. code-block:: bash
42 |
43 | $ ln -s /usr/lib/python3/dist-packages/gi /lib/python3.5/site-packages/
44 |
45 | .. code-block:: bash
46 |
47 | $ pip install vexbot[process_manager]
48 |
49 | Make sure your virtual environment is activated. Then run:
50 |
51 | .. code-block:: bash
52 |
53 | $ vexbot_generate_certificates
54 |
55 | .. code-block:: bash
56 |
57 | $ vexbot_generate_unit_file
58 |
59 | .. code-block:: bash
60 |
61 | $ systemctl --user daemon-reload
62 |
63 | Your bot is ready to run!
64 |
65 | Running
66 | -------
67 |
68 | .. code-block:: bash
69 |
70 | $ systemctl --user start vexbot
71 |
72 | Or
73 |
74 | .. code-block:: bash
75 |
76 | $ vexbot_robot
77 |
78 | Please note that vexbot has a client/server architecture. The above commands will launch the server. To launch the command line client:
79 |
80 | .. code-block:: bash
81 |
82 | $ vexbot
83 |
84 | Exit the command line client by typing `!exit` or using `ctl+D`.
85 |
86 |
87 | Indices and tables
88 | ==================
89 |
90 | * :ref:`genindex`
91 | * :ref:`modindex`
92 | * :ref:`search`
93 |
--------------------------------------------------------------------------------
/docs/packages.rst:
--------------------------------------------------------------------------------
1 | Packages
2 | ========
3 |
4 | +===================+=========+
5 | | required packages | License |
6 | +===================+=========+
7 | | vexmessage | GPL3 |
8 | +-------------------+---------+
9 | | pyzmq | BSD |
10 | +-------------------+---------+
11 | | rx | Apache |
12 | +-------------------+---------+
13 | | tblib | BSD |
14 | +-------------------+---------+
15 | | tornado | Apache |
16 | +-------------------+---------+
17 |
18 | Optional Packages
19 | -----------------
20 |
21 | +==================+=========+
22 | | nlp | License |
23 | +==================+=========+
24 | | spacy | |
25 | +------------------+---------+
26 | | sklearn | |
27 | +------------------+---------+
28 | | sklearn_crfsuite | |
29 | +------------------+---------+
30 | | wheel | |
31 | +------------------+---------+
32 |
33 | +==================+=========+
34 | | socket_io | License |
35 | +==================+=========+
36 | | requests | |
37 | +------------------+---------+
38 | | websocket-client | |
39 | +------------------+---------+
40 |
41 | +===============+=========+
42 | | summarization | License |
43 | +===============+=========+
44 | | gensim | |
45 | +---------------+---------+
46 | | newspaper3k | |
47 | +---------------+---------+
48 |
49 | +==========================+=========+
50 | | youtube | License |
51 | +==========================+=========+
52 | | google-api-python-client | |
53 | +--------------------------+---------+
54 |
55 | +========+=========+
56 | | dev | License |
57 | +========+=========+
58 | | flake8 | |
59 | +--------+---------+
60 | | twine | |
61 | +--------+---------+
62 | | wheel | |
63 | +--------+---------+
64 |
65 | +===========+=========+
66 | | xmpp | License |
67 | +===========+=========+
68 | | sleekxmpp | |
69 | +-----------+---------+
70 | | dnspython | |
71 | +-----------+---------+
72 |
73 |
74 | +==============+=========+
75 | | process_name | License |
76 | +==============+=========+
77 | | setproctitle | |
78 | +--------------+---------+
79 |
80 |
81 | +==============+=========+
82 | | speechtotext | License |
83 | +==============+=========+
84 | | speechtotext | |
85 | +--------------+---------+
86 |
87 |
88 | +=================+=========+
89 | | process_manager | License |
90 | +=================+=========+
91 | | pydus | |
92 | +-----------------+---------+
93 |
94 |
95 | +=================+=========+
96 | | gui | License |
97 | +=================+=========+
98 | | chatimusmaximus | |
99 | +-----------------+---------+
100 |
101 |
102 | +======+=========+
103 | | irc | License |
104 | +======+=========+
105 | | irc3 | |
106 | +------+---------+
107 |
108 |
109 | +============+=========+
110 | | microphone | License |
111 | +============+=========+
112 | | microphone | |
113 | +------------+---------+
114 |
115 |
116 | +==============+=========+
117 | | speechtotext | License |
118 | +==============+=========+
119 | | speechtotext | |
120 | +--------------+---------+
121 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | import re
3 |
4 | from setuptools import find_packages, setup
5 | from vexbot.extension_metadata import extensions
6 |
7 |
8 | VERSIONFILE = 'vexbot/_version.py'
9 | verstrline = open(VERSIONFILE, 'rt').read()
10 | VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]"
11 | mo = re.search(VSRE, verstrline, re.M)
12 | if mo:
13 | verstr = mo.group(1)
14 | else:
15 | raise RuntimeError("Unable to find version string in {}".format(VERSIONFILE))
16 |
17 |
18 | # directory = os.path.abspath(os.path.dirname(__file__))
19 | """
20 | with open(os.path.join(directory, 'README.rst')) as f:
21 | long_description = f.read()
22 | """
23 |
24 | str_form = '{}={}{}'
25 | extensions_ = []
26 | for name, extension in extensions.items():
27 | extras = extension.get('extras')
28 | if extras is None:
29 | extras = ''
30 | # FIXME: This will error out weirdly if there's not a list
31 | else:
32 | extras = ', '.join(extras)
33 | extras = ' [' + extras + ']'
34 | line = str_form.format(name, extension['path'], extras)
35 | extensions_.append(line)
36 |
37 | setup(
38 | name="vexbot",
39 | version=verstr,
40 | description='Python personal assistant',
41 | # long_description=long_description,
42 | url='https://github.com/benhoff/vexbot',
43 | license='GPL3',
44 | classifiers=[
45 | 'Development Status :: 2 - Pre-Alpha',
46 | 'Intended Audience :: Developers',
47 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
48 | 'Programming Language :: Python :: 3',
49 | 'Programming Language :: Python :: 3.5',
50 | 'Programming Language :: Python :: 3.6',
51 | 'Operating System :: POSIX :: Linux'],
52 |
53 | author='Ben Hoff',
54 | author_email='beohoff@gmail.com',
55 | entry_points={'console_scripts': ['vexbot=vexbot.adapters.shell.__main__:main',
56 | 'vexbot_robot=vexbot.__main__:main',
57 | 'vexbot_irc=vexbot.adapters.irc.__main__:main',
58 | 'vexbot_xmpp=vexbot.adapters.xmpp:main',
59 | 'vexbot_socket_io=vexbot.adapters.socket_io.__main__:main',
60 | 'vexbot_youtube=vexbot.adapters.youtube:main',
61 | 'vexbot_stackoverflow=vexbot.adapters.stackoverflow:main',
62 | 'vexbot_generate_certificates=vexbot.util.generate_certificates:main',
63 | 'vexbot_generate_unit_file=vexbot.util.generate_config_file:main'],
64 | 'vexbot_extensions': extensions_},
65 | packages=find_packages(), # exclude=['docs', 'tests']
66 |
67 | install_requires=[
68 | # 'pluginmanager>=0.4.1',
69 | 'pyzmq',
70 | 'vexmessage>=0.4.0',
71 | 'rx',
72 | 'tblib', # traceback serilization
73 | 'tornado', # zmq asnyc framework
74 | 'prompt_toolkit>=2.0.0', # shell
75 | ],
76 |
77 | extras_require={
78 | 'nlp': ['wheel', 'spacy', 'sklearn', 'sklearn_crfsuite', 'scipy'],
79 | 'socket_io': ['requests', 'websocket-client'],
80 | 'summarization': ['gensim', 'newspaper3k'],
81 | 'youtube': ['google-api-python-client'],
82 | 'dev': ['flake8', 'twine', 'wheel', 'pygments', 'sphinx'],
83 | 'xmpp': ['sleekxmpp', 'dnspython'],
84 | 'process_name': ['setproctitle'],
85 | 'speechtotext': ['speechtotext'],
86 | 'digitalocean': ['python-digitalocean'],
87 | 'process_manager': ['pydbus'],
88 | 'command_line': ['pygments'],
89 | 'microphone': ['microphone'],
90 | 'database': ['vexstorage'],
91 | 'gui': ['chatimusmaximus'],
92 | 'entity': ['duckling'],
93 | 'irc': ['irc3'],
94 | 'system': ['psutil'],
95 | }
96 | )
97 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/tests/__init__.py
--------------------------------------------------------------------------------
/tests/test_command_managers.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import vexmessage
3 | from vexbot.command_managers import (AdapterCommandManager,
4 | BotCommandManager,
5 | CommandManager)
6 |
7 | from vexbot.subprocess_manager import SubprocessManager
8 |
9 |
10 | class Message:
11 | def __init__(self):
12 | self.response = None
13 | self.status = None
14 | self.commands = None
15 |
16 | def send_response(self, *args, **kwargs):
17 | self.response = (args, kwargs)
18 |
19 | def send_status(self, status):
20 | self.status = status
21 |
22 | def send_command(self, *args, **kwargs):
23 | self.commands = [args, kwargs]
24 |
25 |
26 | class MockRobot:
27 | def __init__(self):
28 | self.messaging = Message()
29 | self.subprocess_manager = SubprocessManager()
30 |
31 |
32 | class TestCommandManager(unittest.TestCase):
33 | def setUp(self):
34 | self.command_manager = CommandManager(Message())
35 |
36 | def test_register_command(self):
37 | def _test():
38 | pass
39 |
40 | test_dict = {'blah': _test}
41 | self.command_manager.register_command('test', test_dict)
42 | self.assertIn('test', self.command_manager._commands)
43 | self.assertFalse(self.command_manager.is_command('test'))
44 | self.assertFalse(self.command_manager.is_command('blah'))
45 | self.command_manager.register_command('blah', _test)
46 | self.assertTrue(self.command_manager.is_command('blah'))
47 | # FIXME: ?
48 | self.assertTrue(self.command_manager.is_command('test blah'))
49 |
50 | def test_help(self):
51 | message = vexmessage.Message('target', 'source', 'CMD', command='help')
52 | self.command_manager.parse_commands(message)
53 | response_sent = self.command_manager._messaging.response[1]
54 | self.assertIn('response', response_sent)
55 | message = vexmessage.Message('target',
56 | 'source',
57 | 'CMD',
58 | command='help commands')
59 |
60 | self.command_manager.parse_commands(message)
61 | response_sent = self.command_manager._messaging.response[1]['response']
62 | self.assertIn('help', response_sent)
63 | self.assertIn('commands', response_sent)
64 |
65 | def test_help_specific(self):
66 | message = vexmessage.Message('target',
67 | 'source',
68 | 'CMD',
69 | command='help',
70 | args=['commands'])
71 |
72 | self.command_manager.parse_commands(message)
73 | response = self.command_manager._messaging.response
74 | self.assertIsNotNone(response)
75 |
76 | def test_commands(self):
77 | message = vexmessage.Message('target',
78 | 'source',
79 | 'CMD',
80 | command='commands')
81 |
82 | self.command_manager.parse_commands(message)
83 | response_sent = self.command_manager._messaging.response
84 | answer = response_sent[1]['response']
85 | self.assertIn('commands', answer)
86 | self.assertIn('help', answer)
87 |
88 | def test_is_command_with_call(self):
89 |
90 | def t(*args):
91 | pass
92 |
93 | self.command_manager.register_command('t', t)
94 | called = self.command_manager.is_command('t', True)
95 | self.assertTrue(called)
96 | not_called = self.command_manager.is_command('f', True)
97 | self.assertFalse(not_called)
98 | # FIXME: not sure what actually want here
99 | called_w_args = self.command_manager.is_command('t these are args',
100 | True)
101 |
102 | self.assertTrue(called_w_args)
103 |
104 | def test_get_callback_recursively_none(self):
105 | result = self.command_manager._get_callback_recursively(None)
106 | self.assertEqual(len(result), 3)
107 | for r in result:
108 | self.assertIsNone(r)
109 |
110 | def test_cmd_commands(self):
111 | def t():
112 | pass
113 | self.command_manager.register_command('test', t)
114 | nested = {'test': t}
115 | self.command_manager.register_command('nested', nested)
116 | commands = self.command_manager._cmd_commands(None)
117 | self.assertIn('test', commands)
118 | self.assertIn('nested test', commands)
119 | self.assertNotIn('blah', commands)
120 | # TODO: test `help` and `commands`?
121 | self.assertEqual(len(commands), 4)
122 |
123 |
124 | class MockProcess:
125 | def __init__(self):
126 | self.killed = False
127 |
128 | def poll(self):
129 | return None
130 |
131 | def terminate(self):
132 | pass
133 |
134 | def kill(self):
135 | self.killed = True
136 |
137 |
138 | class TestBotCommandManager(unittest.TestCase):
139 | def setUp(self):
140 | robot = MockRobot()
141 | self.command_manager = BotCommandManager(robot)
142 | self.command_manager._messaging = Message()
143 | self.subprocess_manager = robot.subprocess_manager
144 | self.messaging = self.command_manager._messaging
145 |
146 | def test_settings(self):
147 | message = vexmessage.Message('target',
148 | 'source',
149 | 'CMD',
150 | command='subprocess',
151 | args='settings test',
152 | parsed_args=['test', ])
153 |
154 | self.subprocess_manager.update_settings('test',
155 | {'value': 'test_value'})
156 |
157 | self.command_manager.parse_commands(message)
158 | response = self.messaging.response[1]['response']
159 | self.assertIn('value', response)
160 |
161 | def test_kill(self):
162 | mock = MockProcess()
163 | self.subprocess_manager._subprocess['test'] = mock
164 | message = vexmessage.Message('target',
165 | 'source',
166 | 'CMD',
167 | command='kill test')
168 |
169 | self.command_manager.parse_commands(message)
170 | self.assertTrue(mock.killed)
171 |
172 | def test_killall(self):
173 | def _mock_kill(*args, **kwargs):
174 | # due to scope issues, it's easiest to just tack on to the original
175 | # message to show that it was called correctly
176 | args[0].contents['killed'] = True
177 |
178 | self.subprocess_manager.killall = _mock_kill
179 | self.command_manager._commands['killall'] = _mock_kill
180 |
181 | message = vexmessage.Message('target',
182 | 'source',
183 | 'CMD',
184 | command='killall')
185 |
186 | self.command_manager.parse_commands(message)
187 | self.assertTrue(message.contents.get('killed'))
188 |
189 | def test_restart_bot(self):
190 | pass
191 | """
192 | message = vexmessage.Message('target',
193 | 'source',
194 | 'CMD',
195 | command='restart_bot')
196 |
197 | with self.assertRaises(SystemExit):
198 | self.command_manager.parse_commands(message)
199 | """
200 |
201 | def test_alive(self):
202 | self.subprocess_manager.register('test2', object())
203 | message = vexmessage.Message('tgt', 'source', 'CMD', command='alive')
204 | self.command_manager.parse_commands(message)
205 | commands = self.messaging.commands
206 | self.assertEqual(commands[1].get('command'), 'alive')
207 |
208 | def test_start(self):
209 | # TODO: finish
210 | message = vexmessage.Message('tgt', 'source', 'CMD', command='start')
211 | self.command_manager.parse_commands(message)
212 |
213 | def test_subprocesses(self):
214 | self.subprocess_manager.register('test', object())
215 | message = vexmessage.Message('tgt',
216 | 'source',
217 | 'CMD',
218 | command='subprocesses')
219 |
220 | self.command_manager.parse_commands(message)
221 | response = self.command_manager._messaging.response[1]['response']
222 | self.assertIn('test', response)
223 |
224 | def test_restart(self):
225 | # TODO: finish
226 | message = vexmessage.Message('tgt', 'source', 'CMD', command='restart')
227 | self.command_manager.parse_commands(message)
228 |
229 | def test_terminate(self):
230 | # TODO: finish
231 | message = vexmessage.Message('target',
232 | 'source',
233 | 'CMD',
234 | command='terminate')
235 |
236 | self.command_manager.parse_commands(message)
237 |
238 | def test_running(self):
239 |
240 | self.subprocess_manager._subprocess['test'] = MockProcess()
241 | message = vexmessage.Message('tgt', 'source', 'CMD', command='running')
242 | self.command_manager.parse_commands(message)
243 | response = self.command_manager._messaging.response[1]['response']
244 | self.assertIn('test', response)
245 |
246 |
247 | class TestAdapapterCommandManager(unittest.TestCase):
248 | def setUp(self):
249 | self.command_manager = AdapterCommandManager(Message())
250 |
251 | def test_alive(self):
252 | messaging = self.command_manager._messaging
253 |
254 | self.command_manager._alive()
255 | self.assertIsNotNone(messaging.status)
256 | self.assertEqual(messaging.status, 'CONNECTED')
257 |
--------------------------------------------------------------------------------
/tests/test_messaging.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from time import sleep
3 |
4 | import zmq
5 | from vexmessage import decode_vex_message
6 | from vexbot.messaging import Messaging
7 |
8 |
9 | class TestMessaging(unittest.TestCase):
10 | def __init__(self, *args, **kwargs):
11 | self.subscribe_address = 'tcp://127.0.0.1:4006'
12 | self.publish_address = 'tcp://127.0.0.1:4007'
13 | self.settings = {'subscribe_address': self.subscribe_address,
14 | 'publish_address': self.publish_address}
15 |
16 | context = zmq.Context()
17 | self.messaging = Messaging(self.settings, context)
18 |
19 | self.test_publish_socket = context.socket(zmq.PUB)
20 | self.test_publish_socket.connect(self.publish_address)
21 |
22 | self.test_subscribe_socket = context.socket(zmq.SUB)
23 | self.test_subscribe_socket.connect(self.subscribe_address)
24 | self.test_subscribe_socket.setsockopt_string(zmq.SUBSCRIBE, '')
25 | sleep(.2)
26 | super().__init__(*args, **kwargs)
27 |
28 | """
29 | def test_send_message(self):
30 | self.messaging.send_message('', test='blue')
31 | frame = self.test_subscribe_socket.recv_multipart()
32 | message = decode_vex_message(frame)
33 | self.assertEqual(message.target, '')
34 | self.assertEqual(message.source, 'robot')
35 | self.assertEqual(message.contents['test'], 'blue')
36 | self.assertEqual(message.type, 'MSG')
37 | """
38 |
39 | def test_send_targeted_message(self):
40 | self.test_subscribe_socket.setsockopt_string(zmq.SUBSCRIBE, 'test')
41 | self.messaging.send_message('test', test='blue')
42 | frame = self.test_subscribe_socket.recv_multipart()
43 | message = decode_vex_message(frame)
44 | self.assertEqual(message.target, 'test')
45 | self.assertEqual(message.source, 'robot')
46 | self.assertEqual(message.contents['test'], 'blue')
47 | self.assertEqual(message.type, 'MSG')
48 |
--------------------------------------------------------------------------------
/tests/test_subprocess_manager.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | from vexbot.subprocess_manager import SubprocessManager
4 |
5 |
6 | class TestSubprocessManager(unittest.TestCase):
7 | def setUp(self):
8 | self.subprocess_manager = SubprocessManager()
9 | # register a subprocess? With a name?
10 |
11 | def test_registered_subprocesses(self):
12 | test_obj = object()
13 | self.subprocess_manager.register('test', test_obj)
14 | registered = self.subprocess_manager.registered_subprocesses()
15 | self.assertIn('test', registered)
16 |
17 | def test_register_with_blacklist(self):
18 | blacklisted = 'black'
19 | self.subprocess_manager.blacklist.append(blacklisted)
20 | self.subprocess_manager.register(blacklisted, None)
21 | registered = self.subprocess_manager.registered_subprocesses()
22 | self.assertNotIn(blacklisted, registered)
23 |
24 | def test_update_setting_value(self):
25 | self.subprocess_manager.update_settings('test', {'test_value': 'old'})
26 | self.subprocess_manager.update_setting_value('test',
27 | 'test_value',
28 | 'new')
29 |
30 | result = self.subprocess_manager.get_settings('test')
31 | self.assertEqual(result['test_value'], 'new')
32 | self.assertNotEqual(result['test_value'], 'old')
33 |
34 | def test_update_setting(self):
35 | settings = {'setting': 'value'}
36 | self.subprocess_manager.update_settings('test', settings)
37 | gotten_settings = self.subprocess_manager.get_settings('test')
38 | self.assertEqual(settings, gotten_settings)
39 |
--------------------------------------------------------------------------------
/vexbot/__init__.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from vexbot._version import __version__ # flake8: noqa
3 |
4 | # FIXME: probably don't want to add NullHandler here
5 | # This is here to prevent the default handler being added upon init of the program
6 | logging.getLogger(__name__).addHandler(logging.NullHandler())
7 |
8 |
9 | def _port_configuration_helper(configuration: dict) -> dict:
10 | default_port_config = _get_default_port_config()
11 |
12 | # get the port settings out of the configuration, falling back on defaults
13 | port_config = configuration.get('connection', default_port_config)
14 |
15 | # update the defaults with the retrived port configs
16 | default_port_config.update(port_config)
17 | # Overwrite the port config to be the updated default port config dict
18 | port_config = default_port_config
19 |
20 | return port_config
21 |
22 |
23 | # TODO: Rename this to be `_get_default_robot_port_config` or similar
24 | def _get_default_port_config() -> dict:
25 | """
26 | protocol: 'tcp'
27 | address: '*'
28 | chatter_publish_port: 4000
29 | chatter_subscription_port: [4001,]
30 | command_port: 4002
31 | request_port: 4003
32 | control_port: 4005
33 | """
34 | # Setup some default port configurations
35 | default_port_config= {'protocol': 'tcp',
36 | 'address': '*',
37 | 'chatter_publish_port': 4000,
38 | # FIXME: There's probably not a good reason to have
39 | # this be an iterable
40 | 'chatter_subscription_port': [4001,],
41 | 'request_port': 4003,
42 | 'command_port': 4004,
43 | 'control_port': 4005}
44 |
45 | return default_port_config
46 |
47 |
48 | def _get_default_adapter_config() -> dict:
49 | """
50 | Returns:
51 | dict: The default adapter config for a locally run helper
52 | {
53 | protocol: 'tcp'
54 | address: '127.0.0.1'
55 | chatter_publish_port: 4000
56 | chatter_subscription_port: [4001,]
57 | command_port: 4002
58 | request_port: 4003
59 | control_port: 4005
60 | }
61 | """
62 | config = _get_default_port_config()
63 | config['address'] = '127.0.0.1'
64 | return config
65 |
--------------------------------------------------------------------------------
/vexbot/__main__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # import atexit
4 |
5 | try:
6 | import setproctitle as _setproctitle
7 | except ImportError:
8 | _setproctitle = False
9 |
10 | from vexbot import _port_configuration_helper
11 |
12 | from vexbot.robot import Robot
13 | from vexbot.util.get_config_filepath import get_config_filepath
14 | from vexbot.util.get_kwargs import get_kwargs as _get_kwargs
15 | from vexbot.util.get_config import get_config as _get_config
16 |
17 |
18 | def _configuration_sane_defaults(configuration: dict) -> (dict, str):
19 | default_vexbot_settings = {'bot_name': 'vexbot'}
20 | # Get the settings out of the configuration, falling back on the defaults
21 | vexbot_settings = configuration.get('vexbot', default_vexbot_settings)
22 | # Get the robot name out of the configuration, falling back on the default
23 | robot_name = vexbot_settings.get('bot_name', 'vexbot')
24 |
25 | return robot_name
26 |
27 |
28 | def main(*args, **kwargs):
29 | """
30 | `kwargs`:
31 |
32 | `configuration_filepath`: filepath for the `ini` configuration
33 | """
34 | kwargs = {**kwargs, **_get_kwargs()}
35 | # FIXME: This filepath handeling is messed up and not transparent as it should be
36 | default_filepath = get_config_filepath()
37 | configuration_filepath = kwargs.get('configuration_filepath')
38 | if configuration_filepath is None:
39 | configuration_filepath = default_filepath
40 | # configuration is from an `ini` file
41 | configuration = _get_config(configuration_filepath)
42 | # setup some sane defaults
43 | robot_name = _configuration_sane_defaults(configuration)
44 | # Get the port configuration out of the configuration
45 | port_config = _port_configuration_helper(configuration)
46 | # create the settings manager using the port config
47 | if _setproctitle:
48 | _setproctitle.setproctitle('vexbot')
49 |
50 | robot = Robot(robot_name, port_config)
51 | robot.run()
52 |
53 |
54 | if __name__ == "__main__":
55 | main()
56 |
--------------------------------------------------------------------------------
/vexbot/_logging.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from tblib import Traceback
3 |
4 |
5 | class LoopPubHandler(logging.Handler):
6 | def __init__(self, messaging, level=logging.NOTSET):
7 | super().__init__(level)
8 | self.messaging = messaging
9 |
10 | def emit(self, record):
11 | args = record.args
12 | if isinstance(args, tuple):
13 | args = [str(x) for x in record.args]
14 | elif isinstance(args, dict):
15 | args = str(args)
16 | # NOTE: Might need more specific handling in the future
17 | else:
18 | args = str(args)
19 |
20 | info = {'name': record.name,
21 | 'level': record.levelno,
22 | 'pathname': record.pathname,
23 | 'lineno': record.lineno,
24 | 'msg': record.msg,
25 | 'args': args,
26 | 'exc_info': record.exc_info,
27 | 'func': record.funcName,
28 | 'sinfo': record.stack_info,
29 | 'type': 'log'}
30 | exc_info = info['exc_info']
31 |
32 | if exc_info is not None:
33 | # FIXME: It's hard to transmit an exception because the Error type
34 | # might be defined in a library that is not installed on the
35 | # receiving side. Not sure what the best way to do this is.
36 | info['exc_info'] = Traceback(exc_info[2]).to_dict()
37 | """
38 | new_exc_info = []
39 | first = exc_info[0]
40 | if isinstance(first, Exception):
41 | try:
42 | first = first.__name__
43 | except AttributeError:
44 | first = first.__class__.__name__
45 | print(first, dir(first), isinstance(first, Exception))
46 | new_exc_info.append(first)
47 | new_exc_info.append([str(x) for x in exc_info[1].args])
48 | new_exc_info.append(Traceback(exc_info[2]).to_dict())
49 | info['exc_info'] = new_exc_info
50 | """
51 |
52 |
53 | self.messaging.send_log(**info)
54 |
55 |
56 | class MessagingLogger:
57 | def __init__(self, service_name: str):
58 | name = service_name + '.messaging'
59 | self.command = logging.getLogger(name + '.command')
60 | self.control = logging.getLogger(name + '.control')
61 | self.publish = logging.getLogger(name + '.publish')
62 | self.subscribe = logging.getLogger(name + '.subscribe')
63 | self.request = logging.getLogger(name + '.request')
64 |
--------------------------------------------------------------------------------
/vexbot/_version.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.4.5'
2 |
--------------------------------------------------------------------------------
/vexbot/adapters/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/vexbot/adapters/__init__.py
--------------------------------------------------------------------------------
/vexbot/adapters/irc/__init__.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | import argparse
3 | import logging
4 | import pkg_resources
5 |
6 | from threading import Thread
7 | try:
8 | import irc3
9 | except ImportError:
10 | irc3 = False
11 |
12 | if not irc3:
13 | logging.exception('irc3 not installed!')
14 | raise ImportError('irc3 is not installed. Install irc3 using `pip install '
15 | 'irc3` on the command line')
16 |
17 | from vexbot._logging import LoopPubHandler
18 | from vexbot.adapters.messaging import Messaging as _Messaging
19 | from vexbot.adapters.irc.observer import IrcObserver as _IrcObserver
20 |
21 |
22 | class IrcInterface:
23 | def __init__(self,
24 | service_name: str,
25 | irc_config: dict=None,
26 | connection: dict=None,
27 | **kwargs):
28 |
29 |
30 | if connection is None:
31 | connection = {}
32 |
33 | self.messaging = _Messaging(service_name, run_control_loop=True, **connection)
34 |
35 | self.root_handler = LoopPubHandler(self.messaging)
36 | self.root_logger = logging.getLogger()
37 | self.root_logger.addHandler(self.root_handler)
38 |
39 | self._scheduler_thread = Thread(target=self.messaging.start,
40 | daemon=True)
41 |
42 | self.irc_bot = irc3.IrcBot.from_config(irc_config)
43 | # Duck type messaging to irc bot. Could also subclass `irc3.IrcBot`,
44 | # but seems like overkill
45 | self.irc_bot.messaging = self.messaging
46 | self.command_parser = _IrcObserver(self.irc_bot, self.messaging, self)
47 | self.messaging.command.subscribe(self.command_parser)
48 |
49 | def run(self):
50 | self._scheduler_thread.start()
51 |
52 | self.irc_bot.create_connection()
53 | self.irc_bot.add_signal_handlers()
54 | event_loop = asyncio.get_event_loop()
55 |
56 | """
57 | handle_close = _handle_close(messaging,
58 | event_loop)
59 | signal.signal(signal.SIGINT, handle_close)
60 | signal.signal(signal.SIGTERM, handle_close)
61 | """
62 | event_loop.run_forever()
63 | event_loop.close()
64 |
--------------------------------------------------------------------------------
/vexbot/adapters/irc/__main__.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import atexit
3 | import signal
4 | from vexbot._version import __version__ as version
5 |
6 | from vexbot.adapters.irc import IrcInterface
7 |
8 | """
9 | try:
10 | pkg_resources.get_distribution('irc3')
11 | except pkg_resources.DistributionNotFound:
12 | _IRC3_INSTALLED = False
13 |
14 | if _IRC3_INSTALLED:
15 | import irc3
16 |
17 | else:
18 | pass
19 | """
20 |
21 | import irc3
22 | from irc3 import utils
23 |
24 |
25 | def main(**kwargs):
26 | """
27 | if not _IRC3_INSTALLED:
28 | logging.error('vexbot_irc requires `irc3` to be installed. Please install '
29 | 'using `pip install irc3`')
30 |
31 | sys.exit(1)
32 | """
33 |
34 | config = _from_argv(irc3.IrcBot, kwargs=kwargs)
35 | if not 'includes' in config:
36 | config['includes'] = []
37 |
38 | message_plug = 'vexbot.adapters.irc.echo_to_message'
39 | if not message_plug in config['includes']:
40 | config['includes'].append(message_plug)
41 | service_name = config.get('service_name', 'irc')
42 | connection = config.get('connection', {})
43 | interface = IrcInterface(service_name, irc_config=config, connection=connection)
44 |
45 | interface.run()
46 | sys.exit()
47 |
48 |
49 | # NOTE: This code is from `irc3`
50 | def _from_argv(cls, argv=None, **kwargs) -> dict:
51 | prog = cls.server and 'irc3d' or 'irc3'
52 | # TODO: Add in publish ports and all that jazz.
53 | doc = """
54 | Run an __main__.py instance from a config file
55 |
56 | Usage: __main__.py [options] ...
57 |
58 | Options:
59 | -h, --help Display this help and exit
60 | --version Output version information and exit
61 | --logdir DIRECTORY Log directory to use instead of stderr
62 | --logdate Show datetimes in console output
63 | --host HOST Server name or ip
64 | --port PORT Server port
65 | -v,--verbose Increase verbosity
66 | -r,--raw Show raw irc log on the console
67 | -d,--debug Add some debug commands/utils
68 | """
69 | import os
70 | import docopt
71 | import textwrap
72 | args = argv or sys.argv[1:]
73 | args = docopt.docopt(doc, args, version=version)
74 | cfg = utils.parse_config(
75 | cls.server and 'server' or 'bot', *args[''])
76 | cfg.update(
77 | verbose=args['--verbose'],
78 | debug=args['--debug'],
79 | )
80 | cfg.update(kwargs)
81 | if args['--host']: # pragma: no cover
82 | host = args['--host']
83 | cfg['host'] = host
84 | if host in ('127.0.0.1', 'localhost'):
85 | cfg['ssl'] = False
86 | if args['--port']: # pragma: no cover
87 | cfg['port'] = args['--port']
88 | if args['--logdir'] or 'logdir' in cfg:
89 | logdir = os.path.expanduser(args['--logdir'] or cfg.get('logdir'))
90 | cls.logging_config = config.get_file_config(logdir)
91 | if args['--logdate']: # pragma: no cover
92 | fmt = cls.logging_config['formatters']['console']
93 | fmt['format'] = config.TIMESTAMPED_FMT
94 | if args.get('--help-page'): # pragma: no cover
95 | for v in cls.logging_config['handlers'].values():
96 | v['level'] = 'ERROR'
97 | if args['--raw']:
98 | cfg['raw'] = True
99 |
100 | return cfg
101 |
102 |
103 | if __name__ == '__main__':
104 | main()
105 |
--------------------------------------------------------------------------------
/vexbot/adapters/irc/echo_to_message.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import irc3
3 |
4 |
5 | @irc3.plugin
6 | class EchoToMessage:
7 | requires = ['irc3.plugins.core',
8 | 'irc3.plugins.command']
9 |
10 | def __init__(self, bot):
11 | self.bot = bot
12 | self.logger = logging.getLogger(__name__)
13 |
14 | @irc3.event(irc3.rfc.PRIVMSG)
15 | def message(self, mask, event, target, data):
16 | nick = mask.nick
17 | nick = str(nick)
18 | message = str(data)
19 | target = str(target)
20 | self.logger.info(' message %s %s %s', nick, message, target)
21 | self.bot.messaging.send_chatter(author=nick,
22 | message=message,
23 | channel=target)
24 |
--------------------------------------------------------------------------------
/vexbot/adapters/irc/observer.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import inspect as _inspect
3 | import logging
4 |
5 | from vexmessage import Request
6 | from vexbot.observer import Observer
7 | from vexbot.extensions import develop, admin
8 | from vexbot.extensions import help as vexhelp
9 |
10 |
11 | class IrcObserver(Observer):
12 | extensions = (develop.get_code, vexhelp.help, admin.get_commands)
13 | def __init__(self, bot, messaging, irc_interface):
14 | super().__init__()
15 | self.bot = bot
16 | self.messaging = messaging
17 | self.irc_interface = irc_interface
18 | self._commands = self._get_commands()
19 | self.logger = logging.getLogger(__name__)
20 |
21 | def _get_commands(self) -> dict:
22 | result = {}
23 | for name, method in _inspect.getmembers(self):
24 | if name.startswith('do_'):
25 | result[name[3:]] = method
26 |
27 | return result
28 |
29 | def do_log_level(self, *args, **kwargs):
30 | if not args:
31 | # FIXME
32 | return self.irc_interface.root_logger.level
33 |
34 | def do_debug(self, *args, **kwargs):
35 | self.irc_interface.root_logger.setLevel(logging.DEBUG)
36 | self.irc_interface.root_handler.setLevel(logging.DEBUG)
37 |
38 | def do_info(self, *args, **kwargs):
39 | self.irc_interface.root_logger.setLevel(logging.INFO)
40 | self.irc_interface.root_handler.setLevel(logging.INFO)
41 |
42 | def do_MSG(self, message, channel, *args, **kwargs):
43 | msg_target = kwargs.get('msg_target')
44 | if msg_target:
45 | message = msg_target + ', ' + message
46 | self.bot.privmsg(channel, message)
47 |
48 | def do_join(self, *args, **kwargs):
49 | self.bot.join(*args)
50 |
51 | def do_part(self, *args, **kwargs):
52 | self.bot.part(*args)
53 |
54 | def do_kick(self, channel, target, reason=None, *args, **kwargs):
55 | self.bot.kick(channel, target, reason)
56 |
57 | def do_away(self, message=None, *args, **kwargs):
58 | self.bot.away(message)
59 |
60 | def do_unaway(self, *args, **kwargs):
61 | self.bot.unaway()
62 |
63 | def do_topic(self, channel, topic=None, *args, **kwargs):
64 | self.bot.topic(channel, topic)
65 |
66 | def do_get_nick(self, *args, **kwargs):
67 | return self.bot.get_nick()
68 |
69 | def do_get_ip(self, *args, **kwargs):
70 | return str(self.bot.ip)
71 |
72 | def on_next(self, item: Request):
73 | command = item.command
74 | args = item.args
75 | kwargs = item.kwargs
76 | self.logger.debug(' command recieved: %s %s %s', command, args, kwargs)
77 | try:
78 | callback = self._commands[command]
79 | except KeyError:
80 | self.logger.info(' command not found: %s', command)
81 | return
82 |
83 | try:
84 | result = callback(*args, **kwargs)
85 | except Exception as e:
86 | self.on_error(e, command, args)
87 | return
88 |
89 | if result is None:
90 | self.logger.info('no result for callback: %s', command)
91 | return
92 |
93 | source = item.source
94 | # NOTE: probably need more here
95 | self.logger.debug(' send command response %s %s %s', source, command, result)
96 | service = self.messaging._service_name
97 | self.messaging.send_command_response(source, command, result=result, service=service, *args, **kwargs)
98 |
99 | def on_completed(self, *args, **kwargs):
100 | pass
101 |
102 | def on_error(self, *args, **kwargs):
103 | self.logger.exception('command failed')
104 |
--------------------------------------------------------------------------------
/vexbot/adapters/shell/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/vexbot/adapters/shell/__init__.py
--------------------------------------------------------------------------------
/vexbot/adapters/shell/__main__.py:
--------------------------------------------------------------------------------
1 | from vexbot.adapters.shell.shell import Shell
2 |
3 |
4 | def main(**kwargs):
5 | shell = Shell(**kwargs)
6 | return shell.run()
7 |
8 |
9 | if __name__ == '__main__':
10 | main()
11 |
--------------------------------------------------------------------------------
/vexbot/adapters/shell/completers.py:
--------------------------------------------------------------------------------
1 | import re
2 | from six import string_types
3 | from prompt_toolkit.completion import Completer, Completion
4 |
5 |
6 | _WORD = re.compile(r'([!#a-zA-Z0-9_]+|[^!#a-zA-Z0-9_\s]+)')
7 |
8 |
9 | def _get_previous_word(document):
10 | # reverse the text before the curosr, in order to do an efficient
11 | # backwards search
12 | text_before_cursor = document.text_before_cursor[::-1]
13 | iterator = _WORD.finditer(text_before_cursor)
14 | count = 1
15 | try:
16 | for i, match in enumerate(iterator):
17 | if i == 0 and match.start(1) == 0:
18 | count += 1
19 | if i + 1 == count:
20 | return (-match.end(1), -match.start(1))
21 | except StopIteration:
22 | pass
23 | return None, None
24 |
25 |
26 | class ServiceCompleter(Completer):
27 | def __init__(self, command_completer: Completer):
28 | self._command_completer = command_completer
29 | self.service_command_map = {}
30 |
31 | def _handle_services(self, previous_word: str, document, complete_event):
32 | if previous_word in self.service_command_map:
33 | return self.service_command_map[previous_word].get_completions(document, complete_event)
34 | else:
35 | return self._command_completer.get_completions(document, complete_event)
36 |
37 | def get_completions(self, document, complete_event):
38 | start, end = _get_previous_word(document)
39 | if start is None:
40 | return self._command_completer.get_completions(document, complete_event)
41 |
42 | offset = document.cursor_position
43 | start += offset
44 | end += offset
45 | previous_word = document.text[start:end]
46 |
47 | if previous_word in self.service_command_map:
48 | return self._handle_services(previous_word, document, complete_event)
49 | else:
50 | return self._command_completer.get_completions(document, complete_event)
51 |
52 | def set_service_completer(self, service: str, completer: Completer):
53 | self.service_command_map[service] = completer
54 |
55 |
56 | class WordCompleter(Completer):
57 | """
58 | Simple autocompletion on a list of words.
59 | :param words: List of words.
60 | :param ignore_case: If True, case-insensitive completion.
61 | :param meta_dict: Optional dict mapping words to their meta-information.
62 | :param WORD: When True, use WORD characters.
63 | :param sentence: When True, don't complete by comparing the word before the
64 | cursor, but by comparing all the text before the cursor. In this case,
65 | the list of words is just a list of strings, where each string can
66 | contain spaces. (Can not be used together with the WORD option.)
67 | :param match_middle: When True, match not only the start, but also in the
68 | middle of the word.
69 | """
70 | def __init__(self, words, ignore_case=False, meta_dict=None, WORD=False,
71 | sentence=False, match_middle=False):
72 | assert not (WORD and sentence)
73 |
74 | self.words = set(words)
75 | self.ignore_case = ignore_case
76 | self.meta_dict = meta_dict or {}
77 | self.WORD = WORD
78 | self.sentence = sentence
79 | self.match_middle = match_middle
80 | assert all(isinstance(w, string_types) for w in self.words)
81 |
82 | def get_completions(self, document, complete_event):
83 | # Get word/text before cursor.
84 | if self.sentence:
85 | word_before_cursor = document.text_before_cursor
86 | else:
87 | word_before_cursor = document.get_word_before_cursor(WORD=self.WORD)
88 |
89 | if self.ignore_case:
90 | word_before_cursor = word_before_cursor.lower()
91 |
92 | def word_matches(word):
93 | """ True when the word before the cursor matches. """
94 | if self.ignore_case:
95 | word = word.lower()
96 |
97 | if self.match_middle:
98 | return word_before_cursor in word
99 | else:
100 | return word.startswith(word_before_cursor)
101 |
102 | for a in self.words:
103 | if word_matches(a):
104 | display_meta = self.meta_dict.get(a, '')
105 | yield Completion(a, -len(word_before_cursor), display_meta=display_meta)
106 |
--------------------------------------------------------------------------------
/vexbot/adapters/shell/intents.py:
--------------------------------------------------------------------------------
1 | import inspect as _inspect
2 |
3 | class CommandItents:
4 | def get_intents(self) -> dict:
5 | result = {}
6 | for name, method in _inspect.getmembers(self):
7 | if name.startswith('do_'):
8 | result[name[3:]] = method
9 | elif method._vex_intent:
10 | result[method._vex_intent_name] = method
11 |
12 | return result
13 |
14 | def do_stop_chatter(self):
15 | pass
16 |
17 | def do_start_chatter(self):
18 | pass
19 |
20 | def do_change_color(self):
21 | pass
22 |
--------------------------------------------------------------------------------
/vexbot/adapters/shell/interfaces.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 | from vexbot.adapters.shell.observers import AuthorObserver
4 | from vexbot.adapters.shell.observers import ServiceObserver
5 | from vexbot.util.lru_cache import LRUCache as _LRUCache
6 |
7 |
8 | def _add_word(completer):
9 | """
10 | Used to add words to the completors
11 | """
12 | def inner(word: str):
13 | completer.words.add(word)
14 | return inner
15 |
16 |
17 | def _remove_word(completer):
18 | """
19 | Used to remove words from the completors
20 | """
21 | def inner(word: str):
22 | try:
23 | completer.words.remove(word)
24 | except Exception:
25 | pass
26 | return inner
27 |
28 |
29 |
30 | class AuthorInterface:
31 | def __init__(self, word_completer, messaging):
32 | self.author_observer = AuthorObserver(self)
33 |
34 | messaging.chatter.subscribe(self.author_observer)
35 | self.authors = _LRUCache(100,
36 | _add_word(word_completer),
37 | _remove_word(word_completer))
38 | self.author_metadata = _LRUCache(100)
39 | self.metadata_words = ['channel', ]
40 |
41 | def add_author(self, author: str, source: str, **metadata: dict):
42 | # NOTE: this fixes the parsing for usernames
43 | author = author.replace(' ', '_')
44 |
45 | self.authors[author] = source
46 | self.author_metadata[author] = {k: v for k, v in metadata.items()
47 | if k in self.metadata_words}
48 |
49 | def is_author(self, author: str):
50 | return author in self.authors
51 |
52 | def get_metadata(self, author: str, kwargs: dict) -> dict:
53 | source = self.author_observer.authors[author]
54 | metadata = self.author_observer.author_metadata[author]
55 | # TODO: Determine if this makes the most sense to update in
56 | metadata.update(kwargs)
57 | kwargs = metadata
58 | kwargs['target'] = source
59 | kwargs['msg_target'] = author
60 | return kwargs
61 |
62 |
63 | class ServiceInterface:
64 | def __init__(self, word_completer, messaging):
65 | self.service_observer = ServiceObserver(self)
66 | messaging.chatter.subscribe(self.service_observer)
67 |
68 | self.services = set()
69 | self.channels = _LRUCache(100,
70 | _add_word(word_completer),
71 | _remove_word(word_completer))
72 |
73 | def add_service(self, source: str, channel: str=None):
74 | if source not in self.services:
75 | self.services.add(source)
76 | # FIXME: hack to append word to completer
77 | self.channels.add_callback(source)
78 |
79 | if channel is not None and channel not in self.channels:
80 | self.channels[channel] = source
81 |
82 | def is_service(self, service: str):
83 | in_service = service in self.services
84 | in_channel = service in self.channels
85 | return in_service or in_channel
86 |
87 |
88 | def get_metadata(self, service: str, kwargs: dict) -> dict:
89 | if service in self.channels:
90 | channel = service
91 | kwargs['channel'] = channel
92 | service = self.channels[channel]
93 | kwargs['target'] = service
94 | return kwargs
95 |
96 |
97 | class EntityInterface:
98 | def __init__(self, author_interface, service_interface):
99 | self.author_interface = author_interface
100 | self.service_interface = service_interface
101 |
102 | def get_entities(self, text: str):
103 | result = []
104 | for service in self.service_interface.services:
105 | for match in re.finditer(service, text):
106 | s = {'start': match.start(),
107 | 'end': match.end(),
108 | 'name': 'service',
109 | 'value': service}
110 | result.append(s)
111 | for channel in self.service_interface.channels.keys():
112 | for match in re.finditer(channel, text):
113 | c = {'start': match.start(),
114 | 'end': match.end(),
115 | 'name': 'channel',
116 | 'value': channel}
117 | result.append(c)
118 |
119 | for author in self.author_interface.authors.keys():
120 | for match in re.finditer(author, text):
121 | a = {'start': match.start(),
122 | 'end': match.end(),
123 | 'name': 'author',
124 | 'value': author}
125 | result.append(a)
126 |
127 | return result
128 |
--------------------------------------------------------------------------------
/vexbot/adapters/shell/parser.py:
--------------------------------------------------------------------------------
1 | def _is_kwarg(string: str):
2 | if string.startswith('-') or string.startswith('--') and not string == '-':
3 | return True
4 | return False
5 |
6 |
7 | def parse(strings: list):
8 | args = []
9 |
10 | # Parse args first
11 | for string in strings:
12 | if _is_kwarg(string):
13 | break
14 | args.append(string)
15 | # Remove the args we've parsed
16 | strings = strings[len(args):]
17 |
18 | kwargs = {}
19 | while strings:
20 | string = strings.pop(0)
21 | if string.startswith('--'):
22 | string = string[2:]
23 | elif string.startswith('-'):
24 | string = string[1:]
25 | try:
26 | value = strings[0]
27 | except IndexError:
28 | kwargs[string] = True
29 | break
30 |
31 | if not _is_kwarg(value):
32 | kwargs[string] = strings.pop(0)
33 | else:
34 | kwargs[string] = True
35 | continue
36 |
37 | # Check for list by checking if the next value has `--` or `-`
38 | try:
39 | value = strings[0]
40 | except IndexError:
41 | continue
42 |
43 | while not _is_kwarg(value):
44 | # if we make it here, than we're in a list loop
45 | value = strings.pop(0)
46 |
47 | # try to append the value onto the reference for the string
48 | try:
49 | kwargs[string].append(value)
50 | # if we fail, then it's probably a string and needs to be converted
51 | # to a list
52 | except AttributeError:
53 | kwargs[string] = [kwargs[string], value]
54 |
55 | # Move on to the next one, checking to see if we're at the end
56 | try:
57 | value = strings[0]
58 | except IndexError:
59 | break
60 |
61 | for k, v in dict(kwargs).items():
62 | if v in ('true', 'True'):
63 | kwargs[k] = True
64 | continue
65 | if v in ('False', 'false'):
66 | kwargs[k] = False
67 | continue
68 | try:
69 | v = int(v)
70 | kwargs[k] = v
71 | continue
72 | except ValueError:
73 | pass
74 | try:
75 | v = float(v)
76 | kwargs[k] = v
77 | # NOTE: End of the line
78 | except ValueError:
79 | continue
80 |
81 | return args, kwargs
82 |
--------------------------------------------------------------------------------
/vexbot/adapters/shell/shell.py:
--------------------------------------------------------------------------------
1 | import re
2 | import time
3 | from os import path
4 | import shlex as _shlex
5 | import pprint as _pprint
6 | import logging
7 | from threading import Thread as _Thread
8 |
9 | from prompt_toolkit.shortcuts import PromptSession
10 | from prompt_toolkit.history import FileHistory
11 | from prompt_toolkit.patch_stdout import patch_stdout
12 | from zmq.eventloop.ioloop import PeriodicCallback
13 |
14 | from vexbot.adapters.shell.parser import parse as _parse
15 | from vexbot.adapters.messaging import Messaging as _Messaging
16 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath
17 | from vexbot.adapters.shell.interfaces import (AuthorInterface,
18 | ServiceInterface,
19 | EntityInterface)
20 |
21 | from vexbot.adapters.shell.observers import (PrintObserver,
22 | CommandObserver,
23 | AuthorObserver,
24 | ServiceObserver,
25 | LogObserver,
26 | ServicesObserver)
27 |
28 | from vexbot.adapters.shell.completers import ServiceCompleter, WordCompleter
29 |
30 |
31 | # add in ! and # to our word completion regex
32 | _WORD = re.compile(r'([!#a-zA-Z0-9_]+|[^!#a-zA-Z0-9_\s]+)')
33 |
34 |
35 | def _get_cmd_args_kwargs(text: str) -> (str, tuple, dict):
36 | # consume shebang
37 | text = text[1:]
38 | args = _shlex.split(text)
39 | try:
40 | command = args.pop(0)
41 | except IndexError:
42 | return '', (), {}
43 | args, kwargs = _parse(args)
44 | return command, args, kwargs
45 |
46 |
47 | def _get_default_history_filepath():
48 | vexdir = get_vexdir_filepath()
49 | return path.join(vexdir, 'vexshell_history')
50 |
51 |
52 | class Shell(PromptSession):
53 | def __init__(self, history_filepath=None):
54 | self._logger = logging.getLogger(__name__)
55 | if history_filepath is None:
56 | self._logger.info(' getting default history filepath.')
57 | history_filepath = _get_default_history_filepath()
58 |
59 | self._logger.info(' history filepath %s', history_filepath)
60 | self.history = FileHistory(history_filepath)
61 | self.messaging = _Messaging('shell', run_control_loop=True)
62 |
63 | # TODO: Cleanup this API access
64 | self.messaging._heartbeat_reciever.identity_callback = self._identity_callback
65 | self._thread = _Thread(target=self.messaging.start,
66 | daemon=True)
67 |
68 | self._bot_status_monitor = PeriodicCallback(self._monitor_bot_state,
69 | 1000)
70 |
71 | self.shebangs = ['!', ]
72 | self.command_observer = CommandObserver(self.messaging, prompt=self)
73 | commands = self.command_observer._get_commands()
74 | self._word_completer = WordCompleter(commands, WORD=_WORD)
75 | self._services_completer = ServiceCompleter(self._word_completer)
76 | self.author_interface = AuthorInterface(self._word_completer,
77 | self.messaging)
78 |
79 | self.service_interface = ServiceInterface(self._word_completer,
80 | self.messaging)
81 |
82 | self.entity_interface = EntityInterface(self.author_interface,
83 | self.service_interface)
84 |
85 | self._bot_status = ''
86 |
87 | super().__init__(message='vexbot: ',
88 | history=self.history,
89 | completer=self._services_completer,
90 | enable_system_prompt=True,
91 | enable_suspend=True,
92 | enable_open_in_editor=True,
93 | complete_while_typing=False)
94 |
95 |
96 | self.print_observer = PrintObserver(self.app)
97 |
98 | self._print_subscription = self.messaging.chatter.subscribe(self.print_observer)
99 | self.messaging.chatter.subscribe(LogObserver())
100 | self.messaging.command.subscribe(self.command_observer)
101 | self.messaging.command.subscribe(ServicesObserver(self._identity_setter,
102 | self._set_service_completion))
103 |
104 | def _identity_setter(self, services: list) -> None:
105 | try:
106 | services.remove(self.messaging._service_name)
107 | except ValueError:
108 | pass
109 |
110 | for service in services:
111 | self.service_interface.add_service(service)
112 | self.messaging.send_command('REMOTE',
113 | remote_command='commands',
114 | target=service,
115 | suppress=True)
116 |
117 | def _set_service_completion(self, service: str, commands: list) -> None:
118 | shebang = self.shebangs[0]
119 | commands = [shebang + command for command in commands]
120 | completer = WordCompleter(commands, WORD=_WORD)
121 | self._services_completer.set_service_completer(service, completer)
122 |
123 | # FIXME: `!commands` does not update these as it
124 | if service == 'vexbot':
125 | for command in commands:
126 | self._word_completer.words.add(command)
127 |
128 | def _identity_callback(self):
129 | self.messaging.send_command('services', suppress=True)
130 | self.messaging.send_command('get_commands', suppress=True)
131 |
132 | def _monitor_bot_state(self):
133 | time_now = time.time()
134 | last_message = self.messaging._heartbeat_reciever._last_message_time
135 | # TODO: put in a countdown since last contact?
136 | delta_time = time_now - last_message
137 | # NOTE: Bot is set to send a heartbeat every 1.5 seconds
138 | if time_now - last_message > 3.4:
139 | self._bot_status = ''
140 | else:
141 | if self._bot_status == '':
142 | return
143 | self._bot_status = ''
144 |
145 | self.app.invalidate()
146 |
147 | def is_command(self, text: str) -> bool:
148 | """
149 | checks for presence of shebang in the first character of the text
150 | """
151 | if text[0] in self.shebangs:
152 | return True
153 |
154 | return False
155 |
156 | def run(self):
157 | self._thread.start()
158 | self._bot_status_monitor.start()
159 | with patch_stdout(raw=True):
160 | while True:
161 | # Get our text
162 | try:
163 | text = self.prompt(rprompt=self._get_rprompt_tokens)
164 | # KeyboardInterrupt continues
165 | except KeyboardInterrupt:
166 | continue
167 | # End of file returns
168 | except EOFError:
169 | return
170 |
171 | # Clean text
172 | text = text.lstrip()
173 | # Handle empty text
174 | if text == '':
175 | self._logger.debug(' empty string found')
176 | continue
177 | # Program specific handeling. Currently either first word
178 | # or second word can be commands
179 | for string in text.split('&&'):
180 | self._handle_text(string)
181 |
182 | def _get_rprompt_tokens(self):
183 | return self._bot_status
184 |
185 | def _handle_text(self, text: str):
186 | """
187 | Check to see if text is a command. Otherwise, check to see if the
188 | second word is a command.
189 |
190 | Commands get handeled by the `_handle_command` method
191 |
192 | If not command, check to see if first word is a service or an author.
193 | Default program is to send message replies. However, we will also
194 | check to see if the second word is a command and handle approparitly
195 |
196 | This method does simple string parsing and high level program control
197 | """
198 | # If first word is command
199 | if self.is_command(text):
200 | self._logger.debug(' first word is a command')
201 | # get the command, args, and kwargs out using `shlex`
202 | command, args, kwargs = _get_cmd_args_kwargs(text)
203 | self._logger.info(' command: %s, %s %s', command, args, kwargs)
204 | # hand off to the `handle_command` method
205 | result = self._handle_command(command, args, kwargs)
206 |
207 | if result:
208 | if isinstance(result, str):
209 | print(result)
210 | else:
211 | _pprint.pprint(result)
212 | # Exit the method here if in this block
213 | return
214 | # Else first word is not a command
215 | else:
216 | self._logger.debug(' first word is not a command')
217 | # get the first word and then the rest of the text.
218 | try:
219 | first_word, second_word = text.split(' ', 1)
220 | self._logger.debug(' first word: %s', first_word)
221 | except ValueError:
222 | self._logger.debug('No second word in chain!')
223 | return self._handle_NLP(text)
224 | # check if second word/string is a command
225 | if self.is_command(second_word):
226 | self._logger.info(' second word is a command')
227 | # get the command, args, and kwargs out using `shlex`
228 | command, args, kwargs = _get_cmd_args_kwargs(second_word)
229 | self._logger.debug(' second word: %s', command)
230 | self._logger.debug(' command %s', command)
231 | self._logger.debug('args %s ', args)
232 | self._logger.debug('kwargs %s', kwargs)
233 | return self._first_word_not_cmd(first_word, command, args, kwargs)
234 | # if second word is not a command, default to NLP
235 | else:
236 | self._logger.info(' defaulting to message since second word '
237 | 'isn\'t a command')
238 |
239 | return self._handle_NLP(text)
240 |
241 | def _handle_NLP(self, text: str):
242 | entities = self.entity_interface.get_entities(text)
243 | self.messaging.send_command('NLP', text=text, entities=entities)
244 |
245 | def _handle_command(self, command: str, args: tuple, kwargs: dict):
246 | if kwargs.get('remote', False):
247 | self._logger.debug(' `remote` in kwargs, sending command')
248 | self.messaging.send_command(command, *args, **kwargs)
249 | return
250 | if self.command_observer.is_command(command):
251 | self._logger.debug(' `%s` is a command', command)
252 | # try:
253 | return self.command_observer.handle_command(command, *args, **kwargs)
254 | """
255 | except Exception as e:
256 | self.command_observer.on_error(e, command, args, kwargs)
257 | return
258 | """
259 | else:
260 | self._logger.debug('command not found! Sending to bot')
261 | self.messaging.send_command(command, *args, **kwargs)
262 |
263 | def _first_word_not_cmd(self,
264 | first_word: str,
265 | command: str,
266 | args: tuple,
267 | kwargs: dict) -> None:
268 | """
269 | check to see if this is an author or service.
270 | This method does high level control handling
271 | """
272 | if self.service_interface.is_service(first_word):
273 | self._logger.debug(' first word is a service')
274 | kwargs = self.service_interface.get_metadata(first_word, kwargs)
275 | self._logger.debug(' service transform kwargs: %s', kwargs)
276 | elif self.author_interface.is_author(first_word):
277 | self._logger.debug(' first word is an author')
278 | kwargs = self.author_interface.get_metadata(first_word, kwargs)
279 | self._logger.debug(' author transform kwargs: %s', kwargs)
280 | if not kwargs.get('remote'):
281 | kwargs['remote_command'] = command
282 | command= 'REMOTE'
283 | self.messaging.send_command(command, *args, **kwargs)
284 | return
285 | else:
286 | self.messaging.send_command(command, *args, **kwargs)
287 |
--------------------------------------------------------------------------------
/vexbot/adapters/socket_io/__init__.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import json
3 | import html
4 | import logging
5 | import pkg_resources
6 | import atexit
7 | from time import sleep
8 | from threading import Thread
9 |
10 | from vexmessage import decode_vex_message
11 |
12 | from vexbot.adapters.messaging import Messaging
13 | from vexbot.adapters.socket_io.observer import SocketObserver
14 | try:
15 | from websocket import WebSocketApp
16 | except ImportError:
17 | logging.error('Socket IO needs `websocket` installed. Please run `pip '
18 | 'install websocket-client`')
19 | try:
20 | import requests
21 | except ImportError as e:
22 | logging.error('Socket IO needs `requests` installed. Please run `pip '
23 | 'install requests`')
24 |
25 |
26 | class WebSocket(WebSocketApp):
27 | def __init__(self,
28 | streamer_name: str,
29 | namespace: str,
30 | website_url: str,
31 | service_name: str,
32 | connection: dict=None):
33 |
34 | self.log = logging.getLogger(__name__)
35 | self.log.setLevel(0)
36 | if connection is None:
37 | connection = {}
38 |
39 | self.messaging = Messaging(service_name, run_control_loop=True, **connection)
40 | self._scheduler_thread = Thread(target=self.messaging.start,
41 | daemon=True)
42 |
43 | self._scheduler_thread.start()
44 | self.observer = SocketObserver(self, self.messaging)
45 | self.messaging.command.subscribe(self.observer)
46 |
47 | self._streamer_name = streamer_name
48 | self.namespace = namespace
49 | self._website_url = website_url
50 | self.log.info('Getting Socket IO key!')
51 | self.key, heartbeat = self._connect_to_server_helper()
52 | self.log.info('Socket IO key got!')
53 | # self.command_manager = AdapterCommandManager(self.messaging)
54 |
55 | # alters URL to be more websocket...ie
56 | self._website_socket = self._website_url.replace('http', 'ws')
57 | self._website_socket += 'websocket/'
58 | self.nick = None
59 | super().__init__(self._website_socket + self.key,
60 | on_open=self.on_open,
61 | on_close=self.on_close,
62 | on_message=self.on_message,
63 | on_error=self.on_error)
64 |
65 | def _connect_to_server_helper(self):
66 | r = requests.post(self._website_url)
67 | params = r.text
68 |
69 | # unused variables are connection_timeout and supported_formats
70 | key, heartbeat_timeout, _, _ = params.split(':')
71 | heartbeat_timeout = int(heartbeat_timeout)
72 | return key, heartbeat_timeout
73 |
74 | def on_open(self, *args):
75 | logging.info('Websocket open!')
76 |
77 | def on_close(self, *args):
78 | logging.info('Websocket closed :(')
79 |
80 | def on_message(self, *args):
81 | message = args[1].split(':', 3)
82 | key = int(message[0])
83 | # namespace = message[2]
84 |
85 | if len(message) >= 4:
86 | data = message[3]
87 | else:
88 | data = ''
89 | if key == 1 and args[1] == '1::':
90 | self.send_packet_helper(1)
91 | elif key == 1 and args[1] == '1::{}'.format(self.namespace):
92 | self.send_packet_helper(5, data={'name': 'initialize'})
93 | data = {'name': 'join',
94 | 'args': ['{}'.format(self._streamer_name)]}
95 |
96 | self.send_packet_helper(5, data=data)
97 | self.log.info('Connected to channel with socket io!')
98 | # self.messaging.send_status('CONNECTED')
99 | elif key == 2:
100 | self.send_packet_helper(2)
101 | elif key == 5:
102 | data = json.loads(data, )
103 | if data['name'] == 'message':
104 | message = data['args'][0]
105 | sender = html.unescape(message['sender'])
106 | if sender == self.nick:
107 | return
108 | message = html.unescape(message['text'])
109 | self.messaging.send_chatter(author=sender, message=message)
110 | elif data['name'] == 'join':
111 | self.nick = data['args'][1]
112 |
113 | def on_error(self, *args):
114 | logging.error(args[1])
115 |
116 | def disconnect(self):
117 | callback = ''
118 | data = ''
119 | # '1::namespace'
120 | self.send(':'.join([str(self.TYPE_KEYS['DISCONNECT']),
121 | callback, self.namespace, data]))
122 |
123 | def send_packet_helper(self,
124 | type_key,
125 | data=None):
126 |
127 | if data is None:
128 | data = ''
129 | else:
130 | data = json.dumps(data)
131 |
132 | # NOTE: callbacks currently not implemented
133 | callback = ''
134 | message = ':'.join([str(type_key), callback, self.namespace, data])
135 | self.send(message)
136 |
--------------------------------------------------------------------------------
/vexbot/adapters/socket_io/__main__.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import configparser
3 |
4 | from vexbot.adapters.socket_io import WebSocket
5 |
6 |
7 | def _get_args():
8 | # FIXME: need to ensure that these values are passed in
9 | parser = argparse.ArgumentParser()
10 | parser.add_argument('configuration_file')
11 | return parser.parse_args()
12 |
13 |
14 | def main():
15 | args = _get_args()
16 | config = configparser.ConfigParser()
17 | config.read(args.configuration_file)
18 | kwargs = config['socket_io']
19 | client = WebSocket(**kwargs)
20 | client.run_forever()
21 |
22 |
23 | if __name__ == '__main__':
24 | main()
25 |
--------------------------------------------------------------------------------
/vexbot/adapters/socket_io/observer.py:
--------------------------------------------------------------------------------
1 | import inspect as _inspect
2 |
3 | from vexmessage import Request
4 | from vexbot.observer import Observer
5 | from vexbot.extensions import develop
6 | from vexbot.extensions import help as vexhelp
7 |
8 |
9 | class SocketObserver(Observer):
10 | extensions = (develop.get_code, develop.get_commands, vexhelp.help)
11 | def __init__(self, bot, messaging):
12 | super().__init__()
13 | self.bot = bot
14 | self.messaging = messaging
15 | self._commands = self._get_commands()
16 |
17 | def _get_commands(self) -> dict:
18 | result = {}
19 | for name, method in _inspect.getmembers(self):
20 | if name.startswith('do_'):
21 | result[name[3:]] = method
22 |
23 | return result
24 |
25 | def do_MSG(self, message, *args, **kwargs):
26 | msg_target = kwargs.get('msg_target')
27 | if msg_target:
28 | message = msg_target + ', ' + message
29 |
30 | data = {}
31 | data['name'] = 'message'
32 | data['args'] = [message, self.bot._streamer_name]
33 |
34 | # FIXME: this API is awful.
35 | self.bot.send_packet_helper(5, data)
36 |
37 | def on_next(self, item: Request):
38 | command = item.command
39 | args = item.args
40 | kwargs = item.kwargs
41 | try:
42 | callback = self._commands[command]
43 | except KeyError:
44 | return
45 |
46 | try:
47 | result = callback(*args, **kwargs)
48 | except Exception as e:
49 | self.on_error(e, command, args)
50 | return
51 |
52 | if result is None:
53 | return
54 |
55 | source = item.source
56 | self.messaging.send_command_response(source, command, result=result, *args, **kwargs)
57 |
58 | def on_error(self, *args, **kwargs):
59 | pass
60 |
61 | def on_completed(self, *args, **kwargs):
62 | pass
63 |
--------------------------------------------------------------------------------
/vexbot/adapters/stackoverflow.py:
--------------------------------------------------------------------------------
1 | from os import path
2 | from time import sleep
3 |
4 | import selenium
5 | from selenium import webdriver
6 | from selenium.webdriver.chrome.options import Options
7 | from vexbot.adapters.messaging import Messaging
8 |
9 |
10 | class StackOverflow:
11 | def __init__(self, room='https://chat.stackoverflow.com/rooms/6/python'):
12 | options = Options()
13 | options.add_argument('--headless')
14 | self.driver = webdriver.Chrome(chrome_options=options)
15 | self.driver.get(room)
16 | self.messaging = Messaging('stack')
17 |
18 | def run(self):
19 | # start messaging
20 | self.messaging._setup()
21 | # Get the chat block
22 | comment_block = self.driver.find_element_by_id('chat')
23 | # Get the comments from the chat block
24 | comments = comment_block.find_elements_by_xpath('//*[@id="chat"]')
25 | # calculate the number of comments
26 | num_comments = len(comments)
27 | # get the last comment
28 | last_comment = comments[-1]
29 | last_message_length = len(last_comment.find_elements_by_class_name('message'))
30 | while True:
31 | comments = comment_block.find_elements_by_xpath('//*[@id="chat"]')
32 | # Check last one here
33 | messages = last_comment.find_elements_by_class_name('message')
34 | message_length = len(messages)
35 | if message_length > last_message_length:
36 | username = last_comment.find_elements_by_class_name('username')[-1].text
37 | messages = messages[last_message_length - message_length:]
38 | texts = []
39 | for message in messages:
40 | texts.append(message.find_element_by_class_name('content').text)
41 | text = '\n'.join(texts)
42 | self.messaging.send_chatter('stack', author=username, message=text, channel='stackoverflow')
43 | last_message_length = message_length
44 |
45 | new_num = len(comments)
46 | if new_num > num_comments:
47 | delta = num_comments - new_num
48 | for comment in comments[delta:]:
49 | username = comment.find_element_by_class_name('username').text
50 | messages = comment.find_elements_by_class_name('messages')
51 | texts = []
52 | for message in comment.find_elements_by_class_name('message'):
53 | texts.append(message.find_element_by_class_name('content').text)
54 | text = '\n'.join(texts)
55 | self.messaging.send_chatter('stack', author=username, message=text, channel='stackoverflow')
56 | # loop maintainence
57 | num_comments = new_num
58 | last_comment = comments[-1]
59 | last_message_length = len(last_comment.find_elements_by_class_name('message'))
60 | else:
61 | sleep(1)
62 |
63 | def __del__(self):
64 | try:
65 | self.driver.close()
66 | self.driver.quit()
67 | except Exception:
68 | pass
69 |
70 |
71 | def main():
72 | s = StackOverflow()
73 | s.run()
74 |
75 |
76 | if __name__ == '__main__':
77 | main()
78 |
--------------------------------------------------------------------------------
/vexbot/adapters/xmpp/__init__.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import logging
3 | import argparse
4 | # import signal
5 | import atexit
6 | import pkg_resources
7 | from threading import Thread
8 |
9 | import zmq
10 | from vexmessage import decode_vex_message
11 |
12 | from vexbot.adapters.messaging import Messaging # flake8: noqa
13 |
14 | from sleekxmpp import ClientXMPP
15 | from sleekxmpp.exceptions import IqError, IqTimeout
16 |
17 |
18 | class XMPPBot(ClientXMPP):
19 | def __init__(self,
20 | jid: str,
21 | password: str,
22 | room: str,
23 | service_name: str,
24 | bot_nick: str='EchoBot',
25 | connection: dict=None,
26 | **kwargs):
27 |
28 | # Initialize the parent class
29 | super().__init__(jid, password)
30 | if connection is None:
31 | connection = {}
32 | self.messaging = Messaging(service_name,
33 | run_control_loop=True,
34 | **connection)
35 |
36 | self.room = room
37 | self.nick = bot_nick
38 | # self.log = logging.getLogger(__file__)
39 | # self['feature_mechanisms'].unencrypted_plain = True
40 |
41 | # One-shot helper method used to register all the plugins
42 | self._register_plugin_helper()
43 |
44 | self.add_event_handler("session_start", self.session_start)
45 | self.add_event_handler("groupchat_message", self.muc_message)
46 | self._thread = Thread(target=self.messaging.start, daemon=True)
47 | self._thread.start()
48 |
49 | def _register_plugin_helper(self):
50 | """
51 | One-shot helper method used to register all the plugins
52 | """
53 | # Service Discovery
54 | self.register_plugin('xep_0030')
55 | # XMPP Ping
56 | self.register_plugin('xep_0199')
57 | # Multiple User Chatroom
58 | self.register_plugin('xep_0045')
59 |
60 | def session_start(self, event):
61 | self.log.info('starting xmpp')
62 | self.send_presence()
63 | self.plugin['xep_0045'].joinMUC(self.room,
64 | self.nick,
65 | wait=True)
66 |
67 | self.get_roster()
68 |
69 | def muc_message(self, msg):
70 | self.messaging.send_chatter(author=msg['mucnick'],
71 | message=msg['body'],
72 | channel=msg['from'].bare)
73 |
--------------------------------------------------------------------------------
/vexbot/adapters/xmpp/__main__.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import configparser
3 | from vexbot.adapters.xmpp import XMPPBot
4 | import logging
5 |
6 |
7 | def _get_args():
8 | parser = argparse.ArgumentParser()
9 | parser.add_argument('configuration_file')
10 | return parser.parse_args()
11 |
12 |
13 | def main(**kwargs)
14 | args = _get_args()
15 | config = configparser.ConfigParser()
16 | config.read(args.configuration_file)
17 | kwargs = config['xmpp']
18 | # TODO: validate this works like I think it does
19 | connection = config.get('connection', fallback={})
20 |
21 | jid = '{}@{}/{}'.format(kwargs['local'], kwargs['domain'], kwargs['resource'])
22 | xmpp_bot = XMPPBot(jid, connection=connection, **kwargs)
23 | # logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s %(message)s')
24 | xmpp_bot.connect()
25 | try:
26 | xmpp_bot.process(block=True)
27 | finally:
28 | xmpp_bot.disconnect()
29 |
30 |
31 | if __name__ == '__main__':
32 | main()
33 |
--------------------------------------------------------------------------------
/vexbot/adapters/youtube/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | import logging
3 | import asyncio
4 | import argparse
5 | from argparse import Namespace
6 | import tempfile
7 | import pkg_resources
8 | from time import sleep
9 |
10 | from vexbot.adapters.messaging import Messaging
11 | from vexbot.adapters.scheduler import Scheduler
12 |
13 | import httplib2
14 |
15 | from apiclient.discovery import build
16 | from oauth2client.client import flow_from_clientsecrets
17 | from oauth2client.file import Storage
18 | from oauth2client.tools import run_flow, argparser
19 |
20 |
21 | class Youtube:
22 | def __init__(self, connection: dict=None, **kwargs):
23 | if connection is None:
24 | connection = {}
25 | self.messaging = Messaging('youtube', **connection)
26 | scope = ['https://www.googleapis.com/auth/youtube',
27 | 'https://www.googleapis.com/auth/youtube.force-ssl',
28 | 'https://www.googleapis.com/auth/youtube.readonly']
29 | self.api = _youtube_authentication(client_secret_filepath, scope)
30 |
31 | def run(self):
32 | self.messaging._setup()
33 | parts = 'snippet'
34 | livestream_response = self.api.liveBroadcasts().list(mine=True,
35 | part=parts,
36 | maxResults=1).execute()
37 |
38 |
39 | live_chat_id = livestream_response.get('items')[0]['snippet']['liveChatId']
40 |
41 | livechat_response = youtube_api.liveChatMessages().list(liveChatId=live_chat_id, part='snippet').execute()
42 |
43 | next_token = livechat_response.get('nextPageToken')
44 | polling_interval = livechat_response.get('pollingIntervalMillis')
45 | polling_interval = _convert_to_seconds(polling_interval)
46 | while True:
47 | await asyncio.sleep(polling_interval)
48 | part = 'snippet, authorDetails'
49 | livechat_response = self.api.liveChatMessages().list(liveChatId=live_chat_id, part=part, pageToken=next_token).execute()
50 |
51 | next_token = livechat_response.get('nextPageToken')
52 | polling_interval = livechat_response.get('pollingIntervalMillis')
53 | polling_interval = _convert_to_seconds(polling_interval)
54 |
55 | for live_chat_message in livechat_response.get('items'):
56 | snippet = live_chat_message['snippet']
57 | if not bool(snippet['hasDisplayContent']):
58 | continue
59 | message = snippet['displayMessage']
60 | author = live_chat_message['authorDetails']['displayName']
61 | self.messaging.send_chatter(author=author, message=message)
62 |
63 |
64 | async def _run(messaging, live_chat_messages, live_chat_id, ):
65 | command_manager = AdapterCommandManager(messaging)
66 | frame = None
67 | for message in messaging.run(250):
68 | if message.type == 'CMD':
69 | command_manager.parse_commands(message)
70 | elif message.type == 'RSP':
71 | message = message.contents.get('response')
72 | body={'snippet':{'type': 'textmessageEvent',
73 | 'liveChatId': live_chat_id,
74 | 'textMessageDetails': {'messageText': message}}}
75 |
76 | live_chat_messages.insert(part='snippet',
77 | body=body).execute()
78 |
79 | await asyncio.sleep(1)
80 |
81 |
82 |
83 | def _convert_to_seconds(milliseconds: str):
84 | return float(milliseconds)/1000.0
85 |
86 |
87 |
88 |
89 | _YOUTUBE_API_SERVICE_NAME = 'youtube'
90 | _YOUTUBE_API_VERSION = 'v3'
91 | _ALL = "https://www.googleapis.com/auth/youtube"
92 |
93 |
94 | def _youtube_authentication(client_filepath, youtube_scope=_READ_ONLY):
95 | missing_client_message = "You need to populate the client_secrets.json!"
96 | flow = flow_from_clientsecrets(client_filepath,
97 | scope=youtube_scope,
98 | message=missing_client_message)
99 |
100 | dir = os.path.abspath(__file__)
101 | filepath = "{}-oauth2.json".format(dir)
102 | # TODO: Determine if removing file is needed
103 | # remove old oauth files to be safe
104 | if os.path.isfile(filepath):
105 | os.remove(filepath)
106 |
107 | storage = Storage(filepath)
108 | credentials = storage.get()
109 | if credentials is None or credentials.invalid:
110 | args = Namespace(auth_host_name='localhost',
111 | auth_host_port=[8080, 8090],
112 | noauth_local_webserver=False,
113 | logging_level='ERROR')
114 |
115 | credentials = run_flow(flow, storage, args)
116 | return build(_YOUTUBE_API_SERVICE_NAME,
117 | _YOUTUBE_API_VERSION,
118 | http=credentials.authorize(httplib2.Http()))
119 |
120 |
121 | def _get_youtube_link(client_secrets_filepath):
122 | youtube_api = youtube_authentication(client_secrets_filepath)
123 | parts = 'id, snippet, status'
124 | livestream_requests = youtube_api.liveBroadcasts().list(mine=True,
125 | part=parts,
126 | maxResults=5)
127 |
128 | while livestream_requests:
129 | response = livestream_requests.execute()
130 | # TODO: add better parsing here
131 | youtube_id = response.get('items', [])[0]['id']
132 | return 'http://youtube.com/watch?v={}'.format(youtube_id)
133 |
--------------------------------------------------------------------------------
/vexbot/adapters/youtube/__main__.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | from configparser import ConfigParser
3 |
4 | from vexbot.adapters.youtube import Youtube
5 |
6 |
7 | def _get_kwargs():
8 | parser = argparse.ArgumentParser()
9 | parser.add_argument('client_secret_filepath')
10 | args = parser.parse_args()
11 | kwargs = vars(args)
12 |
13 | return kwargs
14 |
15 | def main(client_secret_filepath: str):
16 | config_parser = ConfigParser.read(client_secret_filepath)
17 | youtube = Youtube()
18 | youtube.run()
19 |
20 |
21 | if __name__ == '__main__':
22 | kwargs = _get_kwargs()
23 | main(**kwargs)
24 |
--------------------------------------------------------------------------------
/vexbot/command.py:
--------------------------------------------------------------------------------
1 | import functools as _functools
2 |
3 |
4 | def command(function=None,
5 | alias: list=None,
6 | hidden: bool=False,
7 | roles: list=None,
8 | utterances: list=None,
9 | short: str=None):
10 |
11 | if function is None:
12 | return _functools.partial(command,
13 | alias=alias,
14 | hidden=hidden,
15 | roles=roles,
16 | utterances=utterances,
17 | short=short)
18 |
19 | # https://stackoverflow.com/questions/10176226/how-to-pass-extra-arguments-to-python-decorator
20 | @_functools.wraps(function)
21 | def wrapper(*args, **kwargs):
22 | return function(*args, **kwargs)
23 | # TODO: check for string and convert to list
24 | if alias is not None:
25 | wrapper.alias = alias
26 | wrapper.hidden = hidden
27 | wrapper.command = True
28 | wrapper.roles = roles
29 | wrapper.utterances = utterances
30 | wrapper.short = short
31 | return wrapper
32 |
--------------------------------------------------------------------------------
/vexbot/command_observer.py:
--------------------------------------------------------------------------------
1 | import sys as _sys
2 | import shelve
3 | import logging
4 | import inspect as _inspect
5 | import typing
6 | import importlib
7 | from os import path
8 |
9 | from tblib import Traceback
10 |
11 | from vexmessage import Request
12 | from vexbot.observer import Observer
13 | from vexbot.messaging import Messaging
14 | from vexbot.intents import intent
15 | from vexbot.command import command
16 | from vexbot.extensions import help as vexhelp
17 | from vexbot.extensions import (log,
18 | admin,
19 | extensions)
20 |
21 | from vexbot.util.get_cache_filepath import get_cache
22 | from vexbot.util.get_cache_filepath import get_cache_filepath as get_cache_dir
23 | from vexbot.util.create_cache_filepath import create_cache_directory
24 |
25 |
26 | class CommandObserver(Observer):
27 | extensions = (admin.get_commands,
28 | admin.get_disabled,
29 | admin.disable,
30 | vexhelp.help,
31 | extensions.add_extensions,
32 | extensions.get_extensions,
33 | extensions.remove_extension,
34 | extensions.get_installed_extensions)
35 |
36 | def __init__(self,
37 | bot,
38 | messaging: Messaging,
39 | subprocess_manager: 'vexbot.subprocess_manager.SubprocessManager',
40 | language):
41 |
42 | super().__init__()
43 | self.bot = bot
44 | self.messaging = messaging
45 | self.subprocess_manager = subprocess_manager
46 | self.language = language
47 |
48 | cache_dir = get_cache_dir()
49 | mkdir = not path.isdir(cache_dir)
50 | if mkdir:
51 | os.makedirs(cache_dir, exist_ok=True)
52 |
53 | filepath = get_cache(__name__ + '.pickle')
54 | init = not path.isfile(filepath)
55 |
56 | self._config = shelve.open(filepath, writeback=True)
57 |
58 | if init:
59 | self._config['extensions'] = {}
60 | self._config['disabled'] = {}
61 | self._config['modules'] = {}
62 |
63 | self._commands = {}
64 | self.init_commands()
65 | self._intents = self._get_intents()
66 | for key, values in self._config['extensions'].items():
67 | self.add_extensions(key, **values)
68 | self.logger = logging.getLogger(self.messaging._service_name + '.observers.command')
69 |
70 | self.root_logger = logging.getLogger()
71 | # self.root_logger.setLevel(logging.DEBUG)
72 | # logging.basicConfig()
73 | self.root_logger.addHandler(self.messaging.pub_handler)
74 |
75 | def _get_intents(self) -> dict:
76 | result = {}
77 | # FIXME: define a `None` callback
78 | # result[None] = self.do_commands
79 | for name, method in _inspect.getmembers(self):
80 | is_intent = getattr(method, '_vex_intent', False)
81 | if name.startswith('do_') or is_intent:
82 | try:
83 | name = method._vex_intent_name
84 | except AttributeError:
85 | name = name[3:]
86 | result[name] = method
87 |
88 | return result
89 |
90 | def init_commands(self):
91 | for key, value in self._config['extensions'].items():
92 | self.add_extensions(key, **value, update=False)
93 | self.update_commands()
94 | # FIXME: not handeling the conflict
95 |
96 | def update_commands(self):
97 | for name, method in _inspect.getmembers(self):
98 | if name.startswith('do_'):
99 | self._commands[name[3:]] = method
100 | elif getattr(method, 'command', False):
101 | self._commands[name] = method
102 | else:
103 | continue
104 |
105 | if getattr(method, 'alias', False):
106 | for alias in method.alias:
107 | self._commands[alias] = method
108 |
109 | def _get_commands(self) -> dict:
110 | result = {}
111 | for name, method in _inspect.getmembers(self):
112 | if name.startswith('do_'):
113 | result[name[3:]] = method
114 | elif getattr(method, 'command', False):
115 | result[name] = method
116 | else:
117 | continue
118 |
119 | if getattr(method, 'alias', False):
120 | for alias in method.alias:
121 | result[alias] = method
122 | return result
123 |
124 | @command(alias=['MSG',], roles=['admin',])
125 | def do_REMOTE(self,
126 | target: str,
127 | remote_command: str,
128 | source: list,
129 | *args,
130 | **kwargs) -> None:
131 | """
132 | Send a remote command to a service. Used
133 |
134 | Args:
135 | target: The service that the command gets set to
136 | remote_command: The command to do remotely.
137 | source: the binary source of the zmq_socket. Packed to send to the
138 | """
139 | if target == self.messaging._service_name:
140 | info = 'target for remote command is the bot itself! Returning the function'
141 | self.logger.info(info)
142 | return self._handle_command(remote_command, source, *args, **kwargs)
143 |
144 | try:
145 | target = self.messaging._address_map[target]
146 | except KeyError:
147 | warn = ' Target %s, not found in addresses. Are you sure that %s sent an IDENT message?'
148 | self.logger.warn(warn, target, target)
149 | # TODO: raise an error instead of returning?
150 | # NOTE: Bail here since there's no point in going forward
151 | return
152 |
153 | self.logger.info(' REMOTE %s, target: %s | %s, %s',
154 | remote_command, target, args, kwargs)
155 |
156 | # package the binary together
157 | source = target + source
158 | self.messaging.send_command_response(source,
159 | remote_command,
160 | *args,
161 | **kwargs)
162 |
163 | @command(roles=['admin',])
164 | def do_NLP(self, *args, **kwargs):
165 | # FIXME: Do entity extraction here
166 | intent, confidence, entities = self.language.get_intent(*args, **kwargs)
167 | self.logger.debug('intent from do_NLP: %s', intent)
168 |
169 | # FIXME: `None` returns `do_command`
170 | try:
171 | callback = self._intents[intent]
172 | except NameError:
173 | self.logger.debug('intent not found! intent: %s', intent)
174 | # TODO: send intent back to source
175 | return
176 |
177 | # FIXME:Pass entites in as kwargs?
178 | result = callback(*args, entities=entities, **kwargs)
179 | return result
180 |
181 | @command(roles=['admin',])
182 | def do_TRAIN_INTENT(self, *args, **kwargs):
183 | self.logger.debug('starting training!')
184 | intents = self.bot.intents.get_intents()
185 | for k, v in intents.items():
186 | intents[k] = v()
187 | # self.logger.debug('intents: %s', intents)
188 | self.language.train_classifier(intents)
189 | self.logger.debug('training finished')
190 |
191 | @command(roles=['admin',])
192 | def do_IDENT(self, service_name: str, source: list, *args, **kwargs) -> None:
193 | """
194 | Perform identification of a service to a binary representation.
195 |
196 | Args:
197 | service_name: human readable name for service
198 | source: zmq representation for the socket source
199 | """
200 | self.logger.info(' IDENT %s as %s', service_name, source)
201 | self.messaging._address_map[service_name] = source
202 |
203 | @command(roles=['admin',])
204 | def do_AUTH_FOR_SUBSCRIPTION_SOCKET(self):
205 | self.messaging.subscription_socket.close()
206 | self.messaging._setup_subscription_socket(True)
207 |
208 | @command(roles=['admin',])
209 | def do_NO_AUTH_FOR_SUBSCRIPTION_SOCKET(self):
210 | self.messaging.subscription_socket.close()
211 | self.messaging._setup_subscription_socket(False)
212 |
213 | @command(roles=['admin',])
214 | @intent(name='get_services')
215 | def do_services(self, *args, **kwargs) -> tuple:
216 | return tuple(self.messaging._address_map.keys())
217 |
218 | @command(alias=['get_last_error',], roles=['admin'])
219 | @intent(name='get_last_error')
220 | def do_show_last_error(self, *args, **kwargs):
221 | exc_info = _sys.exc_info()
222 | if exc_info[2] is None:
223 | return 'No problems here boss!'
224 | self.logger.debug('exc_info: %s', exc_info)
225 | return Traceback(exc_info[2]).to_dict()
226 |
227 | def _handle_result(self,
228 | command: str,
229 | source: list,
230 | result,
231 | *args,
232 | **kwargs) -> None:
233 |
234 | self.logger.info('send response %s %s %s', source, command, result)
235 | self.messaging.send_command_response(source, command, result=result, *args, **kwargs)
236 |
237 | def _handle_command(self,
238 | command: str,
239 | source: list,
240 | *args,
241 | **kwargs) -> None:
242 |
243 | try:
244 | callback = self._commands[command]
245 | except KeyError:
246 | self.logger.info(' command not found! %s', command)
247 | return
248 |
249 | self.logger.debug(' handle_command kwargs: %s', kwargs)
250 | kwargs['source'] = source
251 | try:
252 | result = callback(*args, **kwargs)
253 | except Exception as e:
254 | self.on_error(e, command, *args, **kwargs)
255 | return
256 | kwargs.pop('source')
257 |
258 | if result is None:
259 | self.logger.debug(' No result from callback on command: %s', command)
260 | else:
261 | service = self.messaging._service_name
262 | self._handle_result(command, source, result, service=service, *args, **kwargs)
263 |
264 | def on_next(self, item: Request) -> None:
265 | command = item.command
266 | args = item.args
267 | kwargs = item.kwargs
268 | source = item.source
269 | self.logger.info(' Request recieved, %s %s %s %s',
270 | command, source, args, kwargs)
271 |
272 | if not kwargs.get('result'):
273 | self._handle_command(command, source, *args, **kwargs)
274 | else:
275 | # TODO: Verify that this makes sense
276 | # NOTE: Stripping the first address on here as it should be the bot address
277 | source = source[1:]
278 | self._handle_result(command,
279 | source,
280 | kwargs.pop('result'),
281 | *args,
282 | **kwargs)
283 |
284 | def on_error(self, error: Exception, command, *args, **kwargs):
285 | self.logger.warn('on_error called for %s %s %s', command, args, kwargs)
286 | self.logger.warn('on_error called %s:%s', error.__class__.__name__, error.args)
287 | self.logger.exception(' on_next error for command {}'.format(command))
288 |
289 | def on_completed(self, *args, **kwargs):
290 | self.logger.info(' command observer completed!')
291 |
292 | @command(alias=['quit', 'exit'], roles=['admin',])
293 | @intent(name='exit')
294 | def do_kill_bot(self, *args, **kwargs):
295 | """
296 | Kills the instance of vexbot
297 | """
298 | self.logger.warn(' Exiting bot!')
299 | _sys.exit()
300 |
--------------------------------------------------------------------------------
/vexbot/dynamic_entities.py:
--------------------------------------------------------------------------------
1 | # NOTE: Unimplemented
2 |
3 |
4 | def pass():
5 | levels = ('debug', 'info', 'warning', 'warn', 'error', 'critical',
6 | 'DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL')
7 | examples = ('set logging to {}',
8 | 'set log to {}',
9 | 'logging to {}')
10 |
11 | class BotIntents:
12 | def __init__(self):
13 | self._intents = self.get_intents()
14 |
15 | def get_intents(self) -> dict:
16 | result = {}
17 | for name, method in _inspect.getmembers(self):
18 | isintent = getattr(method, '_vex_intent', False)
19 |
20 | if name.startswith('do_') or isintent:
21 | if isintent:
22 | name = method._vex_intent_name
23 | else:
24 | name = name[3:]
25 |
26 | result[name] = method
27 |
28 | return result
29 |
30 | def get_intent_names(self, *args, **kwargs) -> tuple:
31 | if not args:
32 | return tuple(self._intents.keys())
33 | results = {}
34 | for arg in args:
35 | # FIXME: I hate this API
36 | intent_values = self._intents.get(arg)
37 | if intent_values is not None:
38 | intent_values = intent_values()
39 | if intent_values is None:
40 | intent_values = ()
41 | results[arg] = intent_values
42 | return results
43 |
44 | def do_set_log(self) -> tuple:
45 | levels = ('debug', 'info', 'warning', 'warn', 'error', 'critical',
46 | 'DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL')
47 | examples = ('set logging to {}',
48 | 'set log to {}',
49 | 'logging to {}')
50 |
51 | # FIXME: finish
52 | values = ()
53 | return values
54 |
55 | def do_get_code(self) -> tuple:
56 | values = ('get code for that commands',
57 | 'get code for last',
58 | 'get code for {}',
59 | 'get source for {}',
60 | 'show me the code',
61 | 'show code',
62 | 'display code')
63 |
64 | def do_get_commands(self) -> tuple:
65 | pass
66 |
67 | def do_restart_program(self) -> tuple:
68 | pass
69 |
70 | def do_start_program(self) -> tuple:
71 | pass
72 |
73 | def do_stop(self) -> tuple:
74 | pass
75 |
76 | def do_status(self) -> tuple:
77 | pass
78 |
79 |
80 | def do_trace(self) -> tuple:
81 | values = ('run that back for me',
82 | 'trace last command',
83 | 'trace command',
84 | 'trace')
85 |
86 | return values
87 |
88 | def do_lint(self) -> tuple:
89 | values = ('lint file',
90 | 'flake file')
91 |
92 | return values
93 |
94 |
95 |
--------------------------------------------------------------------------------
/vexbot/entity_extraction.py:
--------------------------------------------------------------------------------
1 | import sklearn_crfsuite
2 | import spacy
3 |
4 |
5 | class EntityExtraction:
6 | def __init__(self, language_model=None):
7 | values = {'algorithm': 'lbfgs',
8 | # coefficient for L1 penalty
9 | 'c1': 1,
10 | # coefficient for L2 penalty
11 | 'c2': 1e-3,
12 | 'max_iterations': 50,
13 | # include transitions that are possible, but not observed
14 | 'all_possible_transitions': True}
15 |
16 | self.ent_tagger = sklearn_crfsuite.CRF(**values)
17 | self._language_model = language_model
18 |
19 | def train(self, data):
20 | # _sentence_to_features
21 | x_train = []
22 | # _sentence_to_labels
23 | y_train = []
24 |
25 | self.ent_tagger.fit(x_train, y_train)
26 |
27 | def get_entites(self, text: str, entities: list=None, *args, **kwargs):
28 | spacy_entities = self.get_spacy_entites(text)
29 | # TODO: Implement
30 | duckling_entities = self.get_duckling_entities(text)
31 | custom_entities = self.get_custom_entities(text)
32 |
33 | # FIXME
34 | return spacy_entities
35 |
36 | def get_custom_entities(self, text: str) -> list:
37 | """
38 | Custom, Domain-specific entities
39 | Uses conditional random field
40 | Needs to be trained
41 | """
42 | # NOTE: MITIE uses structured SVM
43 | pass
44 |
45 | def get_duckling_entities(self, text: str) -> list:
46 | """
47 | Pretrained (duckling)
48 | Uses context-free grammer
49 | Dates, Amounts of Money, Druations, Distances, Ordinals
50 | """
51 | pass
52 |
53 | def get_spacy_entites(self, text: str) -> list:
54 | """
55 | Pretrained (spaCy)
56 | Uses averaged preceptron
57 | Places, dates, people, organizations
58 | """
59 | # NOTE: spaCy doc
60 | doc = self._language_model(text)
61 |
62 | entities = [{"entity": ent.label_,
63 | "value": ent.text,
64 | "start": ent.start_char,
65 | "end": ent.end_char} for ent in doc.ents]
66 |
67 | return entities
68 |
69 | # NOTE: used for conditional random field training
70 | def _sentence_to_features(self, sentence: list):
71 | # type: (List[Tuple[Text, Text, Text, Text]]) -> List[Dict[Text, Any]]
72 | """Convert a word into discrete features in self.crf_features,
73 | including word before and word after."""
74 |
75 | sentence_features = []
76 | prefixes = ('-1', '0', '+1')
77 |
78 | for word_idx in range(len(sentence)):
79 | # word before(-1), current word(0), next word(+1)
80 | word_features = {}
81 | for i in range(3):
82 | if word_idx == len(sentence) - 1 and i == 2:
83 | word_features['EOS'] = True
84 | # End Of Sentence
85 | elif word_idx == 0 and i == 0:
86 | word_features['BOS'] = True
87 | # Beginning Of Sentence
88 | else:
89 | word = sentence[word_idx - 1 + i]
90 | prefix = prefixes[i]
91 | features = self.crf_features[i]
92 | for feature in features:
93 | # append each feature to a feature vector
94 | value = self.function_dict[feature](word)
95 | word_features[prefix + ":" + feature] = value
96 | sentence_features.append(word_features)
97 | return sentence_features
98 |
99 | # NOTE: used for conditional random field training
100 | def _sentence_to_labels(self, sentence):
101 | # type: (List[Tuple[Text, Text, Text, Text]]) -> List[Text]
102 |
103 | return [label for _, _, label, _ in sentence]
104 |
--------------------------------------------------------------------------------
/vexbot/extension_metadata.py:
--------------------------------------------------------------------------------
1 | extensions = {'pip_install': {'path': 'vexbot.extensions.admin:install',
2 | 'short': 'Install packages using pip',},
3 | 'pip_uninstall': {'path': 'vexbot.extensions.admin:uninstall',
4 | 'short': 'Uninstall packages using pip'},
5 | 'pip_update': {'path': 'vexbot.extensions.admin:update',
6 | 'short': 'Update packages using pip'},
7 | 'get_commands': {'path': 'vexbot.extensions.admin:get_commands',
8 | 'short': 'Get commands from command observer'},
9 | 'get_command_modules': {'path': 'vexbot.extensions.admin:get_command_modules',
10 | 'short': 'Get all the module names for the commands that are available'},
11 | 'get_disabled': {'path': 'vexbot.extensions.admin:get_disabled',
12 | 'short': 'Get all disabled commands from command observer'},
13 | 'disable': {'path': 'vexbot.extensions.admin:disable',
14 | 'short': 'Remove a command from the commands'},
15 | 'get_cache': {'path': 'vexbot.extensions.admin:get_cache',
16 | 'short': 'Get the values of the configuration cache'},
17 | 'delete_cache': {'path': 'vexbot.extensions.admin:delete_cache',
18 | 'short': 'Remove the values from the configuration cache'},
19 | 'get_code': {'path': 'vexbot.extensions.develop:get_code',
20 | 'short': 'Get the source code from a method using the inspect module'},
21 | 'get_method_names': {'path': 'vexbot.extensions.develop:get_members',
22 | 'short': 'Get all the method names from a class using inspect module'},
23 | 'get_size': {'path': 'vexbot.extensions.digitalocean:get_size',
24 | 'short': 'Get the size of a digital ocean droplet',
25 | 'extras': ['digitalocean',]},
26 | 'power_off': {'path': 'vexbot.extensions.digitalocean:power_off',
27 | 'short': 'Power off a digital ocean droplet',
28 | 'extras': ['digitalocean',]},
29 | 'power_on': {'path': 'vexbot.extensions.digitalocean:power_on',
30 | 'short': 'Power on a digital ocean droplet',
31 | 'extras': ['digitalocean',]},
32 | 'log_level': {'path': 'vexbot.extensions.log:log_level',
33 | 'short': 'Set or get the root logger to a level'},
34 | 'set_debug': {'path': 'vexbot.extensions.log:set_debug',
35 | 'short': 'Set the root logger to `logging.DEBUG'},
36 | 'set_info': {'path': 'vexbot.extensions.log:set_info',
37 | 'short': 'Set the root logger to `logging.INFO'},
38 | 'set_default': {'path': 'vexbot.extensions.log:set_default',
39 | 'short': 'Set the root logger to `logging.WARNING'},
40 | 'filter_logs': {'path': 'vexbot.extensions.log:filter_logs',
41 | 'short': 'Filter logs'},
42 | 'anti_filter': {'path': 'vexbot.extensions.log:anti_filter',
43 | 'short': 'Filter anything that is not this'},
44 | 'get_all_droplets': {'path': 'vexbot.extensions.digitalocean:get_all_droplets',
45 | 'short': 'Get all droplets from digital ocean',
46 | 'extras': ['digitalocean',]},
47 | 'add_extensions': {'path': 'vexbot.extensions.extensions:add_extensions',
48 | 'short': 'Add an extension to a command observer'},
49 | 'get_extensions': {'path': 'vexbot.extensions.extensions:get_extensions',
50 | 'short': 'Get all the extensions from an observer instance'},
51 | 'remove_extension': {'path': 'vexbot.extensions.extensions:remove_extension',
52 | 'short': 'Remove an extension from an observer instance'},
53 | 'get_installed_extensions': {'path': 'vexbot.extensions.extensions:get_installed_extensions',
54 | 'short': 'See all installed extensions for Vexbot'},
55 | 'get_installed_modules': {'path': 'vexbot.extensions.extensions:get_installed_modules',
56 | 'short': 'See all installed modules for Vexbot'},
57 | 'add_extensions_from_dict': {'path': 'vexbot.extensions.extensions:add_extensions_from_dict',
58 | 'short': 'Used to intialize command observers efficient'},
59 | 'help': {'path': 'vexbot.extensions.help:help',
60 | 'short': 'Get the help from a method'},
61 | 'hidden': {'path': 'vexbot.extensions.hidden:hidden',
62 | 'short': 'Get all hidden method methods'},
63 | 'bot_intents': {'path': 'vexbot.extensions.intents:get_intents',
64 | 'short': 'Get the intents from the bot'},
65 | 'bot_intent': {'path': 'vexbot.extensions.intents:get_intent'},
66 | 'get_google_trends': {'path': 'vexbot.extensions.news:get_hot_trends',
67 | 'short': 'Get the google trends',
68 | 'extras': ['summarization']},
69 | 'get_news_urls': {'path': 'vexbot.extensions.news:get_popular_urls',
70 | 'extras': ['summarization']},
71 | 'summarize_news_url': {'path': 'vexbot.extensions.news:summarize_article',
72 | 'extras': ['summarization']},
73 | 'start_process': {'path': 'vexbot.extensions.subprocess:start',
74 | 'extras': ['process_manager'],
75 | 'short': 'Start a process'},
76 | 'stop_process': {'path': 'vexbot.extensions.subprocess:stop',
77 | 'extras': ['process_manager'],
78 | 'short': "Stop a process"},
79 | 'restart_process': {'path': 'vexbot.extensions.subprocess:restart',
80 | 'extras': ['process_manager'],
81 | 'short': "Restart a process"},
82 | 'status_process': {'path': 'vexbot.extensions.subprocess:status',
83 | 'extras': ['process_manager'],
84 | 'short': 'Get the status of a process'},
85 | 'process_uptime': {'path': 'vexbot.extensions.subprocess:uptime',
86 | 'extras': ['process_manager'],
87 | 'short': 'Get the uptime of a process'},
88 | 'cpu_count': {'path': 'vexbot.extensions.system:cpu_count',
89 | 'short': 'Show the total number of CPU\'s',
90 | 'extras': ['system']},
91 | 'cpu_frequency': {'path': 'vexbot.extensions.system:cpu_frequency',
92 | 'extras': ['system']},
93 | 'virtual_memory_percent': {'path': 'vexbot.extensions.system:virtual_memory_percent',
94 | 'extras': ['system']},
95 | 'virtual_memory_total': {'path': 'vexbot.extensions.system:virtual_memory_total',
96 | 'short': 'Show the total amoun of virtual memory available in Mb\'s',
97 | 'extras': ['system']},
98 | 'virtual_memory_used': {'path': 'vexbot.extensions.system:virtual_memory_used',
99 | 'short': 'Show the total amount of virtual memory in use in MBs',
100 | 'extras': ['system']},
101 | 'swap': {'path': 'vexbot.extensions.system:swap',
102 | 'short': 'Show the total amount of swap used in MB',
103 | 'extras': ['system']}}
104 |
--------------------------------------------------------------------------------
/vexbot/extensions/__init__.py:
--------------------------------------------------------------------------------
1 | from vexbot.extension_metadata import extensions as _ext_meta
2 |
3 | def extension(base,
4 | alias: list=None,
5 | name: str=None,
6 | hidden: bool=False,
7 | instancemethod: bool=False,
8 | roles: list=None,
9 | short: str=None):
10 |
11 | def wrapper(function):
12 | # Try to use the meta first
13 | if short is None and hasattr(function, '_meta'):
14 | new_short = _ext_meta.get(function._meta, {}).get('short')
15 | elif short:
16 | new_short = short
17 | else:
18 | new_short = None
19 | # Then try to use the function name
20 | if short is None and new_short is None:
21 | new_short = _ext_meta.get(function.__name__, {}).get('short')
22 |
23 | new_name = name or function.__name__
24 | function.command = True
25 | function.hidden = hidden
26 | function.roles = roles
27 | function.alias = alias
28 | function.short = new_short
29 |
30 |
31 | if instancemethod:
32 | # FIXME: implement
33 | if hasattr(base, '_commands'):
34 | pass
35 | else:
36 | pass
37 | else:
38 | setattr(base, new_name, function)
39 |
40 | return function
41 |
42 | return wrapper
43 |
44 |
45 | def extend(base,
46 | function,
47 | alias: list=None,
48 | name: str=None,
49 | hidden: bool=False,
50 | instancemethod: bool=False,
51 | roles: list=None,
52 | short: str=None):
53 |
54 | wrapper = extension(base, alias, name, hidden, instancemethod, roles, short)
55 | wrapper(function)
56 |
57 |
58 | def extendmany(base,
59 | *functions,
60 | hidden: bool=None,
61 | instancemethod: bool=False,
62 | roles: list=None):
63 | for function in functions:
64 | wrapper = extension(base,
65 | hidden=hidden,
66 | instancemethod=instancemethod,
67 | roles=roles)
68 |
69 | wrapper(function)
70 |
--------------------------------------------------------------------------------
/vexbot/extensions/admin.py:
--------------------------------------------------------------------------------
1 | import pip
2 | import inspect
3 |
4 |
5 | def install(self, *args, **kwargs):
6 | pip.main(['install', *args])
7 |
8 |
9 | def uninstall(self, *args, **kwargs):
10 | pip.main(['uninstall', *args])
11 |
12 |
13 | def update(self, *args, **kwargs):
14 | pip.main(['install', '--upgrade', *args])
15 |
16 |
17 | def get_commands(self, *args, **kwargs):
18 | # TODO: add in a short summary
19 | # TODO: also end in some entity parsing? or getting of the args and
20 | # kwargs
21 | commands = self._commands.keys()
22 | return sorted(commands, key=str.lower)
23 |
24 |
25 | def get_disabled(self, *args, **kwargs):
26 | commands = self._config['disabled'].keys()
27 | return sorted(commands, key=str.lower)
28 |
29 |
30 | def disable(self, name: str, *args, **kwargs):
31 | self._commands.pop(name)
32 |
33 |
34 | def get_command_modules(self, *args, **kwargs):
35 | result = set()
36 | for cb in self._commands.values():
37 | module = inspect.getmodule(cb)
38 | result.add(module.__name__)
39 | return result
40 |
41 |
42 | def get_cache(self, *args, **kwargs):
43 | return dict(self._config)
44 |
45 |
46 | def delete_cache(self, *args, **kwargs):
47 | for key in list(self._config.keys()):
48 | self._config.pop(key)
49 | self._config.sync()
50 |
51 | return ('Cache cleared. Please restart service to remove traces of old '
52 | 'cache.')
53 |
--------------------------------------------------------------------------------
/vexbot/extensions/develop.py:
--------------------------------------------------------------------------------
1 | import inspect as _inspect
2 |
3 |
4 | def get_code(self, *args, **kwargs):
5 | """
6 | get the python source code from callback
7 | """
8 | # FIXME: Honestly should allow multiple commands
9 | callback = self._commands[args[0]]
10 | # TODO: syntax color would be nice
11 | source = _inspect.getsourcelines(callback)[0]
12 | """
13 | source_len = len(source)
14 | source = PygmentsLexer(CythonLexer).lex_document(source)()
15 | """
16 | # FIXME: formatting sucks
17 | return "\n" + "".join(source)
18 |
19 |
20 | def get_members(self, *args, **kwargs):
21 | return [x[0] for x in _inspect.getmembers(self)]
22 |
--------------------------------------------------------------------------------
/vexbot/extensions/digitalocean.py:
--------------------------------------------------------------------------------
1 | import os
2 | import digitalocean
3 |
4 |
5 | _MANAGER = False
6 | _DEFAULT_DROPLET = os.getenv('DIGITAL_OCEAN_DROPLET_ID')
7 | _DIGITAL_OCEAN_KEY = os.getenv('DIGITAL_OCEAN_KEY')
8 | if _DIGITAL_OCEAN_KEY:
9 | _MANAGER = digitalocean.Manager(token=_DIGITAL_OCEAN_KEY)
10 |
11 |
12 | def get_size(*args, **kwargs):
13 | id_ = kwargs.get('id', _DEFAULT_DROPLET)
14 | if id_ is None:
15 | # FIXME: implement a way to pass this in as a config
16 | raise RuntimeError('Must pass in an `id_` or set the `DITIAL_OCEAN_DROPLET_ID` in your envirnoment')
17 | droplet = _MANAGER.get_droplet(id_)
18 | return droplet.size
19 |
20 |
21 | def power_off(*args, **kwargs):
22 | id_ = kwargs.get('id', _DEFAULT_DROPLET)
23 | if id_ is None:
24 | # FIXME: implement a way to pass this in as a config
25 | raise RuntimeError('Must pass in an `id_` or set the `DITIAL_OCEAN_DROPLET_ID` in your envirnoment')
26 | droplet = _MANAGER.get_droplet(id_)
27 | droplet.power_off()
28 |
29 |
30 | def power_on(*args, **kwargs):
31 | id_ = kwargs.get('id', _DEFAULT_DROPLET)
32 | if id_ is None:
33 | # FIXME: implement a way to pass this in as a config
34 | raise RuntimeError('Must pass in an `id_` or set the `DITIAL_OCEAN_DROPLET_ID` in your envirnoment')
35 | droplet = _MANAGER.get_droplet(id_)
36 | droplet.power_on()
37 |
38 |
39 | def get_all_droplets(*args, **kwargs):
40 | droplets = _MANAGER.get_all_droplets()
41 | return ['{}: {}'.format(x.name, x.id) for x in droplets]
42 |
--------------------------------------------------------------------------------
/vexbot/extensions/dynamic_loading.py:
--------------------------------------------------------------------------------
1 | def get_dynamic_extensions(self, *args, **kwargs):
2 | pass
3 |
--------------------------------------------------------------------------------
/vexbot/extensions/extensions.py:
--------------------------------------------------------------------------------
1 | import pip
2 | import pkg_resources
3 | from vexbot.extensions import extend as _extend
4 | from vexbot.extension_metadata import extensions as _meta_data
5 |
6 |
7 | def get_installed_extensions(self, *args, **kwargs):
8 | verify_requirements = True
9 | name = 'vexbot_extensions'
10 | if not args:
11 | extensions = ['{}: {}'.format(x.name, _meta_data[x.name].get('short', 'No Documentation')) for x in pkg_resources.iter_entry_points(name)]
12 | else:
13 | extensions = ['{}: {}'.format(x.name, _meta_data[x.name].get('short', 'No Documentation')) for x in pkg_resources.iter_entry_points(name) if x.module_name in args]
14 | return sorted(extensions, key=str.lower)
15 |
16 |
17 | def get_installed_modules(self, *args, **kwargs):
18 | result = set()
19 | [result.add(x.module_name) for x in pkg_resources.iter_entry_points('vexbot_extensions')]
20 | return result
21 |
22 |
23 | def _install(package) -> pkg_resources.Distribution:
24 | pip.main(['install', package.project_name])
25 | pkg_resources._initialize_master_working_set()
26 | return pkg_resources.get_distribution(package.project_name)
27 |
28 |
29 | def add_extensions(self,
30 | *args,
31 | alias: list=None,
32 | call_name=None,
33 | hidden: bool=False,
34 | update: bool=True,
35 | **kwargs):
36 |
37 | # FIXME: implement
38 | not_found = set()
39 | # NOTE: The dist should be used to figure out which name we want, not by grabbing blindly
40 | entry_points = [x for x in pkg_resources.iter_entry_points('vexbot_extensions') if x.name in args]
41 | for entry in entry_points:
42 | entry.require(installer=_install)
43 | function = entry.resolve()
44 | values = {'alias': alias, 'call_name': call_name, 'hidden': hidden, 'kwargs': kwargs}
45 | self._config['extensions'][entry.name] = values
46 | self.extend(function, alias=alias, name=call_name, hidden=hidden, update=False)
47 | self._config.sync()
48 | update_method = getattr(self, 'update_commands')
49 | if update and update_method:
50 | update_method()
51 | if not_found:
52 | raise RuntimeError('Packages not found/installed: {}'.format(not_found))
53 |
54 |
55 | def add_extensions_from_dict(self, extensions: dict):
56 | keys = tuple(extensions.keys())
57 | entry_points = [x for x in pkg_resources.iter_entry_points('vexbot_extensions') if x.name in keys]
58 | for entry in entry_points:
59 | entry.require(installer=_install)
60 | function = entry.resolve()
61 | values = dict(extensions[entry.name])
62 | kwargs = values.pop('kwargs', {})
63 | values.update(kwargs)
64 | if values.get('short') is None:
65 | meta = _meta_data.get(entry.name)
66 | if meta is None:
67 | values['short'] = _meta_data.get(getattr(function, '_meta', 'No Documentation'))
68 | else:
69 | values['short'] = meta.get('short', 'No Documentation')
70 |
71 | name = values.pop('call_name', function.__name__)
72 | _extend(self.__class__, function, **values)
73 |
74 |
75 | def get_extensions(self, *args, values: bool=False, **kwargs):
76 | if values:
77 | return self._config['extensions']
78 | return tuple(self._config['extensions'].keys())
79 |
80 |
81 | def remove_extension(self, *args, **kwargs) -> str:
82 | for arg in args:
83 | popped = self._config['extensions'].pop(arg, None)
84 |
--------------------------------------------------------------------------------
/vexbot/extensions/help.py:
--------------------------------------------------------------------------------
1 | def help(self, *arg, **kwargs):
2 | """
3 | Help helps you figure out what commands do.
4 | Example usage: !help code
5 | To see all commands: !commands
6 | """
7 | name = arg[0]
8 | try:
9 | callback = self._commands[name]
10 | except KeyError:
11 | self._logger.info(' !help not found for: %s', name)
12 | return self.do_help.__doc__
13 |
14 | return callback.__doc__
15 |
--------------------------------------------------------------------------------
/vexbot/extensions/hidden.py:
--------------------------------------------------------------------------------
1 | def hidden(self, *args, **kwargs):
2 | results = []
3 | for k, v in self._commands.items():
4 | if hasattr(v, 'hidden') and v.hidden:
5 | results.append('!' + k)
6 | else:
7 | continue
8 | return results
9 |
--------------------------------------------------------------------------------
/vexbot/extensions/intents.py:
--------------------------------------------------------------------------------
1 | def get_intents(self, *args, **kwargs):
2 | return self.bot.intents.get_intent_names()
3 |
4 |
5 | def get_intent(self, *args, **kwargs):
6 | return self.bot.intents.get_intent_names(*args, **kwargs)
7 |
--------------------------------------------------------------------------------
/vexbot/extensions/log.py:
--------------------------------------------------------------------------------
1 | import typing
2 | import logging
3 |
4 |
5 | def log_level(self,
6 | level: typing.Union[str, int]=None,
7 | *args,
8 | **kwargs) -> typing.Union[None, str]:
9 | """
10 | Args:
11 | level:
12 |
13 | Returns:
14 | The log level if a `level` is passed in
15 | """
16 | if level is None:
17 | return self.root_logger.getEffectiveLevel()
18 | # NOTE: `setLevel` takes both string and integers. Try to cast to an integer first
19 | try:
20 | value = int(level)
21 | # if we can't cast to an int, it's probably a string
22 | except ValueError:
23 | pass
24 |
25 | self.root_logger.setLevel(value)
26 |
27 |
28 | def set_debug(self, *args, **kwargs) -> None:
29 | self.root_logger.setLevel(logging.DEBUG)
30 | try:
31 | self.messaging.pub_handler.setLevel(logging.DEBUG)
32 | except Exception:
33 | pass
34 |
35 |
36 | def set_info(self, *args, **kwargs) -> None:
37 | """
38 | Sets the log level to `INFO`
39 | """
40 | self.root_logger.setLevel(logging.INFO)
41 | try:
42 | self.messaging.pub_handler.setLevel(logging.INFO)
43 | except Exception:
44 | pass
45 |
46 | def set_default(self, *args, **kwargs):
47 | self.root_logger.setLevel(logging.WARN)
48 |
49 |
50 | def filter_logs(self, *args, **kwargs):
51 | if not args:
52 | raise ValueError('Must supply something to filter against!')
53 | for name in args:
54 | for handler in self.root_logger.handlers:
55 | handler.addFilter(logging.Filter(name))
56 |
57 |
58 | def anti_filter(self, *args, **kwargs):
59 | def filter_(record: logging.LogRecord):
60 | if record.name in args:
61 | return False
62 | return True
63 | for handler in self.root_logger.handlers:
64 | handler.addFilter(filter_)
65 |
--------------------------------------------------------------------------------
/vexbot/extensions/modules.py:
--------------------------------------------------------------------------------
1 | import importlib
2 |
3 |
4 | def add_module(self, function_name: str, alias: list=None, call_name: str=None, hidden: bool=False, *args, **kwargs) -> None:
5 | # split off the last dot as the function name, the rest is the module
6 | module, function = function_name.rsplit('.', maxsplit=1)
7 | # strip off any whitespace
8 | function = function.rstrip()
9 | # import the module
10 | module = importlib.import_module(module, package=None)
11 | # get the function from the module
12 | function = getattr(module, function)
13 | self.extend(function, alias=alias, name=call_name, hidden=hidden)
14 |
15 | values = {'function_name': function_name, 'alias': alias, 'call_name': call_name, 'hidden': hidden}
16 |
17 | self._config['modules'][function_name] = values
18 | self._config.sync()
19 |
--------------------------------------------------------------------------------
/vexbot/extensions/news.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | import gensim
4 | import newspaper
5 |
6 |
7 | def summarize_article(self, url: str, *args, **kwargs):
8 | article = newspaper.Article(url)
9 | article.download()
10 | article.parse()
11 | summarization = gensim.summarization.summarize(article.text)
12 | return summarization
13 |
14 |
15 | def get_hot_trends(self, *args, **kwargs):
16 | return newspaper.hot()
17 |
18 |
19 | def get_popular_urls(self, *args, **kwargs):
20 | return newspaper.popular_urls()
21 |
--------------------------------------------------------------------------------
/vexbot/extensions/subprocess.py:
--------------------------------------------------------------------------------
1 | # from vexbot.intents import intent
2 |
3 |
4 | # @intent(name='restart_program')
5 | def restart(self, *args, mode: str='replace', **kwargs) -> None:
6 | self.logger.info(' restart service %s in mode %s', args, mode)
7 | for target in args:
8 | self.subprocess_manager.restart(target, mode)
9 |
10 |
11 | # @intent(CommandObserver, name='stop_program')
12 | def stop(self, *args, mode: str='replace', **kwargs) -> None:
13 | self.logger.info(' stop service %s in mode %s', args, mode)
14 | for target in args:
15 | self.subprocess_manager.stop(target, mode)
16 |
17 |
18 | # @intent(CommandObserver, name='get_status')
19 | def status(self, name: str, *args, **kwargs) -> str:
20 | self.logger.debug(' get status for %s', name)
21 | return self.subprocess_manager.status(name)
22 |
23 |
24 | # @intent(name='start_program')
25 | def start(self, *args, mode: str='replace', **kwargs) -> None:
26 | """
27 | Start a program.
28 |
29 | Args:
30 | name: The name of the serivce. Will often end with `.service` or
31 | `.target`. If no argument is provided, will default to
32 | `.service`
33 | mode:
34 |
35 | Raises:
36 | GError: if the service is not found
37 | """
38 | for target in args:
39 | self.logger.info(' start service %s in mode %s', target, mode)
40 | self.subprocess_manager.start(target, mode)
41 |
42 |
43 | def uptime(self, *args, name: str='vexbot.service', **kwargs):
44 | return self.subprocess_manager.uptime(name)
45 |
46 |
47 | uptime._meta = 'process_uptime'
48 | restart._meta = 'restart_process'
49 | start._meta = 'start_process'
50 | status._meta = 'status_process'
51 | stop._meta = 'stop_process'
52 |
--------------------------------------------------------------------------------
/vexbot/extensions/system.py:
--------------------------------------------------------------------------------
1 | import psutil
2 |
3 | _mb_conversion = 1024 * 1024
4 |
5 | def cpu_times(*args, **kwargs):
6 | return psutil.cpu_times()
7 |
8 |
9 | def cpu_count(logical=True, *args, **kwargs):
10 | cores = psutil.cpu_count(logical)
11 | if cores == 1:
12 | word = 'Core'
13 | else:
14 | word = 'Cores'
15 | return '{} CPU {}'.format(cores, word)
16 |
17 |
18 | def cpu_frequency(*args, **kwargs):
19 | freq = psutil.cpu_freq()
20 | if freq is None:
21 | return ('CPU frequency file moved or not present. See: '
22 | 'https://stackoverflow.com/questions/42979943/python3-psutils')
23 |
24 | return [x.max for x in freq]
25 |
26 |
27 | def virtual_memory_percent(*arg, **kwargs):
28 | percent = psutil.virtual_memory().percent
29 | return '{}%'.format(percent)
30 |
31 |
32 | def virtual_memory_total(*args, **kwargs):
33 | total = int(psutil.virtual_memory().total / _mb_conversion)
34 | return '{} Mb'.format(total)
35 |
36 |
37 | def virtual_memory_used(*args, **kwargs):
38 | used = int(psutil.virtual_memory().used / _mb_conversion)
39 | return '{} Mb'.format(used)
40 |
41 |
42 | def swap(*args, **kwargs):
43 | swap = psutil.swap_memory()
44 | used = swap.used
45 | total = swap.total
46 | used = int(used/_mb_conversion)
47 | total = int(total/_mb_conversion)
48 | return 'Used: {} | Total: {}'.format(used, total)
49 |
--------------------------------------------------------------------------------
/vexbot/intents.py:
--------------------------------------------------------------------------------
1 | import inspect as _inspect
2 | import functools as _functools
3 |
4 |
5 | # http://rasa-nlu.readthedocs.io/en/latest/tutorial.html#preparing-the-training-data
6 | def intent(function=None,
7 | name: str=None):
8 |
9 | if function is None:
10 | return _functools.partial(intent,
11 | name=name)
12 |
13 | function._vex_intent = True
14 | function._vex_intent_name = name
15 | return function
16 |
17 |
18 | class Entity:
19 | def __init__(self, text: str, start: int, end: int, name: str, type_: str, value: str=None):
20 | """
21 | Args:
22 | text: Example utterance
23 | start: Starting position of the value
24 | end: Ending position of the value
25 | name: Name of the entity
26 | type_: Entity Type
27 | """
28 | self.text = text
29 | self.start = start
30 | self.end = end
31 | self.name = name
32 | self.type = type_
33 | if value is None:
34 | value = name
35 | self.value = value
36 |
37 |
38 | class FindEntity:
39 | def __init__(text, value, name):
40 | pass
41 |
--------------------------------------------------------------------------------
/vexbot/language.py:
--------------------------------------------------------------------------------
1 | import code
2 | import pickle
3 | import logging
4 | import multiprocessing
5 |
6 | import numpy as np
7 | import spacy
8 |
9 | from sklearn.svm import SVC
10 | from sklearn.preprocessing import LabelEncoder
11 | from sklearn.model_selection import GridSearchCV
12 |
13 | from vexbot.entity_extraction import EntityExtraction
14 | from vexbot.util.get_classifier_filepath import (get_classifier_filepath,
15 | get_entity_filepath)
16 |
17 |
18 | MAX_CV_FOLDS = 5
19 |
20 |
21 | class Language:
22 | def __init__(self, classifier_filepath: str=None, language: str='en'):
23 | if classifier_filepath is None:
24 | classifier_filepath = get_classifier_filepath()
25 |
26 | self._classifier_filepath = classifier_filepath
27 | self.label_encoder = LabelEncoder()
28 | self.feature_extractor = None
29 | self.logger = logging.getLogger(__name__)
30 | self.language_model = None
31 | self._language_model_loaded = False
32 | self.classifier = None
33 | self.synonyms = {}
34 | self.entity_extractor = EntityExtraction()
35 |
36 | def predict(self, X):
37 | pred_result = self.classifier.predict_proba(X)
38 | sorted_indicies = np.fliplr(np.argsort(pred_result, axis=1))
39 | return sorted_indicies, pred_result[:, sorted_indicies]
40 |
41 | def get_intent(self, text: str, entities: list=None, *args, **kwargs):
42 | # Text classification
43 | # Entity extraction
44 | if self.classifier is None:
45 | self.load_classifier()
46 | if not self._language_model_loaded:
47 | self._load_models()
48 |
49 | langague_features = self.language_model(text).vector
50 | langague_features = langague_features.reshape(1, -1)
51 | intents, probabilities = self.predict(langague_features)
52 | if intents.size > 0 and probabilities.size > 0:
53 | ranking = list(zip(list(intents), list(probabilities)))[:10]
54 | intent_ranking = [{"name": self.label_encoder.inverse_transform(intent_name), "confidence": score} for intent_name, score in ranking]
55 | first_name = intent_ranking[0]['name'][0]
56 | first_confidence = intent_ranking[0]['confidence'][0]
57 | else:
58 | first_name = None
59 | first_confidence = 0
60 | self.logger.debug('name: %s', first_name)
61 | self.logger.debug('confidence: %s', first_confidence)
62 | extracted = self.entity_extractor.get_entites(text, entities)
63 | # TODO: clean or merge entities here?
64 |
65 | return first_name, first_confidence, entities
66 |
67 | # text -> tokens
68 | # text -> entities
69 | # tokens + entites -> intent
70 |
71 | # total_word_feature_extractor
72 |
73 | # FIXME: currently unused
74 | def train(self, examples: dict):
75 | self.train_classifier(examples)
76 | # FIXME: Finish method
77 | # self.train_entity_extractor(examples)
78 |
79 | def _load_models(self):
80 | language = 'en'
81 | self.language_model = spacy.load(language, parser=False)
82 | self.entity_extractor._language_model = self.language_model
83 | self._language_model_loaded = True
84 |
85 | def load_classifier(self, filename: str=None):
86 | if filename is None:
87 | filename = self._classifier_filepath
88 |
89 | # FIXME: this code is super brittle for various reasons
90 | with open(filename, 'rb') as f:
91 | self.classifier = pickle.load(f)
92 | with open(filename + 'label', 'rb') as f:
93 | self.label_encoder = pickle.load(f)
94 |
95 | def get_spacy_text_features(self, text: str):
96 | # type: np.ndarray
97 | doc = self.language_model(text)
98 | vector = doc.vector
99 | return vector
100 |
101 | def train_classifier(self, examples: dict, filename: str=None, persist=True):
102 | # tokenizer_spacy -> Tokenizes -> Eh.
103 | # NOTE: this should keep track of a numpy list that is [#, 0/1] showing
104 | # which known regex pattern and if it was found (1) or not found (0)
105 | # AKA a feature matrix
106 |
107 | # intent_entity_featurizer_regex |entity| -> regex
108 |
109 | # NOTE: Start here
110 | # nlp_spacy -> spacy language initializer -> Done
111 | # intent_featurizer_spacy -> adds text features -> Eh.
112 |
113 | # ner_crf |entity| -> conditional random field entity extraction -> crf_entity_extractor
114 | # ner_synonyms |entity| -> dict replacing the entity values bascially
115 |
116 | # intent_classifier_sklearn -> end of pipeline
117 | if not self._language_model_loaded:
118 | self._load_models()
119 |
120 | if filename is None:
121 | filename = self._classifier_filepath
122 |
123 | self.logger.debug('train classifier filepath: %s', filename)
124 | # - Y -
125 | numbers = []
126 | # text_features
127 | # - X -
128 | values = []
129 | for k, v in examples.items():
130 | self.logger.debug('intent: %s', k)
131 | # FIXME: I don't think I want this to be a callback.
132 | if v is None:
133 | continue
134 | for value in v:
135 | numbers.append(k)
136 | language_values = self.language_model(value)
137 | # get the vector from spacy
138 | values.append(language_values.vector)
139 |
140 | values = np.stack(values)
141 | # Y
142 | numbers = self.label_encoder.fit_transform(numbers)
143 | # aim for 5 examples in each fold
144 | cv_splits = max(2,
145 | min(MAX_CV_FOLDS,
146 | np.min(np.bincount(numbers)) // 5))
147 |
148 | estimator = SVC(C=1, probability=True, class_weight='balanced')
149 |
150 | number_threads = multiprocessing.cpu_count()
151 | kernel = 'linear'
152 | # NOTE: the `kernel` parameter needs to be a list for sklearn
153 | tuned_parameters = [{'C': [1, 2, 5, 10, 20, 100],
154 | 'kernel': [kernel]}]
155 | # create the classifier
156 | self.classifier = GridSearchCV(estimator,
157 | param_grid=tuned_parameters,
158 | n_jobs=number_threads,
159 | cv=cv_splits,
160 | scoring='f1_weighted',
161 | verbose=1)
162 |
163 | # self.classifier.fit(X, y)
164 | self.classifier.fit(values, numbers)
165 | if persist:
166 | with open(filename, 'wb') as f:
167 | pickle.dump(self.classifier, f)
168 | with open(filename + 'label', 'wb') as f:
169 | pickle.dump(self.label_encoder, f)
170 |
171 | def get_entities(self, text: str, *args, **kwargs):
172 | pass
173 |
174 | # FIXME: Finish
175 | # https://github.com/RasaHQ/rasa_nlu/blob/master/rasa_nlu/extractors/mitie_entity_extractor.py
176 | def train_entity_extractor(self, examples: list, filename: str=None):
177 | if filename is None:
178 | filename = get_entity_filepath()
179 |
180 | trainer = mitie.ner_trainer(filename)
181 | trainer.num_threads = multiprocessing.cpu_count()
182 | for example in examples:
183 | tokens = self.tokenize(example['text'])
184 | for entity in example['entities']:
185 | pass
186 |
187 |
188 | def tokenize(self, text: str):
189 | return text.split()
190 |
--------------------------------------------------------------------------------
/vexbot/observer.py:
--------------------------------------------------------------------------------
1 | from abc import ABCMeta as _ABCMeta
2 | from abc import abstractmethod as _abstractmethod
3 | from vexbot.extensions import extendmany as _extendmany
4 | from vexbot.extensions import extend as _extend
5 |
6 |
7 | class Observer(metaclass=_ABCMeta):
8 | extensions = ()
9 | def __init__(self, *args, **kwargs):
10 | extensions = list(self.extensions)
11 | dicts = [x for x in extensions if isinstance(x, dict)]
12 | for d in dicts:
13 | extensions.remove(d)
14 | self.extend(**d, update=False)
15 | _extendmany(self.__class__, *extensions)
16 |
17 | @_abstractmethod
18 | def on_next(self, value):
19 | return NotImplemented
20 |
21 | @_abstractmethod
22 | def on_error(self, error):
23 | return NotImplemented
24 |
25 | @_abstractmethod
26 | def on_completed(self):
27 | return NotImplemented
28 |
29 | def extend(self,
30 | method,
31 | alias: list=None,
32 | name: str=None,
33 | hidden: bool=False,
34 | instancemethod: bool=False,
35 | roles: list=None,
36 | update: bool=True,
37 | short: str=None):
38 |
39 | _extend(self.__class__, method, alias, name, hidden, instancemethod, roles, short)
40 | update_method = getattr(self, 'update_commands')
41 | if update and update_method:
42 | update_method()
43 |
44 | def extendmany(self, *methods):
45 | _extendmany(self.__class__, *methods)
46 | update = getattr(self, 'update_commands')
47 | if update:
48 | update()
49 |
--------------------------------------------------------------------------------
/vexbot/robot.py:
--------------------------------------------------------------------------------
1 | import sys as _sys
2 | import logging as _logging
3 |
4 | from vexbot import _get_default_port_config
5 | # from vexbot.intents import BotIntents
6 | from vexbot.messaging import Messaging as _Messaging
7 | from vexbot.adapters.shell.observers import LogObserver
8 | from vexbot.command_observer import CommandObserver as _CommandObserver
9 |
10 | try:
11 | from vexbot.subprocess_manager import SubprocessManager
12 | except ImportError:
13 | SubprocessManager = False
14 | _logging.exception('Import Error for subprocess manager!')
15 |
16 | try:
17 | from vexbot.language import Language
18 | except ImportError:
19 | Language = False
20 | _logging.exception('Import Error for language!')
21 |
22 | class Robot:
23 | def __init__(self,
24 | bot_name: str,
25 | connection: dict=None,
26 | subprocess_manager=None):
27 |
28 | if connection is None:
29 | connection = _get_default_port_config()
30 |
31 | self.messaging = _Messaging(bot_name, **connection)
32 | log_name = __name__ if __name__ != '__main__' else 'vexbot.robot'
33 | self._logger = _logging.getLogger(log_name)
34 | if subprocess_manager is None and SubprocessManager:
35 | subprocess_manager = SubprocessManager()
36 | else:
37 | err = ('If you would like to use the subporcess manager, please '
38 | 'run `pip install -e .[process_manager]` from the '
39 | 'directory')
40 |
41 | self._logger.warn(err)
42 |
43 | # NOTE: probably need some kind of config here
44 | if Language:
45 | self.language = Language()
46 | else:
47 | err = ('If you would like to use natural language processing,'
48 | ' please run `pip install -e.[nlp]` from the directory')
49 | self._logger.warn(err)
50 | self.language = False
51 |
52 | # self.intents = BotIntents()
53 | self.subprocess_manager = subprocess_manager
54 | self.command_observer = _CommandObserver(self,
55 | self.messaging,
56 | self.subprocess_manager,
57 | self.language)
58 |
59 | self.messaging.command.subscribe(self.command_observer)
60 | self.messaging.chatter.subscribe(LogObserver(pass_through=True))
61 |
62 | def run(self):
63 | if self.messaging is None:
64 | e = ' No `messaging` provided to `Robot` on initialization'
65 | self._logger.error(e)
66 | _sys.exit(1)
67 |
68 | self._logger.info(' Start Messaging')
69 | # NOTE: blocking call
70 | try:
71 | self.messaging.start()
72 | except KeyboardInterrupt:
73 | _sys.exit(0)
74 |
--------------------------------------------------------------------------------
/vexbot/scheduler.py:
--------------------------------------------------------------------------------
1 | from zmq.eventloop.zmqstream import ZMQStream
2 |
3 |
4 | class Scheduler:
5 | def __init__(self):
6 | # FIXME: Move into the SocketFactory
7 | self._streams = []
8 |
9 | def register_socket(self, socket, loop, on_recv):
10 | stream = ZMQStream(socket, loop)
11 | stream.on_recv(on_recv)
12 | self._streams.append(stream)
13 |
--------------------------------------------------------------------------------
/vexbot/subprocess_manager.py:
--------------------------------------------------------------------------------
1 | import time
2 | # import atexit
3 | # import signal
4 |
5 |
6 | # TODO: try/catch around this. If pydbus isn't installed, probably an
7 | # incomplete installation
8 | from pydbus import SessionBus as _SessionBus
9 | from pydbus import SystemBus as _SystemBus
10 |
11 |
12 | def _name_helper(name: str):
13 | """
14 | default to returning a name with `.service`
15 | """
16 | name = name.rstrip()
17 | if name.endswith(('.service', '.socket', '.target')):
18 | return name
19 | return name + '.service'
20 |
21 |
22 | def _pretty_time_delta(seconds):
23 | seconds = abs(int(seconds))
24 | days, seconds = divmod(seconds, 86400)
25 | hours, seconds = divmod(seconds, 3600)
26 | minutes, seconds = divmod(seconds, 60)
27 | if days > 0:
28 | return '%dday %dh' % (days, hours)
29 | elif hours > 0:
30 | return '%dh %dmin' % (hours, minutes)
31 | elif minutes > 0:
32 | return '%dmin %dsec' % (minutes, seconds)
33 | else:
34 | return '{}s'.format(seconds)
35 |
36 |
37 | # TODO: think of a better name
38 | class SubprocessManager:
39 | def __init__(self):
40 | self.session_bus_available = True
41 | try:
42 | self.bus = _SessionBus()
43 | # NOTE: GError from package `gi`
44 | except Exception:
45 | # No session bus if we're here. Depending on linux distro, that's
46 | # not surprising
47 | self.session_bus_available = False
48 |
49 | # TODO: It's possible that the user is on a system that is not using
50 | # systemd, which means that this next call will fail. Should probably
51 | # have a try/catch and then just default to not having a subprocess
52 | # manager if that happens.
53 | self.system_bus = _SystemBus()
54 |
55 | # TODO: Verify that we can start services as the system bus w/o root
56 | # permissions
57 | if self.session_bus_available:
58 | self.systemd = self.bus.get('.systemd1')
59 | else:
60 | self.systemd = self.system_bus.get('.systemd1')
61 |
62 | # atexit.register(self._close_subprocesses)
63 | # signal.signal(signal.SIGINT, self._handle_close_signal)
64 | # signal.signal(signal.SIGTERM, self._handle_close_signal)
65 |
66 | def start(self, name: str, mode: str='replace') -> None:
67 | # TODO: add in some parsing if fail
68 | # https://github.com/LEW21/pydbus/issues/35
69 | name = _name_helper(name)
70 | self.systemd.StartUnit(name, mode)
71 |
72 | def restart(self, name: str, mode: str='replace') -> None:
73 | name = _name_helper(name)
74 | self.systemd.ReloadOrRestartUnit(name, mode)
75 |
76 | def stop(self, name: str, mode: str='replace') -> None:
77 | name = _name_helper(name)
78 | self.systemd.StopUnit(name, mode)
79 |
80 | def _uptime(self, unit):
81 | time_start = unit.ConditionTimestamp/1000000
82 | delta = time.time() - time_start
83 | delta = _pretty_time_delta(delta)
84 | return delta
85 |
86 | def uptime(self, name: str) -> str:
87 | unit = self.bus.get('.systemd1', self.systemd.GetUnit(name))
88 | return self._uptime(unit)
89 |
90 | def status(self, name: str) -> str:
91 | name = _name_helper(name)
92 | unit = self.bus.get('.systemd1', self.systemd.GetUnit(name))
93 | # NOTE: what systemctl status shows
94 | # freenode.service - IRC Client
95 | # active (running) since Tue 2017-10-17 10:36:03 UTC; 19s ago
96 | delta = self._uptime(unit)
97 | fo = '%a %b %d %H:%M:%S %Y'
98 | time_start = time_start = unit.ConditionTimestamp/1000000
99 | time_stamp = time.strftime(fo, time.gmtime(time_start))
100 | return '{}: {} ({}) since {}; {} ago'.format(unit.Id,
101 | unit.ActiveState,
102 | unit.SubState,
103 | time_stamp,
104 | delta)
105 |
106 | def get_units(self):
107 | return self.systemd.ListUnits()
108 |
109 | """
110 | def mask(self, name: str):
111 | pass
112 |
113 | def unmask(self, name: str):
114 | pass
115 |
116 | def killall(self):
117 | pass
118 | """
119 |
--------------------------------------------------------------------------------
/vexbot/util/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/benhoff/vexbot/9b844eb20e84eea92a0e7db7d86a90094956c38f/vexbot/util/__init__.py
--------------------------------------------------------------------------------
/vexbot/util/create_cache_filepath.py:
--------------------------------------------------------------------------------
1 | from os import path, mkdir
2 | from vexbot.util.get_cache_filepath import get_cache_filepath
3 |
4 |
5 | def create_cache_directory():
6 | home_direct = path.expanduser("~")
7 | cache = path.abspath(path.join(home_direct,
8 | '.cache'))
9 |
10 | if not path.isdir(cache):
11 | mkdir(cache)
12 | cache = path.join(cache, 'vexbot')
13 | if not path.isdir(cache):
14 | mkdir(cache)
15 |
16 |
--------------------------------------------------------------------------------
/vexbot/util/create_config_file.py:
--------------------------------------------------------------------------------
1 | from os import path
2 | from configparser import ConfigParser as _ConfigParser
3 |
4 | from vexbot import _get_default_port_config
5 | from vexbot.util.get_config_filepath import get_config_filepath as _get_config_filepath
6 |
7 |
8 | def create_config_file(settings=None):
9 | filepath = _get_config_filepath()
10 | if path.isfile(filepath):
11 | return
12 |
13 | config = _ConfigParser()
14 | config['vexbot'] = {'kill_on_exit': False,
15 | 'profile': 'default'}
16 |
17 | config['vexbot_ports'] = _get_default_port_config()
18 |
19 | with open(filepath, 'w') as f:
20 | config.write(f)
21 |
--------------------------------------------------------------------------------
/vexbot/util/create_vexdir.py:
--------------------------------------------------------------------------------
1 | from os import path, mkdir
2 |
3 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath as _get_vexdir_filepath
4 | from vexbot.util.get_vexdir_filepath import get_config_dir
5 |
6 |
7 | def create_vexdir():
8 | config_dir = get_config_dir()
9 | vexdir = _get_vexdir_filepath()
10 |
11 | if not path.isdir(config_dir):
12 | mkdir(config_dir)
13 | if not path.isdir(vexdir):
14 | mkdir(vexdir)
15 |
16 | return vexdir
17 |
--------------------------------------------------------------------------------
/vexbot/util/generate_certificates.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Generate client and server CURVE certificate files then move them into the
4 | appropriate store directory, private_keys or public_keys.
5 |
6 | Adapted from:
7 | https://github.com/zeromq/pyzmq/blob/master/examples/security/generate_certificates.py
8 |
9 | Original Author: Chris Laws
10 | """
11 |
12 | import os
13 | import sys
14 | import shutil
15 | import logging
16 | import zmq.auth
17 |
18 | from prompt_toolkit import prompt
19 |
20 | from vexbot.util.create_vexdir import create_vexdir
21 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath
22 | from vexbot.util.get_certificate_filepath import get_certificate_filepath
23 |
24 |
25 | def generate_certificates(base_dir: str, remove_certificates: bool=False):
26 | ''' Generate client and server CURVE certificate files'''
27 | public_keys_dir = os.path.join(base_dir, 'public_keys')
28 | secret_keys_dir = os.path.join(base_dir, 'private_keys')
29 |
30 | # Make the public and private key directories
31 | for path in (public_keys_dir, secret_keys_dir):
32 | if not os.path.exists(path):
33 | os.mkdir(path)
34 |
35 | # create new keys in certificates dir
36 | server_public_file, server_secret_file = zmq.auth.create_certificates(base_dir, "vexbot")
37 | client_public_file, client_secret_file = zmq.auth.create_certificates(base_dir, "client")
38 |
39 | vexbot_public = os.path.join(public_keys_dir, 'vexbot.key')
40 | client_public = os.path.join(public_keys_dir, 'client.key')
41 |
42 | vexbot_secret = os.path.join(secret_keys_dir, 'vexbot.key_secret')
43 | client_secret = os.path.join(secret_keys_dir, 'client.key_secret')
44 |
45 | if os.path.exists(vexbot_public) and remove_certificates:
46 | os.unlink(vexbot_public)
47 | try:
48 | os.unlink(vexbot_secret)
49 | except OSError:
50 | pass
51 | elif os.path.exists(vexbot_public) and not remove_certificates:
52 | os.unlink(server_public_file)
53 | os.unlink(server_secret_file)
54 |
55 | if os.path.exists(client_public) and remove_certificates:
56 | os.unlink(client_public)
57 | try:
58 | os.unlink(client_secret)
59 | except OSError:
60 | pass
61 | elif os.path.exists(client_public) and not remove_certificates:
62 | os.unlink(client_public_file)
63 | os.unlink(client_secret_file)
64 |
65 | # move public keys to appropriate directory
66 | for key_file in os.listdir(base_dir):
67 | if key_file.endswith(".key"):
68 | shutil.move(os.path.join(base_dir, key_file),
69 | os.path.join(public_keys_dir, '.'))
70 |
71 | # move secret keys to appropriate directory
72 | for key_file in os.listdir(base_dir):
73 | if key_file.endswith(".key_secret"):
74 | shutil.move(os.path.join(base_dir, key_file),
75 | os.path.join(secret_keys_dir, '.'))
76 |
77 |
78 | def _check_vexbot_filepath(create_if_missing=True):
79 | filepath = get_vexdir_filepath()
80 | is_missing = os.path.isdir(filepath)
81 |
82 | if create_if_missing and is_missing:
83 | create_vexdir()
84 | elif is_missing and not create_if_missing:
85 | logging.error('Vexbot filepath not found! Please create')
86 |
87 |
88 | def main():
89 | _check_vexbot_filepath(create_if_missing=True)
90 | cert_path = get_certificate_filepath()
91 | if not os.path.exists(cert_path):
92 | os.makedirs(cert_path, exist_ok=True)
93 |
94 | remove_certificates = input('Remove certificates if present? Y/n: ')
95 | remove_certificates = remove_certificates.lower()
96 | if remove_certificates == 'y':
97 | remove_certificates = True
98 | else:
99 | remove_certificates = False
100 | generate_certificates(cert_path, remove_certificates)
101 |
102 |
103 | if __name__ == '__main__':
104 | main()
105 |
--------------------------------------------------------------------------------
/vexbot/util/generate_config_file.py:
--------------------------------------------------------------------------------
1 | import os
2 | import logging
3 | import inspect
4 | import configparser
5 | import pkg_resources
6 | from os import path
7 |
8 | from vexbot.util.get_vexdir_filepath import get_config_dir
9 |
10 |
11 | def _systemd_user_filepath() -> str:
12 | config = get_config_dir()
13 | config = path.join(config, 'systemd', 'user')
14 | os.makedirs(config, exist_ok=True)
15 | return path.join(config, 'vexbot.service')
16 |
17 |
18 | def _get_vexbot_robot() -> str:
19 | for entry_point in pkg_resources.iter_entry_points('console_scripts'):
20 | if entry_point.name == 'vexbot_robot' and entry_point.dist.project_name == 'vexbot':
21 | # TODO: call the `require` function first and provide an installer
22 | # so that we can get the matching `Distribution` instance?
23 | return inspect.getsourcefile(entry_point.resolve())
24 |
25 | raise RuntimeError('Entry point for `vexbot_robot` not found! Are you '
26 | 'sure it\'s installed?')
27 |
28 |
29 | def config(filepath=None, remove_config=False):
30 | if filepath is None:
31 | filepath = _systemd_user_filepath()
32 | if path.exists(filepath) and remove_config:
33 | os.unlink(filepath)
34 | elif path.exists(filepath) and not remove_config:
35 | logging.error(' Configuration filepath already exsists at: %s',
36 | filepath)
37 | return
38 |
39 | parser = configparser.SafeConfigParser()
40 | parser.optionxform = str
41 | parser['Unit'] = {'Description': 'Helper Bot'}
42 | parser['Service'] = {'Type': 'simple',
43 | 'ExecStart': _get_vexbot_robot(),
44 | 'StandardOutput': 'syslog',
45 | 'StandardError': 'syslog'}
46 |
47 | with open(filepath, 'w') as f:
48 | parser.write(f)
49 |
50 | print('Config file created at: {}'.format(filepath))
51 |
52 |
53 | def main():
54 | remove_config = input('Remove config if present? [Y]/n: ')
55 | remove_config = remove_config.lower()
56 | if remove_config in ('y', 'yes', 'ye', ''):
57 | remove_config = True
58 | else:
59 | remove_config = False
60 |
61 | config(remove_config=remove_config)
62 |
63 |
64 | if __name__ == '__main__':
65 | main()
66 |
--------------------------------------------------------------------------------
/vexbot/util/get_cache_filepath.py:
--------------------------------------------------------------------------------
1 | from os import path
2 |
3 |
4 | def get_cache_filepath() -> str:
5 | home_direct = path.expanduser("~")
6 | cache = path.abspath(path.join(home_direct,
7 | '.cache'))
8 | cache = path.join(cache, 'vexbot')
9 | return cache
10 |
11 |
12 | def get_cache(name: str) -> str:
13 | filepath = get_cache_filepath()
14 | filepath = path.join(filepath, name)
15 | return filepath
16 |
--------------------------------------------------------------------------------
/vexbot/util/get_certificate_filepath.py:
--------------------------------------------------------------------------------
1 | from os import path
2 |
3 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath
4 |
5 |
6 | def get_certificate_filepath():
7 | vexdir = get_vexdir_filepath()
8 | certs = path.join(vexdir, 'certificates')
9 | return certs
10 |
11 | def get_certificate_directories() -> (str, str):
12 | root = get_certificate_filepath()
13 | public_dir = 'public_keys'
14 | private_dir = 'private_keys'
15 | public_dir = path.join(root, public_dir)
16 | private_dir = path.join(root, private_dir)
17 | return public_dir, private_dir
18 |
19 | def _certificate_helper(public_filepath: str,
20 | secret_filepath: str) -> (str, str):
21 |
22 | public_dir, private_dir = get_certificate_directories()
23 | secret_filepath = path.join(private_dir, secret_filepath)
24 | public_filepath = path.join(public_dir, public_filepath)
25 | return public_filepath, secret_filepath
26 |
27 |
28 | def get_vexbot_certificate_filepath() -> (str, bool):
29 | """
30 | Returns the vexbot certificate filepath and the whether it is the private
31 | filepath or not
32 | """
33 | public_filepath, secret_filepath = _certificate_helper('vexbot.key','vexbot.key_secret')
34 | if path.isfile(secret_filepath):
35 | return secret_filepath, True
36 | if not path.isfile(public_filepath):
37 | err = ('certificates not found. Generate certificates from the '
38 | 'command line using `vexbot_generate_certificates`')
39 | raise FileNotFoundError(err)
40 | return public_filepath, False
41 |
42 |
43 | def get_client_certificate_filepath() -> (str, bool):
44 | _, secret_filepath = _certificate_helper('client.key','client.key_secret')
45 | if not path.isfile(secret_filepath):
46 | err = 'certificates not found. Where they generated?'
47 | raise FileNotFoundError(err)
48 | return secret_filepath
49 |
--------------------------------------------------------------------------------
/vexbot/util/get_classifier_filepath.py:
--------------------------------------------------------------------------------
1 | from os import path
2 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath
3 |
4 |
5 | def get_classifier_filepath(name: str=None) -> str:
6 | if name is None:
7 | name = 'classifier.vex'
8 | directory = get_vexdir_filepath()
9 | return path.join(directory, name)
10 |
11 |
12 | def get_entity_filepath(name: str=None) -> str:
13 | if name is None:
14 | name = 'entity_extractor.vex'
15 | directory = get_vexdir_filepath()
16 | return path.join(directory, name)
17 |
--------------------------------------------------------------------------------
/vexbot/util/get_config.py:
--------------------------------------------------------------------------------
1 | from configparser import ConfigParser as _ConfigParser
2 |
3 | from vexbot.util.get_config_filepath import get_config_filepath as _get_config_filepath
4 |
5 |
6 | def get_config(filepath=None):
7 | if filepath is None:
8 | filepath = _get_config_filepath()
9 | config = _ConfigParser()
10 | config.read(filepath)
11 | config_dict = {s: dict(config.items(s))
12 | for s
13 | in config.sections()}
14 |
15 | return config_dict
16 |
--------------------------------------------------------------------------------
/vexbot/util/get_config_filepath.py:
--------------------------------------------------------------------------------
1 | from os import path
2 |
3 | from vexbot.util.get_vexdir_filepath import get_vexdir_filepath as _get_vexdir_filepath
4 |
5 |
6 | def get_config_filepath():
7 | vexdir = _get_vexdir_filepath()
8 | config_filepath = path.join(vexdir, 'vexbot.ini')
9 | return config_filepath
10 |
--------------------------------------------------------------------------------
/vexbot/util/get_kwargs.py:
--------------------------------------------------------------------------------
1 | import argparse as _argparse
2 |
3 |
4 | def get_kwargs():
5 | parser = _argparse.ArgumentParser()
6 | parser.add_argument('--configuration_filepath', default=None)
7 | args = parser.parse_args()
8 | return vars(args)
9 |
--------------------------------------------------------------------------------
/vexbot/util/get_vexdir_filepath.py:
--------------------------------------------------------------------------------
1 | from os import path
2 |
3 |
4 | def get_config_dir():
5 | home_direct = path.expanduser("~")
6 | config = path.abspath(path.join(home_direct,
7 | '.config'))
8 |
9 | return config
10 |
11 |
12 | def get_vexdir_filepath():
13 | vexdir = path.join(get_config_dir(),
14 | 'vexbot')
15 |
16 | return vexdir
17 |
--------------------------------------------------------------------------------
/vexbot/util/lru_cache.py:
--------------------------------------------------------------------------------
1 | # Class for the node objects.
2 | class _dlnode(object):
3 | def __init__(self):
4 | self.empty = True
5 |
6 |
7 | # https://github.com/jlhutch/pylru
8 | class LRUCache(object):
9 |
10 | def __init__(self, size, add_callback=None, delete_callback=None):
11 |
12 | self.add_callback = add_callback
13 | self.delete_callback = delete_callback
14 |
15 | # Create an empty hash table.
16 | self.table = {}
17 |
18 | # Initialize the doubly linked list with one empty node. This is an
19 | # invariant. The cache size must always be greater than zero. Each
20 | # node has a 'prev' and 'next' variable to hold the node that comes
21 | # before it and after it respectively. Initially the two variables
22 | # each point to the head node itself, creating a circular doubly
23 | # linked list of size one. Then the size() method is used to adjust
24 | # the list to the desired size.
25 |
26 | self.head = _dlnode()
27 | self.head.next = self.head
28 | self.head.prev = self.head
29 |
30 | self.listSize = 1
31 |
32 | # Adjust the size
33 | self.size(size)
34 |
35 | def __len__(self):
36 | return len(self.table)
37 |
38 | def clear(self):
39 | for node in self.dli():
40 | node.empty = True
41 | node.key = None
42 | node.value = None
43 |
44 | self.table.clear()
45 |
46 | def __contains__(self, key):
47 | return key in self.table
48 |
49 | # Looks up a value in the cache without affecting cache order.
50 | def peek(self, key):
51 | # Look up the node
52 | node = self.table[key]
53 | return node.value
54 |
55 | def __getitem__(self, key):
56 | # Look up the node
57 | node = self.table[key]
58 |
59 | # Update the list ordering. Move this node so that is directly
60 | # proceeds the head node. Then set the 'head' variable to it. This
61 | # makes it the new head of the list.
62 | self.mtf(node)
63 | self.head = node
64 |
65 | # Return the value.
66 | return node.value
67 |
68 | def get(self, key, default=None):
69 | """Get an item - return default (None) if not present"""
70 | if key not in self.table:
71 | return default
72 |
73 | return self[key]
74 |
75 | def __setitem__(self, key, value):
76 | # First, see if any value is stored under 'key' in the cache already.
77 | # If so we are going to replace that value with the new one.
78 | if key in self.table:
79 |
80 | # Lookup the node
81 | node = self.table[key]
82 |
83 | # Replace the value.
84 | node.value = value
85 |
86 | # Update the list ordering.
87 | self.mtf(node)
88 | self.head = node
89 |
90 | return
91 |
92 | # TODO First add the value here
93 | if self.add_callback is not None:
94 | self.add_callback(key)
95 |
96 | # Ok, no value is currently stored under 'key' in the cache. We need
97 | # to choose a node to place the new item in. There are two cases. If
98 | # the cache is full some item will have to be pushed out of the
99 | # cache. We want to choose the node with the least recently used
100 | # item. This is the node at the tail of the list. If the cache is not
101 | # full we want to choose a node that is empty. Because of the way the
102 | # list is managed, the empty nodes are always together at the tail
103 | # end of the list. Thus, in either case, by chooseing the node at the
104 | # tail of the list our conditions are satisfied.
105 |
106 | # Since the list is circular, the tail node directly preceeds the
107 | # 'head' node.
108 | node = self.head.prev
109 |
110 | # If the node already contains something we need to remove the old
111 | # key from the dictionary.
112 | if not node.empty:
113 | if self.delete_callback is not None:
114 | self.delete_callback(node.key)
115 | del self.table[node.key]
116 |
117 | # Place the new key and value in the node
118 | node.empty = False
119 | node.key = key
120 | node.value = value
121 |
122 | # Add the node to the dictionary under the new key.
123 | self.table[key] = node
124 |
125 | # We need to move the node to the head of the list. The node is the
126 | # tail node, so it directly preceeds the head node due to the list
127 | # being circular. Therefore, the ordering is already correct, we just
128 | # need to adjust the 'head' variable.
129 | self.head = node
130 |
131 | def update(self, items):
132 |
133 | # Add multiple items to the cache.
134 | for n, v in items.items():
135 | self[n] = v
136 |
137 | def __delitem__(self, key):
138 |
139 | # Lookup the node, then remove it from the hash table.
140 | node = self.table[key]
141 | del self.table[key]
142 |
143 | node.empty = True
144 |
145 | # Not strictly necessary.
146 | node.key = None
147 | node.value = None
148 |
149 | # Because this node is now empty we want to reuse it before any
150 | # non-empty node. To do that we want to move it to the tail of the
151 | # list. We move it so that it directly preceeds the 'head' node. This
152 | # makes it the tail node. The 'head' is then adjusted. This
153 | # adjustment ensures correctness even for the case where the 'node'
154 | # is the 'head' node.
155 | self.mtf(node)
156 | self.head = node.next
157 |
158 | def __iter__(self):
159 |
160 | # Return an iterator that returns the keys in the cache in order from
161 | # the most recently to least recently used. Does not modify the cache
162 | # order.
163 | for node in self.dli():
164 | yield node.key
165 |
166 | def items(self):
167 |
168 | # Return an iterator that returns the (key, value) pairs in the cache
169 | # in order from the most recently to least recently used. Does not
170 | # modify the cache order.
171 | for node in self.dli():
172 | yield (node.key, node.value)
173 |
174 | def keys(self):
175 |
176 | # Return an iterator that returns the keys in the cache in order from
177 | # the most recently to least recently used. Does not modify the cache
178 | # order.
179 | for node in self.dli():
180 | yield node.key
181 |
182 | def values(self):
183 |
184 | # Return an iterator that returns the values in the cache in order
185 | # from the most recently to least recently used. Does not modify the
186 | # cache order.
187 | for node in self.dli():
188 | yield node.value
189 |
190 | def size(self, size=None):
191 |
192 | if size is not None:
193 | assert size > 0
194 | if size > self.listSize:
195 | self.addTailNode(size - self.listSize)
196 | elif size < self.listSize:
197 | self.removeTailNode(self.listSize - size)
198 |
199 | return self.listSize
200 |
201 | # Increases the size of the cache by inserting n empty nodes at the tail
202 | # of the list.
203 | def addTailNode(self, n):
204 | for i in range(n):
205 | node = _dlnode()
206 | node.next = self.head
207 | node.prev = self.head.prev
208 |
209 | self.head.prev.next = node
210 | self.head.prev = node
211 |
212 | self.listSize += n
213 |
214 | # Decreases the size of the list by removing n nodes from the tail of the
215 | # list.
216 | def removeTailNode(self, n):
217 | assert self.listSize > n
218 | for i in range(n):
219 | node = self.head.prev
220 | if not node.empty:
221 | if self.callback is not None:
222 | self.callback(node.key, node.value)
223 | del self.table[node.key]
224 |
225 | # Splice the tail node out of the list
226 | self.head.prev = node.prev
227 | node.prev.next = self.head
228 |
229 | # The next four lines are not strictly necessary.
230 | node.prev = None
231 | node.next = None
232 |
233 | node.key = None
234 | node.value = None
235 |
236 | self.listSize -= n
237 |
238 | # This method adjusts the ordering of the doubly linked list so that
239 | # 'node' directly precedes the 'head' node. Because of the order of
240 | # operations, if 'node' already directly precedes the 'head' node or if
241 | # 'node' is the 'head' node the order of the list will be unchanged.
242 | def mtf(self, node):
243 | node.prev.next = node.next
244 | node.next.prev = node.prev
245 |
246 | node.prev = self.head.prev
247 | node.next = self.head.prev.next
248 |
249 | node.next.prev = node
250 | node.prev.next = node
251 |
252 | # This method returns an iterator that iterates over the non-empty nodes
253 | # in the doubly linked list in order from the most recently to the least
254 | # recently used.
255 | def dli(self):
256 | node = self.head
257 | for i in range(len(self.table)):
258 | yield node
259 | node = node.next
260 |
--------------------------------------------------------------------------------
/vexbot/util/messaging.py:
--------------------------------------------------------------------------------
1 | from zmq.utils.interop import cast_int_addr
2 |
3 | def get_addresses(message: list) -> list:
4 | """
5 | parses a raw list from zmq to get back the components that are the address
6 | messages are broken by the addresses in the beginning and then the
7 | message, like this: ```addresses | '' | message ```
8 | """
9 | # Need the address so that we know who to send the message back to
10 | addresses = []
11 | for address in message:
12 | # if we hit a blank string, then we've got all the addresses
13 | if address == b'':
14 | break
15 | addresses.append(address)
16 |
17 | return addresses
18 |
--------------------------------------------------------------------------------
/vexbot/util/socket_factory.py:
--------------------------------------------------------------------------------
1 | import sys as _sys
2 | import zmq as _zmq
3 | import logging
4 | from os import path
5 |
6 | from zmq.auth.thread import ThreadAuthenticator
7 | from zmq.auth.ioloop import IOLoopAuthenticator
8 |
9 | from vexbot.util.get_certificate_filepath import (get_certificate_filepath,
10 | get_vexbot_certificate_filepath,
11 | get_client_certificate_filepath)
12 |
13 | class SocketFactory:
14 | """
15 | Abstracts out the socket creation, specifically the issues with
16 | transforming ports into zmq addresses.
17 | Also sets up some default protocol and address handeling.
18 | For example, if everything is being run locally on one machine, and the
19 | default is to bind all addresses `*` and the transport protocol
20 | `tcp` means tcp communication
21 | """
22 | def __init__(self,
23 | address: str,
24 | protocol: str='tcp',
25 | context: 'zmq.Context'=None,
26 | logger: 'logging.Logger'=None,
27 | loop=None,
28 | auth_whitelist: list=None,
29 | using_auth: bool=True):
30 |
31 | self.address = address
32 | self.protocol = protocol
33 | self.context = context or _zmq.Context.instance()
34 | if logger is None:
35 | logger = logging.getLogger(__name__)
36 | self.logger = logger
37 | self._server_certs = (None, None)
38 | self.using_auth = using_auth
39 |
40 | if self.using_auth:
41 | self._setup_auth(auth_whitelist, loop)
42 |
43 | def _setup_auth(self, auth_whitelist: list, loop):
44 | if loop is None:
45 | self.auth = ThreadAuthenticator(self.context)
46 | else:
47 | self.auth = IOLoopAuthenticator(self.context, io_loop=loop)
48 |
49 | # allow all local host
50 | self.logger.debug('Auth whitelist %s', auth_whitelist)
51 | if auth_whitelist is not None:
52 | self.auth.allow(auth_whitelist)
53 | self._base_filepath = get_certificate_filepath()
54 | public_key_location = path.join(self._base_filepath, 'public_keys')
55 | self.auth.configure_curve(domain='*', location=public_key_location)
56 | self.auth.start()
57 |
58 | def _set_server_certs(self, any_=True):
59 | secret_filepath, secret = get_vexbot_certificate_filepath()
60 | if not secret and not any_:
61 | err = ('Could not find the vexbot certificates. Generate them '
62 | 'using `vexbot_generate_certificates` from command line')
63 | raise FileNotFoundError(err)
64 |
65 | self._server_certs = _zmq.auth.load_certificate(secret_filepath)
66 |
67 | def _create_using_auth(self, socket, address, bind, on_error, socket_name):
68 | if not any(self._server_certs):
69 | self._set_server_certs(bind)
70 | if bind:
71 | if self._server_certs[1] is None:
72 | err = ('Server Secret File Not Found! Generate using '
73 | '`vexbot_generate_certificates` from the command line')
74 | raise FileNotFoundError(err)
75 |
76 | public, private = self._server_certs
77 | else:
78 | # NOTE: This raises a `FileNotFoundError` if not found
79 | secret_filepath = get_client_certificate_filepath()
80 | public, private = _zmq.auth.load_certificate(secret_filepath)
81 |
82 | socket.curve_secretkey = private
83 | socket.curve_publickey = public
84 |
85 | if bind:
86 | socket.curve_server = True
87 | try:
88 | socket.bind(address)
89 | except _zmq.error.ZMQError:
90 | self._handle_error(on_error, address, socket_name)
91 | socket = None
92 | else: # connect the socket
93 | socket.curve_serverkey = self._server_certs[0]
94 | socket.connect(address)
95 |
96 | def _create_no_auth(self, socket, address, bind, on_error, socket_name):
97 | if bind:
98 | try:
99 | socket.bind(address)
100 | except _zmq.error.ZMQError:
101 | self._handle_error(on_error, address, socket_name)
102 | socket = None
103 | else: # connect the socket
104 | socket.connect(address)
105 |
106 | def create_n_connect(self,
107 | socket_type,
108 | address: str,
109 | bind=False,
110 | on_error='log',
111 | socket_name=''):
112 | """
113 | Creates and connects or binds the sockets
114 | on_error:
115 | 'log': will log error
116 | 'exit': will exit the program
117 | socket_name:
118 | used for troubleshooting/logging
119 | """
120 | self.logger.debug('create and connect: %s %s %s',
121 | socket_type, socket_name, address)
122 |
123 | socket = self.context.socket(socket_type)
124 | if self.using_auth:
125 | self._create_using_auth(socket, address, bind, on_error, socket_name)
126 | else:
127 | self._create_no_auth(socket, address, bind, on_error, socket_name)
128 |
129 | return socket
130 |
131 | def multiple_create_n_connect(self,
132 | socket_type,
133 | addresses: tuple,
134 | bind=False,
135 | on_error='log',
136 | socket_name=''):
137 |
138 | # TODO: Refactor this to remove code duplication with create_n_connect
139 | # Or at least make better sense
140 | socket = self.context.socket(socket_type)
141 | for address in addresses:
142 | if self.auth:
143 | self._create_using_auth(socket, address, bind, on_error, socket_name)
144 | else:
145 | self._create_no_auth(socket, address, bind, on_error, socket_name)
146 | return socket
147 |
148 | def to_address(self, port: str):
149 | """
150 | transforms the ports into addresses.
151 | Will fall through if the port looks like an address
152 | """
153 | # check to see if user passed in a string
154 | # if they did, they want to use that instead
155 | if isinstance(port, str) and len(port) > 6:
156 | return port
157 |
158 | zmq_address = '{}://{}:{}'
159 | return zmq_address.format(self.protocol,
160 | self.address,
161 | port)
162 |
163 | def iterate_multiple_addresses(self, ports: (str, list, tuple)):
164 | """
165 | transforms an iterable, or the expectation of an iterable
166 | into zmq addresses
167 | """
168 | if isinstance(ports, (str, int)):
169 | # TODO: verify this works.
170 | ports = tuple(ports,)
171 |
172 | result = []
173 | for port in ports:
174 | result.append(self.to_address(port))
175 |
176 | return result
177 |
178 | def _handle_error(self, how_to: str, address: str, socket_name: str):
179 | if socket_name == '':
180 | socket_name = 'unknown type'
181 |
182 | if how_to == 'exit':
183 | self._handle_bind_error_by_exit(address, socket_name)
184 | else:
185 | self._handle_bind_error_by_log(address, socket_name)
186 |
187 | def _handle_bind_error_by_log(self, address: str, socket_name: str):
188 | if self.logger is None:
189 | return
190 | s = 'Address bind attempt fail. Address tried: {}'
191 | s = s.format(address)
192 | self.logger.error(s)
193 | self.logger.error('socket type: {}'.format(socket_name))
194 |
195 | def _handle_bind_error_by_exit(self, address: str, socket_type: str):
196 | if self.logger is not None:
197 | s = 'Address bind attempt fail. Alredy in use? Address tried: {}'
198 | s = s.format(address)
199 | self.logger.error(s)
200 | self.logger.error('socket type: {}'.format(socket_type))
201 |
202 | _sys.exit(1)
203 |
--------------------------------------------------------------------------------