├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .pylintrc ├── .travis.yml ├── AUTHORS.md ├── BuildDockerImage.sh ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile.CLAI ├── Dockerfile.CLAIServer ├── Dockerfile.Dev ├── Jenkinsfile ├── LICENSE.txt ├── Makefile ├── Makefile.gnu ├── Makefile.uss ├── README.md ├── RunDockerImage.sh ├── StopDockerImage.sh ├── anonymize.json ├── bin ├── clai-run ├── emulator.py ├── fswatchlog ├── obtain-command-id ├── post-execution ├── process-command └── restore-history ├── clai ├── README.md ├── __init__.py ├── __version__.py ├── clai_run.py ├── datasource │ ├── __init__.py │ ├── action_remote_storage.py │ ├── config_storage.py │ ├── model │ │ ├── __init__.py │ │ ├── pending_actions.py │ │ ├── plugin_config.py │ │ └── stat_event.py │ ├── server_pending_actions_datasource.py │ ├── server_status_datasource.py │ └── stats_tracker.py ├── emulator │ ├── __init__.py │ ├── clai_emulator.py │ ├── docker │ │ └── centos │ │ │ └── Dockerfile │ ├── docker_message.py │ ├── emufileExist.sh │ ├── emulator_docker_bridge.py │ ├── emulator_docker_log_connector.py │ ├── emulator_presenter.py │ ├── expand_less.gif │ ├── expand_more.gif │ ├── log.png │ ├── log_window.py │ ├── refresh.png │ ├── run.gif │ ├── stop.gif │ └── toggled_frame.py ├── fswatchlog_command.py ├── obtain_command_id.py ├── post_execution_command.py ├── process_command.py ├── remote_storage │ └── model_api.py ├── restore_history.py ├── server │ ├── README.md │ ├── __init__.py │ ├── agent.py │ ├── agent_datasource.py │ ├── agent_datasource_executor.py │ ├── agent_executor.py │ ├── agent_runner.py │ ├── clai_client.py │ ├── clai_message_builder.py │ ├── clai_server.py │ ├── client_connector.py │ ├── command_message.py │ ├── command_runner │ │ ├── __init__.py │ │ ├── agent_command_runner.py │ │ ├── agent_descriptor.py │ │ ├── clai_delegate_to_agent_command_runner.py │ │ ├── clai_help_command_runner.py │ │ ├── clai_install_command_runner.py │ │ ├── clai_last_info_command_runner.py │ │ ├── clai_orchestrate_command_runner.py │ │ ├── clai_plugins_command_runner.py │ │ ├── clai_power_command_runner.py │ │ ├── clai_power_disable_command_runner.py │ │ ├── clai_reload_command_runner.py │ │ ├── clai_select_command_runner.py │ │ ├── clai_unselect_command_runner.py │ │ ├── clean_input.py │ │ ├── command_runner.py │ │ └── command_runner_factory.py │ ├── logger.py │ ├── message_handler.py │ ├── orchestration │ │ ├── __init__.py │ │ ├── orchestrator.py │ │ ├── orchestrator_descriptor.py │ │ ├── orchestrator_provider.py │ │ ├── orchestrator_storage.py │ │ └── patterns │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── max_orchestrator │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config.json │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── max_orchestrator.py │ │ │ └── requirements.txt │ │ │ ├── preference_orchestrator │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config.json │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── preference_orchestrator.py │ │ │ └── requirements.txt │ │ │ ├── rltk_bandit_orchestrator │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── bandit_config.json │ │ │ ├── config.yml │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── requirements.txt │ │ │ ├── rltk_bandit_orchestrator.py │ │ │ └── warm_start_datagen.py │ │ │ └── threshold_orchestrator │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── manifest.properties │ │ │ └── threshold_orchestrator.py │ ├── plugins │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dataxplore │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── dataxplore.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ └── requirements.txt │ │ ├── fix_bot │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── fix_bot.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ └── requirements.txt │ │ ├── gitbot │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── gitbot.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── rasa │ │ │ │ ├── __init__.py │ │ │ │ ├── config.yml │ │ │ │ ├── data │ │ │ │ │ └── nlu.md │ │ │ │ └── models │ │ │ │ │ └── rasa-saved-nlu-model.tar.gz │ │ │ ├── requirements.txt │ │ │ ├── run_rasa.sh │ │ │ ├── sample_config.json │ │ │ └── service.py │ │ ├── gpt3 │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── gpt3.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── prompt.json │ │ │ └── requirements.txt │ │ ├── helpme │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config.ini │ │ │ ├── data.py │ │ │ ├── helpme.py │ │ │ ├── install.sh │ │ │ ├── install_darwin.sh.bkp │ │ │ ├── manifest.properties │ │ │ └── requirements.txt │ │ ├── howdoi │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config.ini │ │ │ ├── data.py │ │ │ ├── howdoi.py │ │ │ ├── install.sh │ │ │ ├── install_darwin.sh.bkp │ │ │ ├── manifest.properties │ │ │ ├── question_detection.py │ │ │ └── requirements.txt │ │ ├── ibmcloud │ │ │ ├── Dockerfile │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── app.yaml │ │ │ ├── helper.py │ │ │ ├── ibmcloud.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── planners.py │ │ │ ├── planning-files │ │ │ │ ├── domain.pddl │ │ │ │ ├── problem.pddl │ │ │ │ └── template.pddl │ │ │ ├── requirements.txt │ │ │ ├── sample-application-files │ │ │ │ ├── app.yaml │ │ │ │ ├── requirements.txt │ │ │ │ └── run.py │ │ │ └── temp.dat │ │ ├── linuss │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── equivalencies.json │ │ │ ├── install.sh │ │ │ ├── linuss.gif │ │ │ ├── linuss.py │ │ │ ├── manifest.properties │ │ │ └── requirements.txt │ │ ├── manpage_agent │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config.json │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── manpage_agent.py │ │ │ ├── question_detection.py │ │ │ ├── requirements.txt │ │ │ ├── tldr_wrapper.py │ │ │ └── webapp │ │ │ │ ├── .gitignore │ │ │ │ ├── Procfile │ │ │ │ ├── app.py │ │ │ │ ├── data.py │ │ │ │ ├── manifest.yml │ │ │ │ ├── requirements.txt │ │ │ │ ├── runtime.txt │ │ │ │ ├── train_requirements.txt │ │ │ │ ├── train_sklearn_agent.py │ │ │ │ └── train_sklearn_agent.sh │ │ ├── nlc2cmd │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── nlc2cmd.py │ │ │ ├── remote │ │ │ │ ├── Procfile │ │ │ │ ├── README.md │ │ │ │ ├── config.json │ │ │ │ ├── manifest.yml │ │ │ │ ├── requirements.txt │ │ │ │ ├── run.py │ │ │ │ └── runtime.txt │ │ │ ├── requirements.txt │ │ │ ├── service.py │ │ │ └── wa_skills │ │ │ │ ├── README.md │ │ │ │ ├── __init__.py │ │ │ │ ├── cloudbot.py │ │ │ │ ├── config.json │ │ │ │ ├── grepbot.py │ │ │ │ ├── tarbot.py │ │ │ │ ├── utils.py │ │ │ │ └── zosbot.py │ │ ├── tellina │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── requirements.txt │ │ │ └── tellina.py │ │ ├── voice │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── install_darwin.sh │ │ │ ├── manifest.properties │ │ │ ├── priming.json │ │ │ ├── requirements.txt │ │ │ └── voice.py │ │ └── zmsgcode │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── config.ini │ │ │ ├── install.sh │ │ │ ├── manifest.properties │ │ │ ├── requirements.txt │ │ │ ├── zmsgcode.gif │ │ │ └── zmsgcode.py │ ├── searchlib │ │ ├── README.md │ │ ├── __init__.py │ │ ├── data.py │ │ ├── kc_provider.py │ │ ├── man_provider.py │ │ ├── providers.py │ │ └── se_provider.py │ ├── server_connector.py │ ├── socket_client_connector.py │ ├── socket_server_connector.py │ ├── state_mapper.py │ ├── utilities │ │ ├── __init__.py │ │ └── gpt3 │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ └── gpt3.py │ ├── web_socket_client_connector.py │ └── web_socket_server_connector.py └── tools │ ├── __init__.py │ ├── anonymizer.py │ ├── colorize_console.py │ ├── console_helper.py │ ├── docker_utils.py │ ├── file_util.py │ └── process_utils.py ├── configPlugins.json ├── develop.py ├── develop.sh ├── docker-compose.yaml ├── docs ├── FAQ.md ├── Installation.md ├── Overview.md ├── README.md └── Troubleshooting.md ├── emulator.sh ├── install.py ├── install.sh ├── known-issues.md ├── my_agent ├── __init__.py ├── manifest.properties └── my_agent.py ├── requirements.txt ├── requirements_dev.txt ├── requirements_emulator.txt ├── requirements_test.txt ├── requirements_utilities.txt ├── runLocalTests.sh ├── scripts ├── clai.sh ├── fileExist.sh ├── installOrchestrator.sh └── saveFilesChanges.sh ├── test ├── __init__.py ├── io_utils.py ├── mock_executor.py ├── sample_zcmds │ ├── mvs.txt │ ├── tso.txt │ └── uss.txt ├── state_mother.py ├── test_clai_plugins_helpme.py ├── test_clai_plugins_howdoi.py ├── test_clai_plugins_nlc2cmd_cloud.py ├── test_clai_plugins_zmsgcode.py ├── test_clai_server_install.py ├── test_message_handler.py ├── test_message_power.py ├── test_process_command.py └── test_restore_history.py ├── test_integration ├── __init__.py ├── conftest.py ├── contract_skills.py ├── docker │ └── centos │ │ ├── Dockerfile │ │ └── Dockerfile.no.install ├── test_install.py ├── test_skill_fixit.py ├── test_skill_helpme.py ├── test_skill_howdoi.py ├── test_skill_manpageexplorer.py ├── test_skill_nlc2cmd.py └── test_skill_tellina.py ├── uninstall.py ├── uninstall.sh ├── usersInstalled.json └── utils ├── .bash_profile ├── .bashrc ├── .profile ├── README.md ├── getMakeType.sh └── getVersionInfo.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | # encode all files as EBCDIC unless mentioned elsewhere 2 | * git-encoding=utf-8 zos-working-tree-encoding=ibm-1047 3 | 4 | # encode selected files 5 | *.gif git-encoding=utf-8 zos-working-tree-encoding=utf-8 6 | *.png git-encoding=utf-8 zos-working-tree-encoding=utf-8 7 | *.tar.gz git-encoding=utf-8 zos-working-tree-encoding=utf-8 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Log file** 27 | If you found an error or an unexpected behaviour. It's useful for us the server log. You can find it in ```/var/tmp/app.log``` please attach it to the issue or copy and paste last lines. 28 | 29 | **shell and OS (please complete the following information):** 30 | - OS: [e.g. Ubuntu] 31 | - Shell [e.g. bash] 32 | - Version [e.g. 3.6] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: A new feature for this projec 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Feature description** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Acceptance criteria** 14 | 1. Criteria for check the issue is complete. 15 | 2. Criteria for check the issue is complete. 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### :pushpin: References 2 | * **Issue:** fixes _your issue goes here_ 3 | * **Related pull-requests:** _list of related pull-requests (comma-separated): #1, #2_ 4 | 5 | ### :tophat: What is the goal? 6 | 7 | _Provide a description of the overall goal (you can usually copy the one from the issue)_ 8 | 9 | ### :memo: How is it being implemented? 10 | 11 | _Provide a description of the implementation_ 12 | 13 | 14 | ### :tv: Screenshot or gif showing the result. 15 | 16 | _Introuce here a gif or picture about the work_ 17 | 18 | ### :boom: How can it be tested? 19 | 20 | _If it cannot be tested explain why._ 21 | 22 | - [ ] **Use case 1:** _A brief description of the use case that should be tested_ 23 | - [ ] **Use case 2:** _If the use case requires some complex steps, increase indentation_ 24 | - [ ] _Step 1_ 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Temporary Files 2 | *~ 3 | *# 4 | *.*.swp 5 | .DS_Store 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | .hypothesis/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # dotenv 89 | .env 90 | 91 | # virtualenv 92 | .venv 93 | venv/ 94 | ENV/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | # pycharm 110 | .idea 111 | data 112 | 113 | nohup.out 114 | 115 | # hide wa config 116 | clai/server/plugins/nlc2cmd/remote/config.json 117 | clai/server/plugins/gpt3/remote/key 118 | 119 | # hide local gitbot stuff config 120 | clai/server/plugins/gitbot/config.json 121 | !clai/server/plugins/gitbot/rasa/data 122 | 123 | # hide kube bot config 124 | */kube_agent/config/* 125 | */kube_agent/bb-survey-server/* 126 | 127 | # Ignore Eclipse and PyDev project files 128 | /.project 129 | /.pydevproject 130 | 131 | # editor 132 | .vs 133 | .vscode 134 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [Master] 2 | ignore=plugins 3 | max-line-length=120 4 | 5 | [MESSAGES CONTROL] 6 | disable=missing-docstring 7 | 8 | [SIMILARITIES] 9 | 10 | ignore-imports=yes -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | services: 4 | - docker 5 | 6 | python: 7 | - "3.6" 8 | 9 | # command to install dependencies 10 | install: 11 | - pip install -r requirements.txt 12 | - pip install -r requirements_utilities.txt 13 | - pip install -r requirements_test.txt 14 | - pip install -r requirements_dev.txt 15 | - pip install -r requirements_emulator.txt 16 | 17 | # command to run tests 18 | script: 19 | - pytest ./test 20 | - pytest ./test_integration -s 21 | 22 | before_script: 23 | - pylint --ignore-patterns="venv/[\S+].py" *.py --rcfile .pylintrc clai test test_integration 24 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Contributors 2 | 3 | ## Core Team 4 | * [Mayank Agarwal](mailto:Mayank.Agarwal@ibm.com) 5 | * [Jorge 'Jorhell' Barroso](mailto:Jorge.Barroso.Carmona@ibm.com) aka flipper83 6 | * [Tathagata Chakraborti](mailto:Tathagata.Chakraborti1@ibm.com) 7 | * [Eli M. Dow](mailto:emdow@us.ibm.com) 8 | * [Kshitij Fadnis](mailto:kpfadnis@us.ibm.com) 9 | * [Borja Godoy](mailto:Borja.Godoy@ibm.com) 10 | * [Kartik Talamadupula](mailto:krtalamad@us.ibm.com) 11 | 12 | ## IBM Z Advanced Technology Team 13 | These are the nice folks who modified CLAI to run on z/OS: 14 | * [Tom Conti](mailto:tconti@us.ibm.com) 15 | * [Dan FitzGerald](mailto:danfitz@us.ibm.com) 16 | * [Daniel Gisolfi](mailto:Daniel.Gisolfi@ibm.com) 17 | * [Andrew C. M. Hicks](mailto:achicks@us.ibm.com) 18 | * [Lei A. B. Wang](mailto:wlwangwl@cn.ibm.com) 19 | 20 | ## Other Contributors 21 | * [Kristina Kolibab](mailto:kristina.kolibab@ibm.com) 22 | * [Steve Martinelli](mailto:stevemar@ca.ibm.com) 23 | * Eliezer Mishulovin 24 | * [Madhavan Pallan](mailto:madhpallan@gmail.com) 25 | * [Atharva Veer](mailto:adveer_b17@it.vjti.ac.in) 26 | -------------------------------------------------------------------------------- /BuildDockerImage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ################################################################ 4 | # Licensed Materials - Property of IBM 5 | # "Restricted Materials of IBM" 6 | # (C) Copyright IBM Corp. 2019 ALL RIGHTS RESERVED 7 | ################################################################ 8 | 9 | # Looks for an environment var named DOCKER_BUILD_FLAGS. If not 10 | # defined, uses the default flag value '--no-cache' for the docker build. 11 | # Ex.: To enable docker caching, export DOCKER_BUILD_FLAGS="" (empty value) 12 | # To then disable docker caching again, unset DOCKER_BUILD_FLAGS 13 | flags=${DOCKER_BUILD_FLAGS-"--no-cache"} 14 | 15 | # Looks for an environment var named CLAI_DOCKER_IMAGE_NAME. If not 16 | # defined, uses the default flag value 'claiplayground' for the docker image. 17 | image_name=${CLAI_DOCKER_IMAGE_NAME-"claiplayground"} 18 | 19 | # Looks for an environment var named CLAI_DOCKER_JENKINSBUILD. If it is 20 | # defined, adds it to the docker build command params. 21 | buildargs="" 22 | if [ -n "$CLAI_DOCKER_JENKINSBUILD" ]; then 23 | buildargs="--build-arg jenkinsbuild=true" 24 | fi 25 | 26 | echo "===============================================================" 27 | echo "" 28 | echo " Phase 1: Building CLAI Container $flags" 29 | echo "" 30 | echo "===============================================================" 31 | time docker build -f Dockerfile.CLAI -t $image_name $buildargs . $flags 32 | if [ $? -ne 0 ] 33 | then 34 | echo "Failed to build CLAI Playground Container. Aborting Build." 35 | exit -1 36 | fi 37 | 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Project CLAI 2 | 3 | Adding new features, improving documentation, fixing bugs, or writing tutorials are all examples of helpful contributions. 4 | Furthermore, if you are building a new skill, we strongly encourage you to add it to the 5 | [CLAI skill catalog](clai/server/plugins/) so that others may 6 | also benefit from it as well as contribute to it and improve it. 7 | 8 | Bug fixes can be initiated through GitHub pull requests or PRs. 9 | When making code contributions to Project CLAI, we ask that you follow the `PEP 8` coding standard 10 | and that you provide unit tests for the new features. 11 | 12 | This project uses [DCO](https://developercertificate.org/). 13 | Be sure to sign off your commits using the `-s` flag or adding `Signed-off-By: Name` in the commit message. 14 | 15 | ### Example commit message 16 | ```bash 17 | git commit -s -m 'Informative commit message' 18 | ``` 19 | 20 | ### Unit tests 21 | 22 | Whether you are contributing a new skill or updating the code elsewhere, you need to ensure that the unit tests pass. 23 | 24 | If you are developing a new skill, you can check in sample inputs and outputs in 25 | [here](./test_integration/) to make sure your skill runs as intended. 26 | See [here](./test_integration/test_skill_nlc2cmd.py) for an example test with the [`nlc2cmd`](clai/server/plugins/nlc2cmd) skill. 27 | 28 | Once all the tests have passed, and you are satisfied with your contribution, open a pull request into the `master` branch from **your fork of the repository** to request adding your contributions into the main code base. 29 | -------------------------------------------------------------------------------- /Dockerfile.CLAIServer: -------------------------------------------------------------------------------- 1 | FROM bashell/alpine-bash:latest 2 | ADD . . 3 | 4 | RUN touch ~/.bashrc \ 5 | && apk add --no-cache --update \ 6 | python3-dev \ 7 | gcc \ 8 | build-base \ 9 | libffi-dev \ 10 | openssl-dev \ 11 | && apk add linux-headers 12 | 13 | RUN pip3 install --upgrade pip \ 14 | && pip3 install -r requirements.txt \ 15 | && python3 ./install.py --unassisted --shell bash 16 | 17 | ENV CLAI_PATH /opt/local/share/clai/bin 18 | ENV PYTHONPATH /opt/local/share/clai/bin 19 | 20 | EXPOSE 22:22 21 | 22 | CMD eval $CLAI_PATH/bin/clai-run new --host 0.0.0.0 --port 22 --websocket 23 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 International Business Machines 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 16 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 17 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 18 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # Generic Makefile 9 | # 10 | # Author: Dan FitzGerald 11 | 12 | repo=`echo "clai/__version__.py __title__" | utils/getVersionInfo.sh` 13 | version=`echo "clai/__version__.py __version__" | utils/getVersionInfo.sh` 14 | make_cmd=`ps -o comm | grep make` 15 | makefile=`ps -o comm | grep make | utils/getMakeType.sh` 16 | 17 | intro: 18 | @echo "$(repo) v$(version)" 19 | 20 | init-test: 21 | @$(make_cmd) -f $(makefile) init-test 22 | 23 | clean: intro 24 | @$(make_cmd) -f $(makefile) clean 25 | 26 | test: intro 27 | @$(make_cmd) -f $(makefile) test 28 | 29 | dev: intro 30 | @$(make_cmd) -f $(makefile) dev 31 | 32 | install: intro 33 | @$(make_cmd) -f $(makefile) install 34 | 35 | uninstall: intro 36 | @$(make_cmd) -f $(makefile) uninstall 37 | 38 | MAKE: 39 | intro 40 | install 41 | 42 | .PHONY: intro init-test clean test dev install uninstall -------------------------------------------------------------------------------- /Makefile.gnu: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # Makefile for GNU-style make 9 | # 10 | # Author: Daniel Nicolas Gisolfi 11 | 12 | about: 13 | @echo "Processing a GNU-style Makefile" 14 | 15 | init-test: 16 | @python3 -m pip install -r requirements_test.txt 17 | 18 | clean: about 19 | -rm -f *.pyc *.pyo *.pyd *\$$py.class 20 | # These two files will be updated by CLAI, we dont want to commit the testing data 21 | -git checkout anonymize.json 22 | -git checkout configPlugins.json 23 | 24 | test: about 25 | ifeq (, $(shell type docker-compose 2> /dev/null)) 26 | $(warning docker-compose not in $(PATH), running tests locally) 27 | $(MAKE) init-test 28 | @python3 -m pytest $(PWD)/test 29 | else 30 | @echo "running tests in a docker container" 31 | @docker-compose run clai bash -c "cd /clai && make init-test && python3 -m pytest ./test" 32 | endif 33 | 34 | dev: about 35 | ifeq (, $(shell type docker-compose 2> /dev/null)) 36 | $(warning docker-compose not in $(PATH), running development script locally) 37 | @python3 develop.py install --path $(PWD) 38 | else 39 | @echo "running development script in a docker container" 40 | @docker-compose run clai bash -c "cd /clai && python3 develop.py install --path /clai && bash" 41 | endif 42 | 43 | install: about 44 | ifeq ($(shell whoami), root) 45 | @echo "Installing CLAI as root" 46 | ./install.sh 47 | else 48 | @echo "You are not running as the superuser, will preform an install local to your user" 49 | ./install.sh --user 50 | endif 51 | 52 | uninstall: about 53 | ifeq ($(shell whoami), root) 54 | @echo "Uninstalling CLAI as root" 55 | ./uninstall.sh 56 | else 57 | @echo "You are not running as the superuser, will preform an uninstall local to your user" 58 | ./uninstall.sh --user 59 | endif 60 | 61 | MAKE: 62 | intro 63 | install 64 | 65 | .PHONY: intro init-test clean test dev install uninstall -------------------------------------------------------------------------------- /Makefile.uss: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # Makefile for the default z/OS USS make 9 | # 10 | # Author: Daniel Nicolas Gisolfi 11 | 12 | uid=`id -u` 13 | no-docker-compose = `type docker-compose 2> /dev/null` 14 | 15 | about: 16 | @echo "Processing a z/OS USS make-style Makefile" 17 | 18 | init-test: 19 | @python3 -m pip install -r requirements_test.txt 20 | 21 | clean: about 22 | -rm -f *.pyc *.pyo *.pyd *\$$py.class 23 | # These two files will be updated by CLAI, we dont want to commit the testing data 24 | -git checkout anonymize.json 25 | -git checkout configPlugins.json 26 | 27 | test: about 28 | .IF no-docker-compose 29 | @echo "docker-compose not in $(PATH), running tests locally" 30 | $(MAKE) init-test 31 | @python3 -m pytest $(PWD)/test 32 | .ELSE 33 | @echo "running tests in a docker container" 34 | @docker-compose run clai bash -c "cd /clai && make init-test && python3 -m pytest ./test" 35 | .END 36 | 37 | dev: about 38 | .IF no-docker-compose 39 | @echo "docker-compose not in $(PATH), running development script locally" 40 | @python3 develop.py install --path $(PWD) 41 | .ELSE 42 | @echo "running development script in a docker container" 43 | @docker-compose run clai bash -c "cd /clai && python3 develop.py install --path /clai && bash" 44 | .END 45 | 46 | install: about 47 | .IF uid == 0 48 | @echo "Installing CLAI with superuser privileges" 49 | ./install.sh 50 | .ELSE 51 | @echo "You are not running as the superuser, will preform an install local to your user" 52 | ./install.sh --user 53 | .END 54 | 55 | uninstall: about 56 | .IF uid == 0 57 | @echo "Unstalling CLAI with superuser privileges" 58 | ./uninstall.sh 59 | .ELSE 60 | @echo "You are not running as the superuser, will preform an uninstalling local to your user" 61 | ./uninstall.sh --user 62 | .END 63 | 64 | MAKE: 65 | intro 66 | install 67 | 68 | .PHONY: intro init-test clean test dev install uninstall -------------------------------------------------------------------------------- /StopDockerImage.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | 4 | # This command will stop all instances of the CLAI playground docker container running on the local compute node... 5 | # The command bases this on the docker image type that containers are derived from. 6 | 7 | # Determine if there are any instances to stop... 8 | dockerInstances=$(docker ps -a -q --filter ancestor=CLAIPlaygroundContainer --format="{{.ID}}") 9 | 10 | # If there was anything to stop... 11 | if [[ -n "${dockerInstances/[ ]*\n/}" ]] 12 | then 13 | echo "Stopping CLAI Playground Docker Instances: $dockerInstances" 14 | # We stop those instances... 15 | docker rm $(docker stop $dockerInstances) 16 | else 17 | echo "No CLAI Playgrounds were found running. Nothing stopped." 18 | fi 19 | -------------------------------------------------------------------------------- /anonymize.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /bin/clai-run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import argparse 4 | 5 | from clai.clai_run import launcher_server, NEW_DIRECTIVE, START_DIRECTIVE 6 | 7 | DEFAULT_PORT = str(os.getenv('CLAI_PORT', 8010)) 8 | 9 | 10 | def parse_params(): 11 | parser = argparse.ArgumentParser(description='Start server clai.') 12 | subparsers = parser.add_subparsers(help='new, start', dest='directive') 13 | 14 | parser_new = subparsers.add_parser(NEW_DIRECTIVE, help='run the server on startup') 15 | parser_new.add_argument( 16 | '--host', 17 | help="host to up client and server", 18 | default="localhost" 19 | ) 20 | parser_new.add_argument( 21 | '--port', 22 | help="port listen server", 23 | type=int, 24 | default=DEFAULT_PORT 25 | ) 26 | 27 | parser_new.add_argument( 28 | '--websocket', 29 | help="websocket version for the web server", 30 | action='store_true' 31 | ) 32 | 33 | parser_start = subparsers.add_parser(START_DIRECTIVE, help='start the server') 34 | parser_start.add_argument( 35 | '--host', 36 | help="host to up client and server", 37 | default="localhost" 38 | ) 39 | parser_start.add_argument( 40 | '--port', 41 | help="port listen server", 42 | type=int, 43 | default=DEFAULT_PORT 44 | ) 45 | 46 | parser_start.add_argument( 47 | '--websocket', 48 | help="websocket version for the web server", 49 | action='store_true' 50 | ) 51 | 52 | return parser.parse_args() 53 | 54 | 55 | if __name__ == '__main__': 56 | ARGS = parse_params() 57 | HOST, PORT, DIRECTIVE, WEBSOCKET = ARGS.host, int(ARGS.port), ARGS.directive, ARGS.websocket 58 | launcher_server(HOST, PORT, DIRECTIVE, WEBSOCKET) 59 | -------------------------------------------------------------------------------- /bin/emulator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | #!/usr/bin/env python3 9 | from clai.emulator.emulator_docker_bridge import EmulatorDockerBridge 10 | 11 | if __name__ == '__main__': 12 | emulator_docker_bridge = EmulatorDockerBridge() 13 | 14 | from clai.emulator.clai_emulator import ClaiEmulator 15 | emulator = ClaiEmulator(emulator_docker_bridge) 16 | emulator.launch() 17 | -------------------------------------------------------------------------------- /bin/fswatchlog: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from clai.fswatchlog_command import process_files 5 | 6 | if __name__ == '__main__': 7 | COMMAND_ID = sys.argv[1] 8 | USER_NAME = sys.argv[2] 9 | FILE_CHANGES_ARGS = sys.argv[3:] 10 | 11 | process_files(COMMAND_ID, USER_NAME, FILE_CHANGES_ARGS) 12 | -------------------------------------------------------------------------------- /bin/obtain-command-id: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from clai.obtain_command_id import obtain_command_id 5 | 6 | if __name__ == '__main__': 7 | sys.stdout.write(obtain_command_id()) 8 | sys.stdout.flush() 9 | sys.exit(0) 10 | -------------------------------------------------------------------------------- /bin/post-execution: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from clai.post_execution_command import post_process_command 5 | 6 | MAX_STDERR = 1000 7 | if __name__ == '__main__': 8 | COMMAND_ID = sys.argv[1] 9 | USER_NAME = sys.argv[2] 10 | 11 | CMD_RESULT = 0 12 | if len(sys.argv) > 3: 13 | CMD_RESULT = sys.argv[3] 14 | 15 | STDERR = '' 16 | if len(sys.argv) > 4: 17 | STDERR_FILE_NAME = sys.argv[4] 18 | with open(STDERR_FILE_NAME, "r") as err_file: 19 | STDERR = err_file.read()[:MAX_STDERR] 20 | 21 | post_process_command(COMMAND_ID, USER_NAME, CMD_RESULT, STDERR) 22 | -------------------------------------------------------------------------------- /bin/process-command: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from clai.process_command import process_command 5 | 6 | if __name__ == '__main__': 7 | COMMAND_ID = sys.argv[1] 8 | USER_NAME = sys.argv[2] 9 | COMMAND_TO_CHECK = ' '.join(map(str, sys.argv[3:])) 10 | 11 | process_command(COMMAND_ID, USER_NAME, COMMAND_TO_CHECK) 12 | -------------------------------------------------------------------------------- /bin/restore-history: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from clai.post_execution_command import post_process_command 5 | from clai.restore_history import restore_history 6 | 7 | if __name__ == '__main__': 8 | if len(sys.argv) > 1: 9 | ORIGINAL_COMMAND = sys.argv[1] 10 | restore_history(ORIGINAL_COMMAND) 11 | -------------------------------------------------------------------------------- /clai/README.md: -------------------------------------------------------------------------------- 1 | # CLAI server 2 | 3 | We use socket communication with the CLAI server to listen to requests from the client (Bash terminal) 4 | and respond with commands from the skills. 5 | 6 | ## Usage 7 | 8 | ##### Open socket server 9 | ```bash 10 | >> python3 ./src/clai.py start 11 | ``` 12 | ###### directives 13 | |directive|Description | 14 | |--------|-------------| 15 | | start| Start the socket server if it isn't up | 16 | | stop | Stop the socket server | 17 | 18 | ###### options 19 | 20 | | param | default | description | 21 | |--------|-------------|-------------| 22 | | --host | localhost | host to open server and clients | 23 | | --port | 8010 | port to open server and clients | 24 | 25 | 26 | ##### Open Client 27 | ```bash 28 | >> python3 multiclient_con.py 29 | ``` 30 | 31 | -------------------------------------------------------------------------------- /clai/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from sys import platform as detect_platform 9 | PLATFORM = None 10 | PLATFORM = PLATFORM if PLATFORM is not None else detect_platform.lower() 11 | -------------------------------------------------------------------------------- /clai/__version__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | """General information about this version release. 9 | Includes title, description, URL, version, build, author/email, license, 10 | and copyright. 11 | """ 12 | 13 | __version__ = "1.0.0" 14 | __title__ = "CLAI" 15 | __description__ = "Command Line AI" 16 | __url__ = "https://clai-home.mybluemix.net/" 17 | __build__ = 0x00001 18 | __author__ = "See AUTHORS.md" 19 | __author_email__ = "See AUTHORS.md" 20 | __license__ = "MIT" 21 | __copyright__ = "Copyright 2020 IBM Corporation" 22 | __keywords__ = "CLAI, Command Line AI, Command Line Artificial Intelligence" 23 | __project_urls__ = { 24 | "Bugs": "https://github.com/IBM/clai/issues", 25 | "Documentation": __url__, 26 | "Source Code": "https://github.com/IBM/clai/issues", 27 | } 28 | -------------------------------------------------------------------------------- /clai/clai_run.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import errno 9 | import socket 10 | 11 | from clai.server.clai_server import ClaiServer 12 | from clai.server.web_socket_server_connector import WebSocketServerConnector 13 | 14 | START_DIRECTIVE = 'start' 15 | NEW_DIRECTIVE = 'new' 16 | 17 | 18 | def is_port_busy(host, port, reconnect): 19 | socket_to_check = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 20 | 21 | try: 22 | socket_to_check.bind((host, port)) 23 | except socket.error as exception: 24 | if exception.errno == errno.EADDRINUSE: 25 | if not reconnect: 26 | print("Port is already in use") 27 | return True 28 | 29 | print(f'something else raised in the socket: {exception}') 30 | finally: 31 | socket_to_check.close() 32 | 33 | return False 34 | 35 | 36 | def create_server_socket(host, port, websocket): 37 | if websocket: 38 | server = ClaiServer(connector=WebSocketServerConnector()) 39 | else: 40 | server = ClaiServer() 41 | 42 | server.init_server() 43 | server.create_socket(host, port) 44 | server.listen_client_sockets() 45 | print(f"server created in host: {host} and port: {port}") 46 | 47 | 48 | def launcher_server(host, port, directive, websocket): 49 | if directive == NEW_DIRECTIVE: 50 | if not is_port_busy(host, port, False): 51 | create_server_socket(host, port, websocket) 52 | else: 53 | print('The server is up yet') 54 | if directive == START_DIRECTIVE: 55 | print(f"starting CLAI") 56 | while is_port_busy(host, port, True): 57 | print('') 58 | create_server_socket(host, port, websocket) 59 | -------------------------------------------------------------------------------- /clai/datasource/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/datasource/model/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/datasource/model/pending_actions.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import List, Optional 9 | 10 | from clai.server.command_message import Action 11 | 12 | # pylint: disable=too-few-public-methods 13 | class PendingActions: 14 | def __init__(self, command_id: str, pending_actions: List[Action]): 15 | self.command_id = command_id 16 | self.current_action = 0 17 | self.pending_actions = pending_actions 18 | 19 | def next(self) -> Optional[Action]: 20 | if self.current_action >= len(self.pending_actions): 21 | return None 22 | 23 | action_to_return = self.pending_actions[self.current_action] 24 | self.current_action += 1 25 | 26 | action_to_return.pending_actions = self.current_action < len(self.pending_actions) 27 | 28 | return action_to_return 29 | -------------------------------------------------------------------------------- /clai/datasource/model/plugin_config.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import Dict, List, Union, Optional 9 | 10 | from pydantic import BaseModel 11 | 12 | 13 | # pylint: disable=too-few-public-methods,dangerous-default-value,too-many-arguments 14 | class PluginConfig: 15 | def __init__(self, 16 | selected: List[str] = [], 17 | orchestrator: Optional[str] = None, 18 | default: List[str] = [], 19 | installed: List[str] = [], 20 | report_enable: bool = False, 21 | default_orchestrator: str = "", 22 | user_install: bool = False): 23 | self.selected = selected 24 | self.default = default 25 | self.installed = installed 26 | self.report_enable = report_enable 27 | self.default_orchestrator = default_orchestrator 28 | self.orchestrator = orchestrator 29 | self.user_install = user_install 30 | 31 | 32 | class PluginConfigJson(BaseModel): 33 | selected: Dict[str, list] = {} 34 | default: Union[List[str], str] = ["demo_agent"] 35 | default_orchestrator: str = "max_orchestrator" 36 | installed: List[str] = [] 37 | report_enable: bool = False 38 | orchestrator: Optional[str] = None 39 | user_install: bool = False 40 | -------------------------------------------------------------------------------- /clai/datasource/model/stat_event.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import Mapping 9 | 10 | 11 | # pylint: disable=too-few-public-methods 12 | class StatEvent: 13 | def __init__(self, event_type: str, user: str, data: Mapping[str, str]): 14 | self.event_type = event_type 15 | self.data = data 16 | self.user = user 17 | -------------------------------------------------------------------------------- /clai/datasource/server_pending_actions_datasource.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import Dict, List 9 | 10 | from clai.datasource.model.pending_actions import PendingActions 11 | from clai.server.command_message import Action 12 | 13 | 14 | class ServerPendingActionsDatasource: 15 | def __init__(self): 16 | self.__pending_actions: Dict[str, List[PendingActions]] = {} 17 | 18 | def store_pending_actions(self, command_id: str, actions: List[Action], user_name: str) -> Action: 19 | actions_by_user = self.__find_pending_actions_by_user(user_name) 20 | actions_by_user.append(PendingActions(command_id, actions)) 21 | return self.get_next_action(command_id, user_name) 22 | 23 | def get_next_action(self, command_id: str, user_name: str) -> Action: 24 | actions_by_user = self.__find_pending_actions_by_user(user_name) 25 | pending_actions = next(filter(lambda x: x.command_id == command_id, actions_by_user), None) 26 | return pending_actions.next() 27 | 28 | def __find_pending_actions_by_user(self, user_name: str) -> List[PendingActions]: 29 | if user_name not in self.__pending_actions: 30 | self.__pending_actions[user_name] = [] 31 | return self.__pending_actions[user_name] 32 | -------------------------------------------------------------------------------- /clai/datasource/server_status_datasource.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from collections import deque 9 | from typing import Optional, Dict 10 | 11 | from clai.server.command_message import State 12 | 13 | 14 | class ServerStatusDatasource: 15 | def __init__(self): 16 | self.__power = False 17 | self.__messages_store: Dict[str, deque] = {} 18 | self.running = False 19 | 20 | def __store_info_by_user(self, message: State, user_name: str): 21 | messages_by_user = self.__find_messages_by_user(user_name) 22 | messages_by_user.append(message) 23 | 24 | def __find_messages_by_user(self, user_name: str): 25 | if user_name not in self.__messages_store: 26 | self.__messages_store[user_name] = deque(maxlen=50) 27 | return self.__messages_store[user_name] 28 | 29 | def set_power(self, power: bool): 30 | self.__power = power 31 | 32 | def is_power(self): 33 | return self.__power 34 | 35 | def get_last_messages(self, user_name: str): 36 | return self.__find_messages_by_user(user_name) 37 | 38 | def get_last_message(self, user_name: str, offset: int = 0) -> State: 39 | last_message = None 40 | messages_by_user = self.__find_messages_by_user(user_name) 41 | if messages_by_user and len(messages_by_user) > 1 + offset: 42 | last_message = messages_by_user[-2 - offset] 43 | return last_message 44 | 45 | def store_info(self, message: State) -> State: 46 | stored_message = self.find_message_stored(message.command_id, message.user_name) 47 | if stored_message is not None: 48 | stored_message.merge(message) 49 | return stored_message 50 | 51 | self.__store_info_by_user(message, message.user_name) 52 | return message 53 | 54 | def find_message_stored(self, id_to_find: str, user_mame: str) -> Optional[State]: 55 | messages_by_user = self.__find_messages_by_user(user_mame) 56 | return next(filter(lambda x: x.command_id == id_to_find, messages_by_user), None) 57 | 58 | 59 | # pylint: disable= invalid-name 60 | current_status_datasource = ServerStatusDatasource() 61 | -------------------------------------------------------------------------------- /clai/emulator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/emulator/docker_message.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | 4 | # pylint: disable=too-few-public-methods 5 | class DockerMessage: 6 | def __init__(self, docker_command: str, message: str = ''): 7 | self.docker_command = docker_command 8 | self.message = message 9 | 10 | # pylint: disable=too-few-public-methods 11 | class DockerReply: 12 | def __init__(self, docker_reply: str, message: str = '', info: Optional[str] = None): 13 | self.docker_reply = docker_reply 14 | self.message = message 15 | self.info = info 16 | -------------------------------------------------------------------------------- /clai/emulator/emufileExist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -f ../clai/server/plugins/$1/install.sh ]; then 4 | echo pwd 5 | pushd ../clai/server/plugins/$1 6 | eval "sh install.sh" 7 | popd 8 | echo "Installed plugins dependencies ../clai/server/plugins/$1/install.sh" 9 | else 10 | echo "The plugin don't have dependencies ../clai/server/plugins/$1/install.sh" 11 | fi 12 | -------------------------------------------------------------------------------- /clai/emulator/emulator_docker_log_connector.py: -------------------------------------------------------------------------------- 1 | import docker 2 | from pytest_docker_tools.wrappers import Container 3 | 4 | from clai.emulator.docker_message import DockerMessage, DockerReply 5 | from clai.tools.docker_utils import wait_server_is_started, read 6 | 7 | 8 | # pylint: disable=too-few-public-methods,protected-access 9 | class EmulatorDockerLogConnector: 10 | def __init__(self, pool, log_queue, queue_out): 11 | self.pool_log = pool 12 | self.consumer_log = None 13 | self.log_queue = log_queue 14 | self.queue_out = queue_out 15 | 16 | def start(self): 17 | self.consumer_log = self.pool_log.map_async(__log_consumer__, ((self.log_queue, self.queue_out),)) 18 | 19 | 20 | def __log_consumer__(args): 21 | queue, queue_out = args 22 | my_clai: Container = None 23 | socket = None 24 | print('starting reading the log queue') 25 | while True: 26 | docker_message: DockerMessage = queue.get() 27 | if docker_message.docker_command == 'start_logger': 28 | docker_client = docker.from_env() 29 | docker_container = docker_client.containers.get( 30 | docker_message.message) 31 | my_clai = Container(docker_container) 32 | 33 | if my_clai: 34 | if not socket: 35 | socket = my_clai.exec_run(cmd="bash -l", stdin=True, tty=True, 36 | privileged=True, socket=True) 37 | wait_server_is_started() 38 | 39 | socket.output._sock.send('clai "none" tail -f /var/tmp/app.log\n'.encode()) 40 | read(socket, lambda chunk: queue_out.put(DockerReply(docker_reply='log', message=chunk))) 41 | 42 | queue.task_done() 43 | queue.put(DockerMessage(docker_command='log')) 44 | -------------------------------------------------------------------------------- /clai/emulator/expand_less.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/emulator/expand_less.gif -------------------------------------------------------------------------------- /clai/emulator/expand_more.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/emulator/expand_more.gif -------------------------------------------------------------------------------- /clai/emulator/log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/emulator/log.png -------------------------------------------------------------------------------- /clai/emulator/log_window.py: -------------------------------------------------------------------------------- 1 | import tkinter as tk 2 | from tkinter import ttk 3 | 4 | 5 | class LogWindow: 6 | def __init__(self, root, presenter): 7 | self.root = root 8 | self.presenter = presenter 9 | self.autoscroll_enable = tk.IntVar() 10 | 11 | window = tk.Toplevel(self.root) 12 | window.geometry("900x600") 13 | self.add_toolbar(window) 14 | 15 | self.text_gui = tk.Text(window, height=6, width=40) 16 | self.text_gui.configure(state=tk.DISABLED) 17 | vsb = ttk.Scrollbar(window, orient="vertical", command=self.text_gui.yview) 18 | self.text_gui.configure(yscrollcommand=vsb.set) 19 | vsb.pack(side="right", fill="y") 20 | self.text_gui.pack(side="left", fill="both", expand=True) 21 | presenter.attach_log(self.add_log_data) 22 | 23 | 24 | def add_toolbar(self, window): 25 | toolbar = tk.Frame(window, bd=1, relief=tk.RAISED) 26 | self.autoscroll_button = tk.Checkbutton(toolbar, text="Auto Scroll", relief=tk.SOLID, 27 | var=self.autoscroll_enable) 28 | self.autoscroll_button.pack(pady=5) 29 | toolbar.pack(side=tk.TOP, fill=tk.X) 30 | 31 | def add_log_data(self, chunk): 32 | self.text_gui.configure(state=tk.NORMAL) 33 | self.text_gui.insert("end", chunk) 34 | if self.autoscroll_enable.get() == 1: 35 | self.text_gui.see("end") 36 | self.text_gui.configure(state=tk.DISABLED) 37 | -------------------------------------------------------------------------------- /clai/emulator/refresh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/emulator/refresh.png -------------------------------------------------------------------------------- /clai/emulator/run.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/emulator/run.gif -------------------------------------------------------------------------------- /clai/emulator/stop.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/emulator/stop.gif -------------------------------------------------------------------------------- /clai/emulator/toggled_frame.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import os 9 | 10 | import tkinter as tk 11 | from tkinter import ttk, LEFT 12 | 13 | 14 | # pylint: disable=too-many-ancestors,keyword-arg-before-vararg 15 | class ToggledFrame(tk.Frame): 16 | def __init__(self, parent, text="", *args, **options): 17 | tk.Frame.__init__(self, parent, *args, **options) 18 | 19 | self.show = tk.IntVar() 20 | self.show.set(0) 21 | 22 | self.title_frame = ttk.Frame(self) 23 | self.title_frame.pack( 24 | fill="x", 25 | expand=True) 26 | 27 | title_label = \ 28 | tk.Label(self.title_frame, 29 | text=text, 30 | anchor="nw", 31 | justify=LEFT, 32 | fg='white', 33 | padx=15, 34 | pady=15, 35 | bg="black") 36 | 37 | title_label.pack(side="left", 38 | fill="x", 39 | expand=True) 40 | 41 | path = os.path.dirname(os.path.abspath(__file__)) 42 | self.expand_more_image = tk.PhotoImage(file=f"{path}/expand_less.gif") 43 | self.expand_less_image = tk.PhotoImage(file=f"{path}/expand_more.gif") 44 | 45 | self.toggle_button = \ 46 | ttk.Checkbutton(self.title_frame, 47 | width=2, 48 | image=self.expand_more_image, 49 | command=self.toggle, 50 | variable=self.show, 51 | style='TLabel') 52 | self.toggle_button.pack(side="left") 53 | 54 | self.sub_frame = tk.Frame(self, relief="sunken", borderwidth=1) 55 | 56 | def toggle(self): 57 | if bool(self.show.get()): 58 | self.sub_frame.pack(fill="x", expand=1) 59 | self.toggle_button.configure(image=self.expand_less_image) 60 | else: 61 | self.sub_frame.forget() 62 | self.toggle_button.configure(image=self.expand_more_image) 63 | -------------------------------------------------------------------------------- /clai/obtain_command_id.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import uuid 9 | 10 | 11 | def obtain_command_id(): 12 | commaind_id = uuid.uuid4() 13 | return str(commaind_id) 14 | -------------------------------------------------------------------------------- /clai/post_execution_command.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import importlib 9 | from typing import List 10 | from clai import PLATFORM 11 | from clai.server.clai_client import send_command_post_execute 12 | from clai.server.command_message import Process, ProcessesValues 13 | 14 | try: 15 | PSUTIL = importlib.import_module('psutil') 16 | except ImportError: 17 | if PLATFORM not in ('zos', 'os390'): 18 | print('Error: psutil not installed') 19 | 20 | 21 | EXCLUDE_OWN_PROCESS = 1 22 | SIZE_PROCESS = 11 23 | 24 | 25 | def map_processes(processes) -> List[Process]: 26 | return list(map(lambda _: Process(name=_['name']), processes)) 27 | 28 | 29 | # pylint: disable=fixme 30 | def obtain_last_processes(user_name): 31 | process_changes = [] 32 | 33 | if PLATFORM not in ('zos', 'os390'): 34 | for process in PSUTIL.process_iter(attrs=['pid', 'name', 'username', 'create_time']): 35 | process_changes.append(process.info) 36 | else: 37 | # TODO: Figure out the equivilant on z/OS 38 | pass 39 | 40 | porcess_changes = list(filter(lambda _: _['username'] == user_name, process_changes)) 41 | porcess_changes.sort(key=lambda _: _['create_time'], reverse=True) 42 | return map_processes(process_changes[EXCLUDE_OWN_PROCESS:SIZE_PROCESS]) 43 | 44 | 45 | def post_process_command(command_id: str, user_name: str, cmd_result: str, stderr: str): 46 | process_changes = obtain_last_processes(user_name) 47 | post_command_action = send_command_post_execute( 48 | command_id=command_id, 49 | user_name=user_name, 50 | result_code=cmd_result, 51 | stderr=stderr, 52 | processes=ProcessesValues(last_processes=process_changes) 53 | ) 54 | 55 | if stderr: 56 | print(stderr) 57 | 58 | if post_command_action and post_command_action.description: 59 | print(post_command_action.description) 60 | -------------------------------------------------------------------------------- /clai/remote_storage/model_api.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import Optional, List 9 | 10 | from pydantic import BaseModel 11 | 12 | from clai.server.command_message import ProcessesValues, FilesChangesValues, NetworkValues, Action 13 | 14 | 15 | class StateApi(BaseModel): 16 | command_id: str = None 17 | user_name: str = None 18 | command: str = None 19 | root: bool = False 20 | processes: Optional[ProcessesValues] = None 21 | file_changes: Optional[FilesChangesValues] = None 22 | network: Optional[NetworkValues] = None 23 | result_code: str = None 24 | stderr: str = None 25 | 26 | 27 | class TerminalReplayMemoryApi(BaseModel): 28 | command: StateApi 29 | agent_names: List[str] 30 | candidate_actions: List[Action] 31 | force_response: str 32 | suggested_command: List[Action] 33 | 34 | 35 | class RecordToSendApi(BaseModel): 36 | bashbot_info: TerminalReplayMemoryApi 37 | -------------------------------------------------------------------------------- /clai/restore_history.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import io 9 | 10 | from clai.tools.file_util import read_history, get_history_file_name 11 | 12 | 13 | def restore_history(original_command: str): 14 | lines = read_history() 15 | 16 | new_lines = __remove_clai_history__(lines, original_command + '\n') 17 | io.open(get_history_file_name(), 'w').writelines(new_lines) 18 | 19 | 20 | def __remove_clai_history__(lines, original_command): 21 | if not lines: 22 | return lines 23 | 24 | if original_command not in lines: 25 | return lines 26 | 27 | lines_from_last = lines[::-1] 28 | position = lines_from_last.index(original_command) 29 | position = len(lines) - position - 1 30 | if position > 0 and lines[position - 1] == original_command: 31 | position = position - 1 32 | new_lines = lines[:position + 1] 33 | return new_lines 34 | -------------------------------------------------------------------------------- /clai/server/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/agent_datasource_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from abc import ABC, abstractmethod 9 | from concurrent import futures 10 | 11 | 12 | # pylint: disable=too-few-public-methods 13 | class AgentDatasourceExecutor(ABC): 14 | @abstractmethod 15 | def execute(self, load_agent, pkg_name): 16 | """execute all init agents""" 17 | 18 | 19 | class ThreadAgentDatasourceExecutor(AgentDatasourceExecutor): 20 | NUM_WORKERS = 4 21 | 22 | def __init__(self): 23 | self.executor = futures.ThreadPoolExecutor(max_workers=self.NUM_WORKERS) 24 | 25 | def execute(self, load_agent, pkg_name): 26 | self.executor.submit(load_agent, pkg_name) 27 | 28 | 29 | # pylint: disable= invalid-name 30 | thread_agent_datasource_executor = ThreadAgentDatasourceExecutor() 31 | -------------------------------------------------------------------------------- /clai/server/agent_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from abc import ABC, abstractmethod 9 | from concurrent import futures 10 | from functools import partial 11 | from operator import is_not 12 | from typing import List, Union 13 | 14 | from clai.server.agent import Agent 15 | from clai.server.command_message import Action, State 16 | 17 | 18 | # pylint: disable=too-few-public-methods 19 | class AgentExecutor(ABC): 20 | @abstractmethod 21 | def execute_agents(self, command: State, agents: List[Agent]) -> List[Action]: 22 | """execute all agents in parallel and return the actions""" 23 | 24 | 25 | class ThreadExecutor(AgentExecutor): 26 | MAX_TIME_PLUGIN_EXECUTION = 4 27 | NUM_WORKERS = 4 28 | 29 | def execute_agents(self, command: State, agents: List[Agent]) -> List[Union[Action, List[Action]]]: 30 | with futures.ThreadPoolExecutor(max_workers=self.NUM_WORKERS) as executor: 31 | done, _ = futures.wait( 32 | [executor.submit(plugin_instance.execute, command) for plugin_instance in agents], 33 | timeout=self.MAX_TIME_PLUGIN_EXECUTION) 34 | if not done: 35 | return [] 36 | results = map(lambda future: future.result(), done) 37 | candidate_actions = list(filter(partial(is_not, None), results)) 38 | return candidate_actions 39 | 40 | 41 | # pylint: disable= invalid-name 42 | thread_executor = ThreadExecutor() 43 | -------------------------------------------------------------------------------- /clai/server/client_connector.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import abc 9 | 10 | from clai.server.command_message import StateDTO, Action 11 | 12 | 13 | # pylint: disable=too-few-public-methods 14 | class ClientConnector(abc.ABC): 15 | @abc.abstractmethod 16 | def send(self, message: StateDTO) -> Action: 17 | '''Send a message to the server using a DTO''' 18 | -------------------------------------------------------------------------------- /clai/server/command_runner/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/command_runner/agent_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import Union, List 9 | 10 | from clai.datasource.server_status_datasource import ServerStatusDatasource 11 | from clai.server.agent_runner import AgentRunner 12 | from clai.server.command_message import State, Action 13 | from clai.server.command_runner.command_runner import CommandRunner, PostCommandRunner 14 | 15 | 16 | # pylint: disable=too-few-public-methods 17 | class AgentCommandRunner(CommandRunner, PostCommandRunner): 18 | 19 | def __init__(self, agent_runner: AgentRunner, server_status_datasource: ServerStatusDatasource, 20 | ignore_threshold: bool = False): 21 | self.agent_runner = agent_runner 22 | self.server_status_datasource = server_status_datasource 23 | self.ignore_threshold = ignore_threshold 24 | self.force_agent = None 25 | 26 | def execute(self, state: State) -> Union[Action, List[Action]]: 27 | action = self.agent_runner.process(state, self.ignore_threshold, self.force_agent) 28 | if not action: 29 | action = Action() 30 | return action 31 | 32 | def execute_post(self, state: State) -> Action: 33 | action = self.agent_runner.process_post(state, self.ignore_threshold) 34 | if not action: 35 | action = Action() 36 | action.origin_command = state.command 37 | return action 38 | -------------------------------------------------------------------------------- /clai/server/command_runner/agent_descriptor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # pylint: disable=too-few-public-methods,too-many-arguments,dangerous-default-value,too-many-instance-attributes 9 | class AgentDescriptor: 10 | def __init__(self, pkg_name, name, exclude=[], description="", installed=False, default=False, z_default=False): 11 | self.pkg_name = pkg_name 12 | self.name = name 13 | self.description = description 14 | self.default = default 15 | self.z_default = z_default 16 | self.installed = installed 17 | self.exclude = exclude 18 | self.ready = False 19 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_delegate_to_agent_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import List 9 | 10 | from clai.server.command_message import State, Action 11 | from clai.server.command_runner.agent_command_runner import AgentCommandRunner 12 | from clai.server.command_runner.clai_help_command_runner import ClaiHelpCommandRunner 13 | from clai.server.command_runner.command_runner import CommandRunner, PostCommandRunner 14 | 15 | # pylint: disable=too-few-public-methods 16 | CLAI_COMMAND_NAME = "clai" 17 | 18 | 19 | class ClaiDelegateToAgentCommandRunner(CommandRunner, PostCommandRunner): 20 | 21 | def __init__(self, agent: AgentCommandRunner): 22 | self.agent = agent 23 | 24 | def execute(self, state: State) -> Action: 25 | command_to_check = state.command.replace(CLAI_COMMAND_NAME, "", 1).strip() 26 | state.command = command_to_check 27 | 28 | if not command_to_check.strip(): 29 | return ClaiHelpCommandRunner().execute(state) 30 | 31 | if command_to_check.startswith('"'): 32 | possible_agents = command_to_check.split('"')[1::2] 33 | if possible_agents: 34 | agent_name = possible_agents[0] 35 | self.agent.force_agent = agent_name 36 | state.command = command_to_check.replace(f'"{agent_name}"', "", 1).strip() 37 | 38 | action = self.agent.execute(state) 39 | if not action: 40 | return ClaiHelpCommandRunner().execute(state) 41 | 42 | if isinstance(action, Action): 43 | if action.is_same_command() and not action.description: 44 | return ClaiHelpCommandRunner().execute(state) 45 | 46 | if isinstance(action, List): 47 | diffent_actions = list(filter(lambda value: not value.is_same_action(), action)) 48 | if not diffent_actions: 49 | return ClaiHelpCommandRunner().execute(state) 50 | 51 | return action 52 | 53 | def execute_post(self, state: State) -> Action: 54 | return self.agent.execute_post(state) 55 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_help_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.clai_message_builder import create_message_help 9 | from clai.server.command_message import State, Action 10 | from clai.server.command_runner.command_runner import CommandRunner 11 | 12 | 13 | # pylint: disable=too-few-public-methods 14 | class ClaiHelpCommandRunner(CommandRunner): 15 | 16 | def execute(self, state: State) -> Action: 17 | return create_message_help() 18 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_orchestrate_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.clai_message_builder import create_orchestrator_list 9 | from clai.server.command_message import State, Action 10 | from clai.server.command_runner.clean_input import extract_quoted_agent_name 11 | from clai.server.command_runner.command_runner import CommandRunner 12 | 13 | # pylint: disable=too-few-public-methods 14 | from clai.server.orchestration.orchestrator_provider import OrchestratorProvider 15 | 16 | 17 | class ClaiOrchestrateCommandRunner(CommandRunner): 18 | SELECT_DIRECTIVE = 'clai orchestrate' 19 | __VERBOSE_MODE = '-v' 20 | 21 | def __init__(self, orchestrator_provider: OrchestratorProvider): 22 | self.orchestrator_provider = orchestrator_provider 23 | 24 | def execute(self, state: State) -> Action: 25 | orchestrator_to_select = state.command.replace(f'{self.SELECT_DIRECTIVE}', '').strip() 26 | 27 | verbose = False 28 | if self.__VERBOSE_MODE in orchestrator_to_select: 29 | verbose = True 30 | orchestrator_to_select = "" 31 | else: 32 | orchestrator_to_select = extract_quoted_agent_name(orchestrator_to_select) 33 | 34 | if orchestrator_to_select: 35 | self.orchestrator_provider.select_orchestrator(orchestrator_to_select) 36 | 37 | return create_orchestrator_list( 38 | self.orchestrator_provider.get_current_orchestrator_name(), 39 | self.orchestrator_provider.all_orchestrator(), 40 | verbose 41 | ) 42 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_plugins_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.agent_datasource import AgentDatasource 9 | from clai.server.clai_message_builder import create_message_list 10 | from clai.server.command_message import Action, State 11 | from clai.server.command_runner.command_runner import CommandRunner 12 | 13 | 14 | # pylint: disable=too-few-public-methods 15 | class ClaiPluginsCommandRunner(CommandRunner): 16 | __VERBOSE_MODE = '-v' 17 | 18 | def __init__(self, agent_datasource: AgentDatasource): 19 | self.agent_datasource = agent_datasource 20 | 21 | def execute(self, state: State) -> Action: 22 | action_to_return = create_message_list( 23 | self.agent_datasource.get_current_plugin_name(state.user_name), 24 | self.agent_datasource.all_plugins(), 25 | self.__VERBOSE_MODE in state.command 26 | ) 27 | action_to_return.origin_command = state.command 28 | return action_to_return 29 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_power_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.datasource.server_status_datasource import ServerStatusDatasource 9 | from clai.server.command_message import Action, State, NOOP_COMMAND 10 | from clai.server.command_runner.command_runner import CommandRunner 11 | 12 | 13 | # pylint: disable=too-few-public-methods 14 | class ClaiPowerCommandRunner(CommandRunner): 15 | def __init__(self, server_status_datasource: ServerStatusDatasource): 16 | self.server_status_datasource = server_status_datasource 17 | 18 | def execute(self, state: State) -> Action: 19 | if self.server_status_datasource.is_power(): 20 | text = 'You have the auto mode already enable, use clai manual to deactivate it' 21 | else: 22 | self.server_status_datasource.set_power(True) 23 | text = 'You have enabled the auto mode' 24 | 25 | return Action( 26 | origin_command=state.command, 27 | suggested_command=NOOP_COMMAND, 28 | description=text, 29 | execute=True 30 | ) 31 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_power_disable_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.datasource.server_status_datasource import ServerStatusDatasource 9 | from clai.server.command_message import State, Action, NOOP_COMMAND 10 | from clai.server.command_runner.command_runner import CommandRunner 11 | 12 | 13 | # pylint: disable=too-few-public-methods 14 | class ClaiPowerDisableCommandRunner(CommandRunner): 15 | def __init__(self, server_status_datasource: ServerStatusDatasource): 16 | self.server_status_datasource = server_status_datasource 17 | 18 | def execute(self, state: State) -> Action: 19 | if not self.server_status_datasource.is_power(): 20 | text = 'You have manual mode already enable, use clai auto to activate it' 21 | else: 22 | self.server_status_datasource.set_power(False) 23 | text = 'You have enable the manual mode' 24 | 25 | return Action( 26 | suggested_command=NOOP_COMMAND, 27 | origin_command=state.command, 28 | description=text, 29 | execute=True 30 | ) 31 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_reload_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | from clai.server.agent_datasource import AgentDatasource 8 | from clai.server.command_message import State, Action 9 | from clai.server.command_runner.command_runner import CommandRunner 10 | 11 | # pylint: disable=too-few-public-methods 12 | from clai.tools.colorize_console import Colorize 13 | 14 | 15 | class ClaiReloadCommandRunner(CommandRunner): 16 | 17 | def __init__(self, agent_datasource: AgentDatasource): 18 | self.agent_datasource = agent_datasource 19 | 20 | def execute(self, state: State) -> Action: 21 | self.agent_datasource.reload() 22 | 23 | text = Colorize() \ 24 | .complete() \ 25 | .append("Plugins reloaded.\n") \ 26 | .to_console() 27 | 28 | return Action(suggested_command=":", 29 | execute=True, 30 | description=text, 31 | origin_command=state.command 32 | ) 33 | -------------------------------------------------------------------------------- /clai/server/command_runner/clai_unselect_command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.datasource.config_storage import ConfigStorage 9 | from clai.datasource.stats_tracker import StatsTracker 10 | from clai.server.agent_datasource import AgentDatasource 11 | from clai.server.clai_message_builder import create_message_list 12 | from clai.server.command_message import State, Action 13 | from clai.server.command_runner.clean_input import extract_quoted_agent_name 14 | from clai.server.command_runner.command_runner import CommandRunner 15 | 16 | 17 | # pylint: disable=too-few-public-methods 18 | class ClaiUnselectCommandRunner(CommandRunner): 19 | UNSELECT_DIRECTIVE = 'clai deactivate' 20 | 21 | def __init__(self, config_storage: ConfigStorage, agent_datasource: AgentDatasource): 22 | self.config_storage = config_storage 23 | self.agent_datasource = agent_datasource 24 | self.stats_tracker = StatsTracker() 25 | 26 | def execute(self, state: State) -> Action: 27 | plugin_to_select = state.command.replace(f'{self.UNSELECT_DIRECTIVE}', '').strip() 28 | 29 | plugin_to_select = extract_quoted_agent_name(plugin_to_select) 30 | 31 | selected = self.agent_datasource.unselect_plugin(plugin_to_select, state.user_name) 32 | if selected: 33 | self.stats_tracker.log_deactivate_skills(state.user_name, plugin_to_select) 34 | plugins_config = self.config_storage.read_config(state.user_name) 35 | if plugins_config.selected is not None and selected.name in plugins_config.selected: 36 | plugins_config.selected.remove(selected.name) 37 | self.config_storage.store_config(plugins_config, state.user_name) 38 | 39 | action_to_return = create_message_list( 40 | self.agent_datasource.get_current_plugin_name(state.user_name), 41 | self.agent_datasource.all_plugins() 42 | ) 43 | action_to_return.origin_command = state.command 44 | return action_to_return 45 | -------------------------------------------------------------------------------- /clai/server/command_runner/clean_input.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | def extract_quoted_agent_name(plugin_to_select): 9 | if plugin_to_select.startswith('"'): 10 | names = plugin_to_select.split('"')[1::2] 11 | if names: 12 | return names[0] 13 | 14 | return plugin_to_select 15 | -------------------------------------------------------------------------------- /clai/server/command_runner/command_runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from abc import ABC, abstractmethod 9 | from typing import Union, List 10 | 11 | from clai.server.command_message import State, Action 12 | 13 | 14 | # pylint: disable=too-few-public-methods 15 | class CommandRunner(ABC): 16 | @abstractmethod 17 | def execute(self, state: State) -> Union[Action, List[Action]]: 18 | """Execute the command and return a valid Action""" 19 | 20 | 21 | class PostCommandRunner(ABC): 22 | @abstractmethod 23 | def execute_post(self, state: State) -> Action: 24 | """post Execute the command and return a valid Action""" 25 | -------------------------------------------------------------------------------- /clai/server/logger.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import logging 9 | import logging.handlers as handlers 10 | import os 11 | 12 | class Logger: 13 | MAX_IN_MB = 100000000 14 | 15 | def __init__(self): 16 | 17 | log_formatter = logging.Formatter( 18 | '%(asctime)s %(levelname)s %(funcName)s(%(lineno)d) %(message)s') 19 | log_file = os.getenv('CLAI_LOG_FILE', '/var/tmp/app.log') 20 | self.logger = logging.getLogger('clai_logger') 21 | self.logger.setLevel(logging.INFO) 22 | self.log_handler = handlers.RotatingFileHandler( 23 | log_file, 24 | mode='a', 25 | maxBytes=Logger.MAX_IN_MB, 26 | backupCount=10, 27 | encoding=None, 28 | delay=0) 29 | self.log_handler.setLevel(logging.INFO) 30 | self.log_handler.setFormatter(log_formatter) 31 | self.logger.addHandler(self.log_handler) 32 | 33 | def info(self, text): 34 | self.logger.info(text) 35 | 36 | def warning(self, text): 37 | self.logger.warning(text) 38 | 39 | def debug(self, text): 40 | self.logger.debug(text) 41 | 42 | 43 | # pylint: disable= invalid-name 44 | current_logger = Logger() 45 | -------------------------------------------------------------------------------- /clai/server/orchestration/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/orchestrator_descriptor.py: -------------------------------------------------------------------------------- 1 | #pylint: disable=too-few-public-methods 2 | class OrchestratorDescriptor: 3 | def __init__(self, name: str, exclude: bool, description: str): 4 | self.name = name 5 | self.exclude = exclude 6 | self.description = description 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/orchestrator_storage.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from clai.datasource.action_remote_storage import ActionRemoteStorage 4 | from clai.server.command_message import TerminalReplayMemory, TerminalReplayMemoryComplete 5 | from clai.server.orchestration.orchestrator_provider import OrchestratorProvider 6 | 7 | 8 | class OrchestratorStorage: 9 | def __init__(self, orchestrator_provider: OrchestratorProvider, remote_storage: ActionRemoteStorage): 10 | self._memory: List[TerminalReplayMemoryComplete] = [] 11 | self.orchestrator_provider = orchestrator_provider 12 | self.remote_storage = remote_storage 13 | 14 | def store_pre(self, replay_pre: TerminalReplayMemory): 15 | terminal_replay_complete = TerminalReplayMemoryComplete() 16 | terminal_replay_complete.pre_replay = replay_pre 17 | self._memory.append(terminal_replay_complete) 18 | self.__notify_orchestrator(replay_pre) 19 | 20 | def store_post(self, replay_post: TerminalReplayMemory): 21 | if self._memory: 22 | self._memory[-1].post_replay = replay_post 23 | 24 | def __notify_orchestrator(self, current_pre: TerminalReplayMemory): 25 | if len(self._memory) > 1: 26 | previous_replay = self._memory.pop(0) 27 | orchestrator = self.orchestrator_provider.get_current_orchestrator() 28 | orchestrator.record_transition(previous_replay, current_pre) 29 | self.remote_storage.store(previous_replay.post_replay) 30 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/README.md: -------------------------------------------------------------------------------- 1 | ## Orchestration Patterns 2 | 3 | This directory contains some sample orchestration patterns. The [`max_orchestrator`](./max_orchestrator/) is used by default, 4 | as specified in [configPlugins.json](../../../../configPlugins.json). Use the [`Orchestration API`](../../) to make your own orchestration patterns. 5 | 6 | + **`max_orchestrator`** picks the skills that responds with the highest confidence, above a certain threshold. 7 | + **`preference_orchestrator`** picks the skill with highest confidence above a threshold that do not violate any of the partial preferences specified by the user. 8 | + **`threshold_orchestrator`** is similar to the max_orchestrator but it maintains thresholds for confidences specific to each skill and updates them according to how the end user reacts to them. 9 | + **`bandit_orchestrator`** learns user preferences using contextual bandits. 10 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/README.md: -------------------------------------------------------------------------------- 1 | ## Highest-Confidence Orchestration 2 | 3 | The `preference_orchestrator` picks the skill with highest confidence above a threshold. 4 | The threshold is specified in the [config.json](./config.json) file. -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "threshold": 0.4 3 | } 4 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools for max orchestrator" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | echo " >> Installing python dependencies" 10 | pip3 install -r requirements.txt -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=max orchestrator 3 | description=The max orchestrator picks the skills that responds with the highest confidence, above a certain threshold. -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/max_orchestrator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import Optional, List, Union 9 | from pathlib import Path 10 | import os 11 | import json 12 | import numpy as np 13 | 14 | from clai.server.orchestration.orchestrator import Orchestrator 15 | from clai.server.command_message import State, Action 16 | 17 | 18 | # pylint: disable=too-many-arguments,unused-argument 19 | class MaxOrchestrator(Orchestrator): 20 | 21 | def __init__(self): 22 | super(MaxOrchestrator, self).__init__() 23 | 24 | self._config_path = os.path.join( 25 | Path(__file__).parent.absolute(), 26 | 'config.json' 27 | ) 28 | self.__read_config__() 29 | 30 | def __read_config__(self): 31 | 32 | with open(self._config_path, 'r') as fileobj: 33 | config = json.load(fileobj) 34 | 35 | self.threshold = config['threshold'] 36 | 37 | def choose_action(self, command: State, agent_names: List[str], 38 | candidate_actions: Optional[List[Union[Action, List[Action]]]], 39 | force_response: bool, pre_post_state: str) -> Optional[Action]: 40 | """Choose an action for CLAI to respond with""" 41 | 42 | if not candidate_actions: 43 | return None 44 | 45 | confs = [self.__calculate_confidence__(action) for action in candidate_actions] 46 | idx_maxconf = np.argmax(confs) 47 | 48 | max_conf = confs[idx_maxconf] 49 | selected_candidate = candidate_actions[idx_maxconf] 50 | 51 | if force_response: 52 | return selected_candidate 53 | 54 | if max_conf >= self.threshold: 55 | return selected_candidate 56 | 57 | return None 58 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/max_orchestrator/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/preference_orchestrator/README.md: -------------------------------------------------------------------------------- 1 | ## Preference-based Orchestration 2 | 3 | The `preference_orchestrator` picks the skill with highest confidence above a threshold that do not violate any of the partial (ordering) preferences specified. It pools all the skills whose confidences are above the specified threshold and then goes down the list in decreasing order of confidence, selecting the first one that does not break any of the ordering constraints. 4 | 5 | + The [config.json](./config.json) file allows you to specify the threshold and a set of partial preferences 6 | + Each preference `[x,y]` is evaluated in order and reads as `x is preferred to y` -- the skill with the highest confidence that 7 | does not violate any of these orders is selected for execution. 8 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/preference_orchestrator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/preference_orchestrator/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "threshold" : 0.1, 3 | "preferences" : [["HowDoIAgent", "NLC2CMD"], 4 | ["ManPageAgent", "HowDoIAgent"]] 5 | } -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/preference_orchestrator/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools for preference orchestrator" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | echo " >> Installing python dependencies" 10 | pip3 install -r requirements.txt -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/preference_orchestrator/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=preference orchestrator 3 | description=The preference orchestrator picks the skill with highest confidence above a threshold that do not violate any specified preference. -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/preference_orchestrator/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/rltk_bandit_orchestrator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/rltk_bandit_orchestrator/bandit_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "noop_confidence": 0.1, 3 | "warm_start": true, 4 | "warm_start_config": { 5 | "type": "max-orchestrator", 6 | "kwargs": {} 7 | }, 8 | "reward_match_threshold": 0.7 9 | } -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/rltk_bandit_orchestrator/config.yml: -------------------------------------------------------------------------------- 1 | # Config file using the contextual/thompson pattern and providing its parameters 2 | # This configuration causes the bandit to do no logging of its activity 3 | 4 | pattern: contextual/thompson 5 | num_actions: 10 6 | context_size: 10 7 | 8 | # Number of actions is set to a maximum of 10. This means a maximum of 10 installed skills 9 | # (including a NOOP action) are supported. 10 | # Context size should be equal to the number of actions 11 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/rltk_bandit_orchestrator/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 10 | FRAMEWORK_DIR="${DIR}/framework" 11 | 12 | if [ -d "${FRAMEWORK_DIR}" ]; then 13 | rm -rf "${FRAMEWORK_DIR}" 14 | fi 15 | 16 | mkdir -p "${FRAMEWORK_DIR}" 17 | 18 | 19 | echo " >> Cloning framework libraries" 20 | echo "===============================================================" 21 | 22 | cd "${FRAMEWORK_DIR}" 23 | 24 | # Download and install RLTK library into the rltk folder and uncomment the 25 | # bottom two lines 26 | 27 | 28 | echo " >> Installing RLTK library" 29 | echo "===============================================================" 30 | 31 | # cd "${FRAMEWORK_DIR}/rltk" 32 | # python3 -m pip install -q --user . 33 | 34 | 35 | echo " >> Installing python dependencies" 36 | echo "===============================================================" 37 | 38 | python3 -m pip install -r requirements.txt -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/rltk_bandit_orchestrator/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=bandit orchestrator 3 | description=The bandit orchestrator learns user preferences using contextual bandits. 4 | exclude=true -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/rltk_bandit_orchestrator/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/threshold_orchestrator/README.md: -------------------------------------------------------------------------------- 1 | ## Adaptive-Threshold Orchestration 2 | 3 | This orchestration pattern is similar to the [max_orchestrator](../max_orchestrator/) but it maintains thresholds for confidences specific to each skill, and updates them according to how the end user reacts to them. -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/threshold_orchestrator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/orchestration/patterns/threshold_orchestrator/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=preference orchestrator 3 | description=The threshold orchestrator maintains thresholds for confidences specific to each skill and updates them according to how the end user reacts to them. -------------------------------------------------------------------------------- /clai/server/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/dataxplore/README.md: -------------------------------------------------------------------------------- 1 | # dataXplore 2 | 3 | `Analytics` `NLP` `Support` 4 | 5 | Data science has become one of the most popular real-world applications of ML. This skills is targeted specifically 6 | toward making the CLI easier to adopt and navigate for data scientists. 7 | 8 | ## Implementation 9 | 10 | The current version of the skill provides two functionalities: **summarize** and **plot**. 11 | "Summarize" utilizes the [describe function](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.describe.html) of the popular 12 | [Pandas library](https://pandas.pydata.org/pandas-docs/stable/index.html) to 13 | generate a human-readable summary of a specified CSV file; this functionality is intended to allow data scientists 14 | to quickly examine any data file right from the command line. "Plot" builds on the plot function provided by 15 | [MatPlotLib](https://ieeexplore.ieee.org/document/4160265), 16 | and the Pillow library [[link](https://pillow.readthedocs.io/en/stable/index.html)] 17 | [[link](https://www.pythonware.com/products/pil/)] 18 | to generate a plot of a given CSV file. Such functionalities illustrate basic use cases 19 | of how CLAI can be used as a CLI assistant for data science. 20 | 21 | ## Example Usage 22 | 23 | `>> clai "dataxplore" summarize air_quality.csv` to view the summary of the give data file. 24 | 25 | `>> clai "dataxplore" plot air_quality.csv` to view a plot of the given data file. 26 | 27 | ![figure1](https://www.dropbox.com/s/lin379uw2nc0ts9/dx_summarize_plot_test.png?raw=1) 28 | 29 | ![figure2](https://www.dropbox.com/s/j4xxme9eaj92mh5/dx_summarize_plot_airQuality.png?raw=1) 30 | 31 | Both dataset are courtesy of [pandas](http://pandas.pydata.org/). 32 | 33 | ## [xkcd](https://uni.xkcd.com/) 34 | The contents of any one panel are dependent on the contents of every panel including itself. The graph of panel dependencies is complete and bidirectional, and each node has a loop. The mouseover text has two hundred and forty-two characters. 35 | 36 | ![alt text](https://imgs.xkcd.com/comics/self_description.png "The contents of any one panel are dependent on the contents of every panel including itself. The graph of panel dependencies is complete and bidirectional, and each node has a loop. The mouseover text has two hundred and forty-two characters.") 37 | -------------------------------------------------------------------------------- /clai/server/plugins/dataxplore/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/dataxplore/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | pip3 install -r requirements.txt -------------------------------------------------------------------------------- /clai/server/plugins/dataxplore/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=dataxplore 3 | description=This skill summarizes csv file from your natural language command into a Bash command. 4 | default=no 5 | exclude=OS/390 Z/OS 6 | -------------------------------------------------------------------------------- /clai/server/plugins/dataxplore/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==1.0.3 2 | numpy==1.17.2 3 | matplotlib==3.2.1 4 | Pillow==7.1.1 5 | imageloader==0.0.5 6 | -------------------------------------------------------------------------------- /clai/server/plugins/fix_bot/README.md: -------------------------------------------------------------------------------- 1 | # fixit 2 | 3 | `Plugin Integration`   `Support`   4 | 5 | This skill fixes the last command as per the rules of [`thefuck`](https://github.com/nvbn/thefuck) plugin 6 | as an illustration of how to fold in exiting plugins into the CLAI framework. 7 | 8 | ## Implementation 9 | 10 | The skill responds whenever there's an error in response to a user command. When an error is detected, the command text and the error message is passed to the `thefuck` plugin to get a corrected command. This corrected command is then suggested to the user. 11 | 12 | ## Example Usage 13 | 14 | ![fixit](https://www.dropbox.com/s/r9q8rnjv38bipay/fixit.gif?raw=1) 15 | 16 | Same as [`thefuck`](https://github.com/nvbn/thefuck). 17 | 18 | 1. `>> puthon` 19 | 2. `>> git brnch` 20 | 3. `>> cd dir_does_not_exists` 21 | 22 | ## [xkcd](https://uni.xkcd.com/) 23 | It has a section on motherboard beep codes that lists, for each beep pattern, a song that syncs up well with it. 24 | 25 | ![alt text](https://imgs.xkcd.com/comics/error_code.png "It has a section on motherboard beep codes that lists, for each beep pattern, a song that syncs up well with it.") 26 | -------------------------------------------------------------------------------- /clai/server/plugins/fix_bot/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/fix_bot/fix_bot.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.agent import Agent 9 | from clai.server.command_message import State, Action 10 | from clai.tools.colorize_console import Colorize 11 | 12 | from thefuck.types import Command 13 | from thefuck.conf import settings 14 | from thefuck.corrector import get_corrected_commands 15 | 16 | 17 | class FixBot(Agent): 18 | """ 19 | Fixes the last executed command by running it through the `thefuck` plugin 20 | """ 21 | 22 | def __init__(self): 23 | super(FixBot, self).__init__() 24 | pass 25 | 26 | def get_next_action(self, state: State) -> Action: 27 | return Action(suggested_command=state.command) 28 | 29 | def post_execute(self, state: State) -> Action: 30 | 31 | if state.result_code == '0': 32 | return Action(suggested_command=state.command) 33 | 34 | cmd = str(state.command) 35 | stderr = str(state.stderr) 36 | 37 | try: 38 | # Get corrected command from `thefuck` bot 39 | settings.init() 40 | cmd = Command(cmd, stderr) 41 | cmd_corrected = get_corrected_commands(cmd) 42 | 43 | cmd_to_run = next(cmd_corrected).script 44 | except Exception: 45 | return Action(suggested_command=state.command, 46 | confidence=0.1) 47 | else: 48 | return Action( 49 | description=Colorize() 50 | .info() 51 | .append("Maybe you want to try: {}".format(cmd_to_run)) 52 | .to_console(), 53 | confidence=0.8 54 | ) 55 | -------------------------------------------------------------------------------- /clai/server/plugins/fix_bot/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | if [[ "$1" = "--user" ]]; then 12 | pip3 install --user -r requirements.txt 13 | else 14 | pip3 install -r requirements.txt 15 | fi 16 | -------------------------------------------------------------------------------- /clai/server/plugins/fix_bot/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=fixit 3 | description=Fixes the last command as per the rules of `thefuck` plugin 4 | default=no 5 | exclude=OS/390 Z/OS 6 | -------------------------------------------------------------------------------- /clai/server/plugins/fix_bot/requirements.txt: -------------------------------------------------------------------------------- 1 | thefuck -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/README.md: -------------------------------------------------------------------------------- 1 | # gitbot 2 | 3 | `NLP` `Support` `Automation` 4 | 5 | This skill lets you manage and organize your github repository in natural language. 6 | It also lets you use natural language commands to issue popular git commands. 7 | 8 | ## Implementation 9 | 10 | The skill demonstrates hooks into two interesting design patterns: 11 | 12 | + Similar to the [`nlc2cmd`](../nlc2cmd/) skill, it demonstrates natural language to 13 | command patterns. However, in contrast to the nlc2cmd implementation, here we demonstrate 14 | how to use a natural language classifier local to the machine -- using [RASA](https://rasa.com/) -- 15 | instead of calling an external service like [Watson Assistant](https://www.ibm.com/cloud/watson-assistant/). 16 | 17 | + This skill also demonstrates instances of workflow automation in the context of code 18 | development by using the [GitHub Actions API](https://github.com/features/actions). 19 | 20 | Similar to the `nlc2cmd` and `ibmcloud` skills, this skill is also merely illustrative 21 | of integration of natural language and automation in code management through GitHub. 22 | Contributions are welcome to improve the accuracy of the natural language interpretation, 23 | the breadth of the use cases covered for workflow automation, or new features! 24 | 25 | Before trying it out: 26 | 27 | > Fill out `sample_config.json` and rename it to `config.json` 28 | 29 | > run `./run_rasa.sh 5556` 30 | 31 | ## Example Usage 32 | 33 | ![gitbot](https://www.dropbox.com/s/7snw9sg3ab15rvr/gitbot.png?raw=1) 34 | 35 | ## [xkcd](https://uni.xkcd.com/) 36 | 37 | If that doesn't fix it, git.txt contains the phone number of a friend of mine who understands git. Just wait through a few minutes of 'It's really pretty simple, just think of branches as...' and eventually you'll learn the commands that will fix everything. 38 | 39 | ![alt text](https://imgs.xkcd.com/comics/git_2x.png "If that doesn't fix it, git.txt contains the phone number of a friend of mine who understands git. Just wait through a few minutes of 'It's really pretty simple, just think of branches as...' and eventually you'll learn the commands that will fix everything.") 40 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/gitbot.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | ''' imports ''' 9 | from clai.server.agent import Agent 10 | from clai.server.command_message import State, Action, NOOP_COMMAND 11 | 12 | from clai.server.plugins.gitbot.service import Service 13 | from clai.tools.colorize_console import Colorize 14 | 15 | import os 16 | 17 | ''' main gitbot class ''' 18 | class GITBOT(Agent): 19 | def __init__(self): 20 | super(GITBOT, self).__init__() 21 | self.service = Service() 22 | 23 | # KILL THE RASA SERVER 24 | # WARNING KILLS OTHERS TOO 25 | # os.system('lsof -t -i tcp:{} | xargs kill'.format(_rasa_port_number)) 26 | # BRING UP THE RASA SERVER 27 | # os.system('rasa run --enable-api -m {} -p {}'.format(_path_to_rasa_model, _rasa_port_number)) 28 | 29 | ''' pre execution processing ''' 30 | def get_next_action(self, state: State) -> Action: 31 | command = state.command 32 | return self.service(command) 33 | 34 | ''' pre execution processing ''' 35 | def post_execute(self, state: State) -> Action: 36 | # for a more sophisticated state change mechanism, see the ibmcloud skill 37 | if state.result_code == '0': self.service.parse_command(state.command, stdout = "") 38 | return Action(suggested_command=NOOP_COMMAND) 39 | 40 | def save_agent(self) -> bool: 41 | # KILL THE RASA SERVER 42 | os.system('lsof -t -i tcp:{} | xargs kill'.format(_rasa_port_number)) 43 | 44 | # return to original destruction method 45 | super().save_agent() 46 | 47 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | pip3 install -r requirements.txt 12 | 13 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=gitbot 3 | description=This skill helps you manage your github repositories. 4 | default=no 5 | exclude=OS/390 Z/OS -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/rasa/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/server/plugins/gitbot/rasa/__init__.py -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/rasa/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for Rasa NLU. 2 | # https://rasa.com/docs/rasa/nlu/components/ 3 | language: en 4 | pipeline: 5 | - name: WhitespaceTokenizer 6 | - name: RegexFeaturizer 7 | - name: LexicalSyntacticFeaturizer 8 | - name: CountVectorsFeaturizer 9 | - name: CountVectorsFeaturizer 10 | analyzer: "char_wb" 11 | min_ngram: 1 12 | max_ngram: 4 13 | - name: DIETClassifier 14 | epochs: 100 15 | - name: EntitySynonymMapper 16 | - name: ResponseSelector 17 | epochs: 100 18 | 19 | # Configuration for Rasa Core. 20 | # https://rasa.com/docs/rasa/core/policies/ 21 | policies: 22 | - name: MemoizationPolicy 23 | - name: TEDPolicy 24 | max_history: 5 25 | epochs: 100 26 | - name: MappingPolicy 27 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/rasa/data/nlu.md: -------------------------------------------------------------------------------- 1 | ## intent:commit 2 | - commit 3 | - commit changes 4 | 5 | ## intent:push 6 | - push 7 | - push changes 8 | - push changes into [vv](target) 9 | - push changes into [ww](target) 10 | - push changes from [aaa](source) into [xxx](target) 11 | - push changes from [bbb](source) into [yyy](target) 12 | - push changes from [ccc](source) into [zzz](target) 13 | - push changes from [master](source) into [develop](target) 14 | - push changes from [develop](source) into [master](target) 15 | 16 | ## intent:merge 17 | - merge 18 | - merge into [vv](target) 19 | - merge into [ww](target) 20 | - merge [aaa](source) into [xxx](target) 21 | - merge [bbb](source) into [yyy](target) 22 | - merge [ccc](source) into [zzz](target) 23 | - merge [master](source) into [develop](target) 24 | - merge [develop](source) into [master](target) 25 | 26 | ## intent:comment 27 | - add comment to issue [number][id] "[comment](comment)" 28 | - add comment to PR [number][id] "[comment](comment)" 29 | - add comment to issue [1][id] "[aaa](comment)" 30 | - add comment to PR [1][id] "[xxx](comment)" 31 | - add comment to issue [2][id] "[bbb](comment)" 32 | - add comment to PR [2][id] "[yyy](comment)" 33 | - add comment to issue [3][id] "[ccc](comment)" 34 | - add comment to PR [3][id] "[zzz](comment)" 35 | - add comment to issue [id][id] "[new comment](comment)" 36 | - add comment to PR [id][id] "[new comment](comment)" 37 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/rasa/models/rasa-saved-nlu-model.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/server/plugins/gitbot/rasa/models/rasa-saved-nlu-model.tar.gz -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | rasa -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/run_rasa.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Bringing up RASA server" 4 | 5 | lsof -t -i tcp:$1 | xargs kill 6 | rasa run --enable-api -m /Users/tathagata/clai/clai/server/plugins/gitbot/rasa/models/rasa-saved-nlu-model.tar.gz -p $1 7 | -------------------------------------------------------------------------------- /clai/server/plugins/gitbot/sample_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "github_personal_access_token" : "", 3 | "github_username" : "", 4 | "github_repo" : ["org-name or user-name", "repo-name"], 5 | "path_to_log_file" : "absolute/path/to/log/file/with/write/permissions", 6 | "path_to_rasa_saved_model" : "relative/path/to/rasa/models/rasa-saved-nlu-model.tar.gz", 7 | "rasa_port_number" : "port-number" 8 | } 9 | -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/.gitignore: -------------------------------------------------------------------------------- 1 | openai_api.key -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/README.md: -------------------------------------------------------------------------------- 1 | # gpt3 `OpenAI API` 2 | 3 | `NLP` `Support` 4 | 5 | This skill mirrors the [`tellina`](https://github.com/IBM/clai/tree/master/clai/server/plugins/tellina) skill, 6 | and the [NLC2CMD](http://ibm.biz/nlc2cmd) use case, but uses the recently released GPT-3 model instead. 7 | 8 | ## Implementation 9 | 10 | The skill makes a remote call to the [OpenAI API](https://openai.com/blog/openai-api/), apply 11 | for yours [here](https://forms.office.com/Pages/ResponsePage.aspx?id=VsqMpNrmTkioFJyEllK8s0v5E5gdyQhOuZCXNuMR8i1UQjFWVTVUVEpGNkg3U1FNRDVVRFg3U0w4Vi4u). 12 | 13 | The setup uses the GPT-3 utility in CLAI, please refer to the 14 | instructions [here](https://github.com/IBM/clai/tree/master/clai/server/utilities/gpt3) for more details. 15 | 16 | ### Scoring & Confidence 17 | 18 | Currently, the `gpt3` skill returns a default confidence of `0.0`. Use direct invocation to use `gpt3`. If you have ideas to score and explain the responses from the GPT-3 model, open a PR! 19 | 20 | 21 | ``` 22 | >> clai "gpt3" [command] 23 | ``` 24 | 25 | ## Example Usage 26 | 27 | ![clai-gpt3-snapshot](https://user-images.githubusercontent.com/4764242/88873573-eef7ef80-d1ea-11ea-94c5-06d65b06bf4e.png) 28 | 29 | 30 | ## [xkcd](https://uni.xkcd.com/) 31 | 32 | 33 | The pile gets soaked with data and starts to get mushy over time, so it's technically recurrent. 34 | 35 | ![alt text](https://imgs.xkcd.com/comics/machine_learning.png "The pile gets soaked with data and starts to get mushy over time, so it's technically recurrent.") 36 | -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/gpt3.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.agent import Agent 9 | from clai.server.command_message import State, Action, NOOP_COMMAND 10 | from clai.tools.colorize_console import Colorize 11 | 12 | from clai.server.utilities.gpt3.gpt3 import GPT, Example 13 | 14 | from pathlib import Path 15 | import json 16 | import os 17 | 18 | class GPT3(Agent): 19 | def __init__(self): 20 | super(GPT3, self).__init__() 21 | self._gpt3_api = self.__init_gpt3_api__() 22 | 23 | def __init_gpt3_api__(self): 24 | 25 | current_directory = str(Path(__file__).parent.absolute()) 26 | 27 | path_to_gpt3_key = os.path.join(current_directory, "openai_api.key") 28 | path_to_gpt3_prompts = os.path.join(current_directory, "prompt.json") 29 | 30 | gpt3_key = open(path_to_gpt3_key, 'r').read() 31 | gpt3_prompts = json.load(open(path_to_gpt3_prompts, 'r')) 32 | 33 | gpt3_api = GPT(temperature=0) 34 | gpt3_api.set_api_key(gpt3_key) 35 | 36 | for prompt in gpt3_prompts: 37 | ip, op = prompt['input'], prompt['output'] 38 | example = Example(ip, op) 39 | gpt3_api.add_example(example) 40 | 41 | return gpt3_api 42 | 43 | def get_next_action(self, state: State) -> Action: 44 | 45 | # user typed in, in natural language 46 | command = state.command 47 | 48 | # truncate to first 1000 chars 49 | command = command[:1000] 50 | 51 | try: 52 | 53 | response = self._gpt3_api.get_top_reply(command, strip_output_suffix=True) 54 | response = response.strip() 55 | 56 | return Action( 57 | suggested_command=response, 58 | execute=False, 59 | description="Currently the GPT-3 skill does not provide an explanation. Got an idea? Contribute to CLAI!", 60 | confidence=0.0) 61 | 62 | except Exception as ex: 63 | return [ { "text" : "Method failed with status " + str(ex) }, 0.0 ] 64 | 65 | -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | pip3 install -r requirements.txt -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=gpt3 3 | description=This skill translates your natural language command into a Bash command using the OpenAI API. 4 | default=yes -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/prompt.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": "List files", 4 | "output": "ls -l" 5 | }, 6 | 7 | { 8 | "input": "Count files in a directory", 9 | "output": "ls -l | wc -l" 10 | }, 11 | { 12 | "input": "Disk space used by home directory", 13 | "output": "du ~" 14 | }, 15 | { 16 | "input": "Replace foo with bar in all .py files", 17 | "output": "sed -i .bak -- 's/foo/bar/g' *.py" 18 | }, 19 | { 20 | "input": "Delete the models subdirectory", 21 | "output": "rm -rf ./models" 22 | } 23 | ] -------------------------------------------------------------------------------- /clai/server/plugins/gpt3/requirements.txt: -------------------------------------------------------------------------------- 1 | openai -------------------------------------------------------------------------------- /clai/server/plugins/helpme/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/helpme/data.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import os 9 | import json 10 | import configparser 11 | from pathlib import Path 12 | 13 | import requests 14 | 15 | 16 | class Datastore: 17 | def __init__(self): 18 | config = configparser.ConfigParser() 19 | config.read(os.path.join(str(Path(__file__).parent.absolute()), 'config.ini')) 20 | 21 | self.stack_exchange_api = config['API'].get('stack_exchange_api') 22 | self.manpage_api = config['API'].get('manpage_api') 23 | 24 | def __call_manpage_api__(self, query: str, limit: int = 1): 25 | 26 | payload = { 27 | 'text': query, 28 | 'result_count': limit 29 | } 30 | 31 | headers = {'Content-Type': "application/json"} 32 | 33 | r = requests.post(self.manpage_api, params=payload, headers=headers) 34 | 35 | if r.status_code == 200: 36 | return r.json() 37 | 38 | return None 39 | 40 | def __call_stack_exchange_api__(self, query: str, limit: int = 1): 41 | 42 | payload = { 43 | 'text': query, 44 | 'limit': limit 45 | } 46 | 47 | headers = {'Content-Type': "application/json"} 48 | 49 | r = requests.post(self.stack_exchange_api, data=json.dumps(payload), headers=headers) 50 | 51 | if r.status_code == 200: 52 | return r.json()['hits'] 53 | 54 | return None 55 | 56 | def search(self, query, service='stack_exchange', size=10): 57 | if service == 'stack_exchange': 58 | res = self.__call_stack_exchange_api__(query, size) 59 | elif service == 'manpages': 60 | res = self.__call_manpage_api__(query, size) 61 | else: 62 | raise AttributeError("Please select stack_exchange or manpage as a choice for service") 63 | 64 | return res 65 | -------------------------------------------------------------------------------- /clai/server/plugins/helpme/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo " Phase 1: Installing necessary dependencies" 5 | echo "===============================================================" 6 | # Install Python3 dependencies 7 | if [[ "$1" = "--user" ]]; then 8 | pip3 install --user -r requirements.txt 9 | else 10 | sudo pip3 install -r requirements.txt 11 | fi 12 | -------------------------------------------------------------------------------- /clai/server/plugins/helpme/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=helpme 3 | description=This skill examines an error on the command line and tries to retrieve the most relevant solution from an online, community-sourced corpus (such as the Unix forum on Stack Exchange). 4 | default=no 5 | -------------------------------------------------------------------------------- /clai/server/plugins/helpme/requirements.txt: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/README.md: -------------------------------------------------------------------------------- 1 | # howdoi 2 | 3 | `NLP`   `Q&A`   `Retrieval`   `Support` 4 | 5 | This skill is used to do a natural language search over a corpus to recommend a command based on the user's question. 6 | Currently, it uses the [Stack Exchange Unix forum](https://unix.stackexchange.com/) as the corpus for the search. 7 | 8 | ## Implementation 9 | 10 | The skill is composed of: 11 | 1. A REST API deployed on [IBM Cloud](https://cloud.ibm.com/) that responds to the user query; and 12 | 2. Local code for the skill using the CLAI API that decides when to call the REST API and how to then serve the response back to the CLAI-enabled terminal. 13 | 14 | The user query is searched against the Stack Exchange Unix forum to find 15 | an accepted answer, or the most highly rated answer in case an accepted answer is unavailable. 16 | As an illustration, we have employed a simple index-based [text search](https://docs.mongodb.com/manual/text-search/) 17 | that comes built-in with MongoDB. 18 | The body of that answer is then compared against available manpages using a REST API from 19 | [man page explorer](../manpage_agent/) 20 | to recommend a relevant command to explore. 21 | 22 | [`Download corpus data`](https://archive.org/download/stackexchange/unix.stackexchange.com.7z) 23 | 24 | Note that this is merely illustrative of the "how do i" interaction pattern on the CLAI-enabled command line 25 | and the accuracy of the recommendations is highly dependent on the quality of the implemented search on the 26 | Stack Exchange Unix forum data as well as the answer quality of the matched question. 27 | We hope to build, as a community, standards and benchmarks to make this more accurate over time. 28 | Watch this space! 29 | 30 | ## Example Usage 31 | 32 | ![howdoi](https://www.dropbox.com/s/lje4afzr8pu8r8a/howdoi.gif?raw=1) 33 | 34 | The skill will respond to how/what questions if it finds a reasonable solution. Try out: 35 | 36 | 1. `clai "howdoi" when to use sudo vs su?` 37 | 2. `clai "howdoi" find out disk usage per user?` 38 | 3. `clai "howdoi" how to process gz files?` 39 | 40 | ## [xkcd](https://uni.xkcd.com/) 41 | Sadly, this is a true story. At least I learned about the OS X 'say' command. 42 | 43 | ![alt text](https://imgs.xkcd.com/comics/im_an_idiot.png "Sadly, this is a true story. At least I learned about the OS X 'say' command.") 44 | -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/data.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import os 9 | import json 10 | import configparser 11 | from pathlib import Path 12 | 13 | import requests 14 | 15 | 16 | class Datastore: 17 | def __init__(self): 18 | config = configparser.ConfigParser() 19 | config.read(os.path.join(str(Path(__file__).parent.absolute()), 'config.ini')) 20 | 21 | self.stack_exchange_api = config['API'].get('stack_exchange_api') 22 | self.manpage_api = config['API'].get('manpage_api') 23 | 24 | def __call_manpage_api__(self, query: str, limit: int = 1): 25 | 26 | payload = { 27 | 'text': query, 28 | 'result_count': limit 29 | } 30 | 31 | headers = {'Content-Type': "application/json"} 32 | 33 | r = requests.post(self.manpage_api, params=payload, headers=headers) 34 | 35 | if r.status_code == 200: 36 | return r.json() 37 | 38 | return None 39 | 40 | def __call_stack_exchange_api__(self, query: str, limit: int = 1): 41 | 42 | payload = { 43 | 'text': query, 44 | 'limit': limit 45 | } 46 | 47 | headers = {'Content-Type': "application/json"} 48 | 49 | r = requests.post(self.stack_exchange_api, data=json.dumps(payload), headers=headers) 50 | 51 | if r.status_code == 200: 52 | return r.json()['hits'] 53 | 54 | return None 55 | 56 | def search(self, query, service='stack_exchange', size=10): 57 | if service == 'stack_exchange': 58 | res = self.__call_stack_exchange_api__(query, size) 59 | elif service == 'manpages': 60 | res = self.__call_manpage_api__(query, size) 61 | else: 62 | raise AttributeError("Please select stack_exchange or manpage as a choice for service") 63 | 64 | return res 65 | -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo " Phase 1: Installing necessary dependencies" 5 | echo "===============================================================" 6 | # Install Python3 dependencies 7 | if [[ "$1" = "--user" ]]; then 8 | pip3 install --user -r requirements.txt 9 | else 10 | sudo pip3 install -r requirements.txt 11 | fi 12 | 13 | -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=howdoi 3 | description=The search agent determines the best command to return to user inquiry. 4 | default=no 5 | exclude=OS/390 Z/OS 6 | -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/question_detection.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import spacy 9 | 10 | 11 | class QuestionDetection(object): 12 | 13 | def __init__(self): 14 | self.nlp = spacy.load('en_core_web_sm') 15 | self.WH_TAGS = ['WDT', 'WP', 'WP$', 'WRB'] # WH word tags. These signify the presence of an interrogative word 16 | 17 | def is_question(self, text): 18 | """ 19 | A function to check if text is phrased as a question. 20 | 21 | Algorithm: 22 | 1. Test if the text ends with "?". If yes, then it's a question. 23 | 2. Test if the text contains any "WH" word tags (indicative of WH type of question) 24 | 25 | Args: 26 | text (str): a text to be tested 27 | 28 | Returns: 29 | (bool): True if the text is a question. 30 | """ 31 | 32 | if text.endswith('?'): 33 | return True 34 | 35 | doc = self.nlp(text) 36 | for token in doc: 37 | if token.tag_ in self.WH_TAGS: 38 | return True 39 | 40 | return False 41 | -------------------------------------------------------------------------------- /clai/server/plugins/howdoi/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | spacy 3 | https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz#egg=en_core_web_sm 4 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/Dockerfile: -------------------------------------------------------------------------------- 1 | # Start with a python 3 image 2 | FROM python:3 3 | 4 | # Document who is responsible for this image 5 | MAINTAINER Bashbot "bashbot@us.ibm.com" 6 | 7 | # Expose any ports the app is expecting in the environment 8 | EXPOSE 8081 9 | 10 | # Set up a working folder and install the pre-reqs 11 | WORKDIR /app 12 | ADD sample-application-files/requirements.txt /app 13 | RUN pip install -r requirements.txt 14 | 15 | # Add the code as the last Docker layer because it changes the most 16 | ADD sample-application-files/run.py /app 17 | 18 | # Run the service 19 | CMD [ "python", "run.py" ] 20 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/README.md: -------------------------------------------------------------------------------- 1 | # ibmcloud 2 | 3 | `Automation`   `Planning`   `Reinforcement Learning`   `NLP` 4 | 5 | # Implementation 6 | 7 | This skill provides an example of automation and support for the deployment pipeline of applications to cloud platforms such as IBM Cloud. The skills maintains state and calls an external planner to generate the pipeline on the fly. 8 | 9 | + A toy domain for deploying a dockerized application to the IBM cloud is available [here](planning-files). The task is modeled as a planning problem and needs to be specified by a domain expert. 10 | + An [external service](planners.py#L49) uses the [`FAST-DOWNWARD`](http://www.fast-downward.org/) planner to compute a 11 | solution to the specified task, based on the current state of the system and the user command. 12 | 13 | ![ibmcloud-s2](https://www.dropbox.com/s/dk1fuiulzwymqs1/ss2.png?raw=1) 14 | 15 | + The skill also illustrates how a preliminary execution monitor can be built [here](). We use [`pr2plan`]() to monitor user activity and so that the current state of the system is reflected in the computed solution. For example, here we have asked the skill to run the Dockerfile locally first and so the deployment pipepline no longer needs to build the application again. 16 | 17 | ![ibmcloud-s1](https://www.dropbox.com/s/0df60ceiihgv8o0/ss1.png?raw=1) 18 | 19 | While this skill illustrates an integration of automated planning technologies into user interactions on the command line, the toy domain is written manually now. This approach is unlikely to scale to more complex domains. However, the CLAI API allows one to learn this by observing the user(s) over time. We hope to release datasets and challenges around this task in the future. Watch this space! 20 | 21 | ## Example Usage 22 | 23 | ![ibmcloud-gif](https://www.dropbox.com/s/zkhqldf35xvih4t/ibmcloud.gif?raw=1) 24 | 25 | Try this out (folllow instructions as they show up): 26 | 27 | 1. `>> run Dockerfile` 28 | 2. `>> build yaml` 29 | 3. `>> deploy Dockerfile to kube` 30 | 31 | ## [xkcd](https://uni.xkcd.com/) 32 | All services are microservices if you ignore most of their features. 33 | 34 | ![alt text](https://imgs.xkcd.com/comics/containers.png "All services are microservices if you ignore most of their features.") 35 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: clai-service 5 | labels: 6 | name: clai-service 7 | spec: 8 | type: NodePort 9 | ports: 10 | - port: 8085 11 | targetPort: 8081 12 | selector: 13 | app: clai-app 14 | --- 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | name: clai-app 19 | labels: 20 | name: clai-app 21 | spec: 22 | replicas: 1 23 | selector: 24 | matchLabels: 25 | app: clai-app 26 | template: 27 | metadata: 28 | labels: 29 | app: clai-app 30 | spec: 31 | containers: 32 | - image: us.icr.io//app:v1 33 | name: clai-app 34 | imagePullPolicy: Always 35 | env: 36 | - name: HOST 37 | value: 0.0.0.0 38 | - name: PORT 39 | value: 8081 -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | if [[ "$1" = "--user" ]]; then 12 | pip3 install --user -r requirements.txt 13 | else 14 | pip3 install -r requirements.txt 15 | fi 16 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=ibmcloud 3 | description=This skill helps you deploy applications to the IBM Cloud. 4 | default=no 5 | exclude=OS/390 Z/OS 6 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/planning-files/problem.pddl: -------------------------------------------------------------------------------- 1 | (define (problem deploy) 2 | (:domain kube) 3 | (:objects 4 | username password tag name name-to-delete namespace host_port local_port cluster_name protocol yaml cluster-config - object 5 | tcp nodeport - ptype 6 | ) 7 | 8 | (:init 9 | (= (total-cost) 0) 10 | (known username) 11 | (known password) 12 | (known name) 13 | (known tag) 14 | (known local_port) 15 | ) 16 | 17 | (:goal (and 18 | (deployed) 19 | )) 20 | 21 | (:metric minimize (total-cost)) 22 | 23 | ) 24 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/planning-files/template.pddl: -------------------------------------------------------------------------------- 1 | (define (problem deploy) 2 | (:domain kube) 3 | (:objects 4 | username password tag name name-to-delete namespace host_port local_port cluster_name protocol yaml cluster-config - object 5 | tcp nodeport - ptype 6 | ) 7 | 8 | (:init 9 | (= (total-cost) 0) 10 | ) 11 | 12 | (:goal (and 13 | 14 | )) 15 | 16 | (:metric minimize (total-cost)) 17 | 18 | ) 19 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/requirements.txt: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/sample-application-files/app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: {tag} 2 | kind: Service 3 | metadata: 4 | name: clai-service 5 | labels: 6 | name: clai-service 7 | spec: 8 | type: {protocol} 9 | ports: 10 | - port: {host_port} 11 | targetPort: {local_port} 12 | selector: 13 | app: clai-app 14 | --- 15 | apiVersion: apps/{tag} 16 | kind: Deployment 17 | metadata: 18 | name: clai-app 19 | labels: 20 | name: clai-app 21 | spec: 22 | replicas: 1 23 | selector: 24 | matchLabels: 25 | app: clai-app 26 | template: 27 | metadata: 28 | labels: 29 | app: clai-app 30 | spec: 31 | containers: 32 | - image: us.icr.io/{namespace}/{name}:{tag} 33 | name: clai-app 34 | imagePullPolicy: Always 35 | env: 36 | - name: HOST 37 | value: 0.0.0.0 38 | - name: PORT 39 | value: {local_port} -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/sample-application-files/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==1.0.2 2 | flask-cors 3 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/sample-application-files/run.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, render_template 2 | from flask_cors import CORS 3 | import json, os 4 | 5 | 6 | '''''' 7 | app = Flask(__name__) 8 | app.config['TEMPLATES_AUTO_RELOAD'] = True 9 | CORS(app) 10 | 11 | @app.route("/") 12 | def hello(): 13 | print('Application running!') 14 | return 'Hello World!' 15 | 16 | if __name__ == "__main__": 17 | app.run(host=os.getenv('HOST', '0.0.0.0'), port=int(os.getenv('PORT', 8081)), debug=True) 18 | -------------------------------------------------------------------------------- /clai/server/plugins/ibmcloud/temp.dat: -------------------------------------------------------------------------------- 1 | OK 2 | Name ID State Created Workers Location Version Resource Group Name Provider 3 | bashbot blo0s52d0bg15a2brif0 normal 3 weeks ago 1 Dallas 1.14.6_1531 AICL-Orchestrator classic 4 | -------------------------------------------------------------------------------- /clai/server/plugins/linuss/README.md: -------------------------------------------------------------------------------- 1 | # linuss 2 | 3 | `z/OS`   `MVS` 4 | 5 | The Linux to uss plugin or "Linuss" aims to help users adapt to the uss environment quicker. It uses a set of predefined equivalences found by comparing the man pages of each system. When a command is entered on uss the equivalences are checked and suggested to the user if detected. Suggestions can range from an equivalent flag up to an entire command replacement. 6 | 7 | ## Example Usage 8 | 9 | ![linuss](linuss.gif) 10 | -------------------------------------------------------------------------------- /clai/server/plugins/linuss/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/linuss/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | echo "===============================================================" 5 | echo "" 6 | echo " Phase 1: Installing necessary tools" 7 | echo "" 8 | echo "===============================================================" 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | if [[ "$1" = "--user" ]]; then 12 | pip3 install --user -r requirements.txt 2> /dev/null 13 | else 14 | sudo pip3 install -r requirements.txt 2> /dev/null 15 | fi 16 | 17 | -------------------------------------------------------------------------------- /clai/server/plugins/linuss/linuss.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/server/plugins/linuss/linuss.gif -------------------------------------------------------------------------------- /clai/server/plugins/linuss/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=linuss 3 | description= 4 | default=no 5 | exclude=darwin linux 6 | z_default=yes -------------------------------------------------------------------------------- /clai/server/plugins/linuss/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/server/plugins/linuss/requirements.txt -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/README.md: -------------------------------------------------------------------------------- 1 | # man page explorer 2 | 3 | `Q&A`   `NLP`   `Retrieval` 4 | 5 | This skill allows you to ask Bash for relevant commands by describing your task in natural language. The command whose man page document matches most with the users query is suggested by the skill. 6 | 7 | ## Implementation 8 | 9 | The skills is structured as a REST API deployed on [IBM Cloud](https://cloud.ibm.com/) responding to the users query, and a local agent deciding when to call this REST API, and then serving the API response back to CLAI. 10 | 11 | The REST API is housed in the [`webapp`](./webapp) folder. The [`train_sklearn_agent.sh`](./webapp/train_sklearn_agent.sh) file fetches all the man pages in the system, and then trains a sklearn TF-IDF vectorizer on the collected corpus. All the man pages are then encoded using this trained vectorizer. The [`app.py`](./webapp/app.py) file loads the saved sklearn TF-IDF vectorizer model, and on receiving a query, encodes it using the same vectorizer, and returns the command whose man page has the highest cosine similarity with the encoded query. 12 | 13 | On the local system, the skill decides to call the API if the query is detected as a question [[`code`]](./question_detection.py), gets the suggested command from the API, and also integrates the [`tldr`](https://tldr.sh/) plugin to summarize the suggested command. 14 | 15 | ## Example Usage 16 | 17 | ![manpage-agent](https://www.dropbox.com/s/7so8yumbaip7b0o/man%20page%20agent.gif?raw=1) 18 | 19 | 1. `>> how to clear screen` 20 | 2. `>> how do I change the file owner` 21 | 3. `>> how to compile java class file` 22 | 4. `>> how to compress a file into a tar file?` 23 | 24 | ## [xkcd](https://uni.xkcd.com/) 25 | Life is too short for man pages, but occasionally much too short without them. 26 | 27 | ![alt text](https://imgs.xkcd.com/comics/rtfm.png "Life is too short for man pages, but occasionally much too short without them.") 28 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "API_URL": "https://clai-manpage-agent.mybluemix.net/findManPage/" 3 | } -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | echo "===============================================================" 5 | echo "" 6 | echo " Phase 1: Installing necessary tools" 7 | echo "" 8 | echo "===============================================================" 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | if [[ "$1" = "--user" ]]; then 12 | pip3 install --user -r requirements.txt 2> /dev/null 13 | else 14 | sudo pip3 install -r requirements.txt 2> /dev/null 15 | fi 16 | 17 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=man page explorer 3 | description=This skill allows you to ask for relevant man pages by describing your task in natural language. 4 | default=no 5 | exclude=OS/390 Z/OS 6 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/question_detection.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import spacy 9 | 10 | 11 | class QuestionDetection(object): 12 | 13 | def __init__(self): 14 | self.nlp = spacy.load('en_core_web_sm') 15 | self.WH_TAGS = ['WDT', 'WP', 'WP$', 'WRB'] # WH word tags. These signify the presence of an interrogative word 16 | 17 | def is_question(self, text): 18 | 19 | if text.endswith('?'): 20 | return True 21 | 22 | doc = self.nlp(text) 23 | for token in doc: 24 | if token.tag_ in self.WH_TAGS: 25 | return True 26 | 27 | return False 28 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/requirements.txt: -------------------------------------------------------------------------------- 1 | tldr 2 | requests 3 | spacy 4 | https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz#egg=en_core_web_sm -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/tldr_wrapper.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.tools.colorize_console import Colorize 9 | import sys 10 | 11 | try: 12 | import tldr 13 | except Exception as _: 14 | pass 15 | 16 | TLDR_AVAILABLE = 'tldr' in sys.modules 17 | 18 | 19 | def get_command_tldr(cmd): 20 | 21 | if not TLDR_AVAILABLE: 22 | return '' 23 | 24 | cmd_tldr = tldr.get_page(cmd) 25 | 26 | if cmd_tldr is None: 27 | return '' 28 | 29 | description = Colorize() 30 | 31 | for i, line in enumerate(cmd_tldr): 32 | line = line.rstrip().decode('utf-8') 33 | 34 | if i == 0: 35 | description.append('-'*50 + '\n') 36 | 37 | if len(line) < 1: # Empty line 38 | description.append('\n') 39 | elif line[0] == '#': 40 | line = line[1:] 41 | description.warning().append(line.strip() + '\n') 42 | elif line[0] == '>': # Description line 43 | line = ' ' + line[1:] 44 | description.normal().append(line.strip() + '\n') 45 | elif line[0] == '-': # Example line 46 | description.normal().append(line.strip() + '\n') 47 | elif line[0] == '`': # Example command 48 | line = ' ' + line[1:-1] 49 | description.info().append(line.strip() + '\n') 50 | 51 | description.normal().append('summary provided by tldr package\n') 52 | description.normal().append('-'*50 + '\n') 53 | 54 | return description.to_console() 55 | 56 | 57 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/.gitignore: -------------------------------------------------------------------------------- 1 | data/ -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/Procfile: -------------------------------------------------------------------------------- 1 | web: python app.py > stdout.txt 2>stderr.txt 2 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/app.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import os 9 | from flask import Flask, request, jsonify 10 | from data import Datastore 11 | 12 | app = Flask(__name__) 13 | port = int(os.getenv("PORT", 5000)) 14 | 15 | _DATASTORE = Datastore() 16 | 17 | 18 | @app.route('/', methods=['GET']) 19 | def homepage(): 20 | return "Agent up" 21 | 22 | 23 | @app.route('/findManPage/', methods=['POST']) 24 | def find_man_page(): 25 | 26 | try: 27 | text = request.args.get('text', None) 28 | result_count = request.args.get('result_count', 1) 29 | result_count = int(result_count) 30 | 31 | if text is None: 32 | return 33 | 34 | result = _DATASTORE.search(text.strip(), result_count) 35 | 36 | commands = [x[0] for x in result] 37 | dists = [x[1] for x in result] 38 | except Exception as err: 39 | payload = { 40 | 'result': 'error', 41 | 'message': str(err) 42 | } 43 | else: 44 | payload = { 45 | 'status': 'success', 46 | 'commands': commands, 47 | 'dists': dists 48 | } 49 | finally: 50 | return jsonify(payload) 51 | 52 | 53 | if __name__ == "__main__": 54 | app.run(host="0.0.0.0", port=int(port)) 55 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/data.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import pickle 9 | import numpy as np 10 | from sklearn.metrics.pairwise import cosine_similarity 11 | 12 | 13 | class Datastore: 14 | def __init__(self): 15 | self.path = './data' 16 | self.transformer_func, self.vectors, self.commands = self.read() 17 | 18 | def read(self): 19 | with open(self.path + '/model/func.p', 'rb') as f: 20 | transformer_func = pickle.load(f) 21 | 22 | with open(self.path + '/model/vectors.p', 'rb') as f: 23 | vectors = pickle.load(f) 24 | 25 | with open(self.path + '/model/keys.p', 'rb') as f: 26 | commands = pickle.load(f) 27 | 28 | return transformer_func, vectors, commands 29 | 30 | def search(self, query, size=1): 31 | vector = self.transformer_func.transform([query]) 32 | dist = cosine_similarity(self.vectors, vector).reshape(-1) 33 | 34 | return [(self.commands[i], dist[i]) for i in np.argsort(dist)[-size:]] 35 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/manifest.yml: -------------------------------------------------------------------------------- 1 | applications: 2 | - name: clai-manpage-agent 3 | memory: 1G 4 | path: . 5 | buildpacks: 6 | - python_buildpack 7 | -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | sklearn 3 | flask -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.6.9 -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/train_requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | sklearn -------------------------------------------------------------------------------- /clai/server/plugins/manpage_agent/webapp/train_sklearn_agent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DIR="data/" 4 | MANPAGES_DIR="data/manpages" 5 | MODEL_DIR="data/model" 6 | 7 | 8 | # Remove data folder if exists already 9 | if [[ -d "${DIR}" ]]; then rm -Rf ${DIR}; fi 10 | mkdir "${DIR}" 11 | 12 | echo "===============================================================" 13 | echo "" 14 | echo " Phase 1: Installing necessary tools" 15 | echo "" 16 | echo "===============================================================" 17 | # Install Python3 dependencies 18 | echo ">> Installing python dependencies" 19 | python3 -m pip install -r train_requirements.txt 20 | 21 | echo "===============================================================" 22 | echo "" 23 | echo " Phase 2: Recovering manpages from the operating system" 24 | echo "" 25 | echo "===============================================================" 26 | # Remove manpages folder if exists already 27 | if [[ -d "${MANPAGES_DIR}" ]]; then rm -Rf ${MANPAGES_DIR}; fi 28 | 29 | echo " >> Making manpages directory" 30 | mkdir "${MANPAGES_DIR}" 31 | 32 | echo " >> Getting a list of commands" 33 | # Get a list of commands and store in file 34 | man -k . | awk '{print $1}' | sed 's/(.*//' > "${MANPAGES_DIR}/cmds.txt" 35 | 36 | echo " >> Getting manual page data for each command" 37 | # Read each command from the file and extract man page content 38 | while read -r line 39 | do 40 | 41 | man ${line} | col -b > "${MANPAGES_DIR}/${line}.txt" 42 | 43 | done< "${MANPAGES_DIR}/cmds.txt" 44 | 45 | echo "===============================================================" 46 | echo "" 47 | echo " Phase 3: Building Elastic search index for forum and manpages" 48 | echo "" 49 | echo "===============================================================" 50 | # Remove model folder if exists already 51 | if [[ -d "${MODEL_DIR}" ]]; then rm -Rf ${MODEL_DIR}; fi 52 | 53 | echo " >> Making manpages directory" 54 | mkdir "${MODEL_DIR}" 55 | 56 | python3 train_sklearn_agent.py 57 | 58 | echo " >> Removing manpage corpus directory" 59 | rm -rf "${MANPAGES_DIR}" 60 | 61 | echo "===============================================================" 62 | echo " Training complete " 63 | echo "===============================================================" -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | if [[ "$1" = "--user" ]]; then 12 | pip3 install --user -r requirements.txt 13 | else 14 | pip3 install -r requirements.txt 15 | fi 16 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=nlc2cmd 3 | description=This skill helps you compress archives, find files, etc. using English commands. 4 | default=no -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/nlc2cmd.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.agent import Agent 9 | from clai.server.command_message import State, Action, NOOP_COMMAND 10 | 11 | from clai.server.plugins.nlc2cmd.service import Service 12 | from clai.tools.colorize_console import Colorize 13 | 14 | 15 | class NLC2CMD(Agent): 16 | def __init__(self): 17 | super(NLC2CMD, self).__init__() 18 | self.service = Service() 19 | 20 | def get_next_action(self, state: State) -> Action: 21 | command = state.command 22 | data, confidence = self.service(command) 23 | response = data["text"] 24 | 25 | return Action( 26 | suggested_command=NOOP_COMMAND, 27 | execute=True, 28 | description=Colorize().info().append(response).to_console(), 29 | confidence=confidence) 30 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/remote/Procfile: -------------------------------------------------------------------------------- 1 | web: python3 run.py > stdout.txt 2>stderr.txt 2 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/remote/README.md: -------------------------------------------------------------------------------- 1 | ## Remote WA router 2 | 3 | This flask application is a router for requests to the watson assistant instance 4 | that processes most of the `nlc2cmd` use cases. To bring up the server: 5 | 6 | 1. Fill in the details of your assistant and skill in `config.json` 7 | 2. Run `>> python3 run.py` 8 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/remote/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "i_am_id" : "", 3 | "skills" : { 4 | 5 | "tarbot" : "", 6 | "grepbot" : "", 7 | "zosbot" : "" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/remote/manifest.yml: -------------------------------------------------------------------------------- 1 | applications: 2 | - name: nlc2cmd 3 | memory: 1G 4 | path: . 5 | buildpacks: 6 | - python_buildpack 7 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/remote/requirements.txt: -------------------------------------------------------------------------------- 1 | flask>=1.0.0 2 | ibm-watson>=4.1.0 3 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/remote/runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.6.9 -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/requirements.txt: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/service.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # itemgetter is faster than lambda functions for sorting 9 | from operator import itemgetter 10 | 11 | import clai.server.plugins.nlc2cmd.wa_skills as wa_skills 12 | import threading 13 | 14 | class Service: 15 | def __init__(self): 16 | pass 17 | 18 | def __call__(self, *args, **kwargs): 19 | 20 | # call to WA evaluators 21 | def __compute(*args): 22 | result.append( eval('wa_skills.' + args[0])(args[1]) ) 23 | 24 | # Extract user input 25 | msg = args[0] 26 | 27 | result = [] 28 | threads = [] 29 | 30 | for item in dir(wa_skills): 31 | if 'wa_skill_processor' in item: 32 | threads.append( threading.Thread(target=__compute, args=(item, msg)) ) 33 | 34 | for t in threads: t.start() 35 | for t in threads: t.join() 36 | 37 | # return wa skill with the highest confidence 38 | return sorted(result, key=itemgetter(1), reverse=True)[0] 39 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/wa_skills/README.md: -------------------------------------------------------------------------------- 1 | # nlc2cmd for IBM Cloud CLI and z/OS 2 | 3 | `NLP` `Support` `ibmcloud` 4 | 5 | This contains scripts that turn the recognized intents into actionable commands on the command line. 6 | Previously in [../](../) we looked at some common examples using the `tar` and `grep` commands. 7 | Here we want to highlight another instantiation of the nlc2cmd use case, this time for the [IBM Cloud CLI](https://www.ibm.com/cloud/cli) and the `ibmcloud` family of commands on the shell, as well as more specialized 8 | commands in the z/OS USS terminal in mainframes. 9 | 10 | ## Example Usage 11 | 12 | ![nlc2cmd4ibmcloudcli](https://www.dropbox.com/s/35oucoqq4o8pwau/nlc2cloud.gif?raw=1) 13 | 14 | The script [cloudbot.py](cloudbot.py) supports some illustrative commands from the [IBM Cloud CLI cheatsheet](https://github.com/ibm-cloud-docs/cli/blob/master/IBM%20Cloud%20CLI%20quick%20reference.pdf) -- CLAI currently handles all these instances. Try them out, or the ones below as a quick start! 15 | 16 | 1. `>> How do I login?` 17 | 2. `>> how do i organize my resources` 18 | 3. `>> show me how to add tags` 19 | 4. `>> billing costs` 20 | 5. `>> available plugins` 21 | 6. `>> I want to invite someone to my cloud` 22 | 23 | 24 | 25 | Similarly, the script [zosbot.py](zosbot.py) supports some illustrative commands from the 26 | USS (UNIX System Services) terminal in the z/OS operating system. 27 | 28 | 1. `>> copy file to PDS member` 29 | 2. `>> lookup reason codes` 30 | 3. `>> view mvs file` 31 | 32 | ## [xkcd](https://uni.xkcd.com/) 33 | 34 | There's planned downtime every night when we turn on the Roomba and it runs over the cord. 35 | 36 | ![alt text](https://imgs.xkcd.com/comics/the_cloud.png "There's planned downtime every night when we turn on the Roomba and it runs over the cord.") 37 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/wa_skills/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.plugins.nlc2cmd.wa_skills.tarbot import wa_skill_processor_tarbot 9 | from clai.server.plugins.nlc2cmd.wa_skills.grepbot import wa_skill_processor_grepbot 10 | from clai.server.plugins.nlc2cmd.wa_skills.cloudbot import wa_skill_processor_cloudbot 11 | from clai.server.plugins.nlc2cmd.wa_skills.zosbot import wa_skill_processor_zosbot 12 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/wa_skills/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "cloudbot" : "https://ibmcloud-helper.mybluemix.net/message", 3 | "tarbot" : "http://nlc2cmd.mybluemix.net/tarbot", 4 | "grepbot" : "http://nlc2cmd.mybluemix.net/grepbot", 5 | "zosbot" : "http://nlc2cmd.mybluemix.net/zosbot" 6 | } -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/wa_skills/grepbot.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | ''' grep command handler ''' 9 | 10 | ''' imports ''' 11 | from clai.server.plugins.nlc2cmd.wa_skills.utils import call_wa_skill, get_own_name 12 | import re 13 | 14 | ''' globals ''' 15 | __self = get_own_name(__file__) 16 | __flags = { "line-number" : 'n', 17 | "match-case" : 'i', 18 | "count" : 'c', 19 | "verbose" : 'v' } 20 | 21 | def wa_skill_processor_grepbot(msg): 22 | 23 | # Confidence remains at 0 unless an intent has been detected 24 | confidence = 0.0 25 | data = {} 26 | 27 | try: 28 | 29 | match_string = re.findall(r'\"(.+?)\"', msg)[0] 30 | msg = msg.replace('\"{}\"'.format(match_string), "") 31 | 32 | except: 33 | match_string = "" 34 | 35 | response, success = call_wa_skill(msg, __self) 36 | if not success: return {"text" : response}, 0.0 37 | 38 | if msg.startswith('grep for'): confidence = 1.0 39 | else: confidence = max([item['confidence'] for item in response['intents']]) 40 | 41 | filename = "" 42 | dirname = None 43 | 44 | for item in response['entities']: 45 | if item['entity'] == 'directory': 46 | dirname = "" 47 | 48 | if item['entity'] == 'starts-with': 49 | match_string = '^' + match_string 50 | 51 | if item['entity'] == 'ends-with': 52 | match_string = match_string + '$' 53 | 54 | flags = "-" 55 | if dirname: 56 | flags += 'r' 57 | 58 | for intent in response['intents']: 59 | if intent['confidence'] > 0.25 and intent['intent'] != 'find': 60 | flags += __flags[intent['intent']] 61 | 62 | if flags == "-": 63 | flags = "" 64 | 65 | command = 'grep {} \"{}\"'.format(flags, match_string) 66 | 67 | if dirname: command += " {}".format(dirname) 68 | else: command += " {}".format(filename) 69 | 70 | data = { "text": "Try >> " + command } 71 | return data, confidence 72 | 73 | -------------------------------------------------------------------------------- /clai/server/plugins/nlc2cmd/wa_skills/utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | ''' helper functions for wa skills package ''' 9 | 10 | ''' imports ''' 11 | from typing import List 12 | 13 | import requests 14 | import json 15 | import os 16 | 17 | ''' globals ''' 18 | SUCCESS = True 19 | FAILURE = False 20 | 21 | _real_path = '/'.join(os.path.realpath(__file__).split('/')[:-1]) 22 | _path_to_config_file = _real_path + '/config.json' 23 | 24 | services = json.loads( open(_path_to_config_file).read() ) 25 | 26 | 27 | ''' call to remote host of WA service ''' 28 | def call_wa_skill(msg: str, name: str) -> List: 29 | 30 | wa_endpoint = services[name] 31 | 32 | try: 33 | response = requests.put(wa_endpoint, json={'text': msg}).json() 34 | 35 | if response['result'] == 'success': 36 | return response['response']['output'], SUCCESS 37 | else: 38 | return response['result'], FAILURE 39 | 40 | except Exception as ex: 41 | return "Method failed with status " + str(ex), FAILURE 42 | 43 | ''' return name of file being run ''' 44 | def get_own_name(file: str) -> str: 45 | return os.path.basename(file).replace('.py', '') 46 | -------------------------------------------------------------------------------- /clai/server/plugins/tellina/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/tellina/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo "" 5 | echo " Phase 1: Installing necessary tools" 6 | echo "" 7 | echo "===============================================================" 8 | 9 | # Install Python3 dependencies 10 | echo ">> Installing python dependencies" 11 | pip3 install -r requirements.txt -------------------------------------------------------------------------------- /clai/server/plugins/tellina/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=tellina 3 | description=This skill translates your natural language command into a Bash command. 4 | default=no 5 | exclude=OS/390 Z/OS -------------------------------------------------------------------------------- /clai/server/plugins/tellina/requirements.txt: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /clai/server/plugins/tellina/tellina.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.agent import Agent 9 | from clai.server.command_message import State, Action, NOOP_COMMAND 10 | from clai.tools.colorize_console import Colorize 11 | 12 | from clai.server.logger import current_logger as logger 13 | 14 | import requests 15 | 16 | ''' globals ''' 17 | tellina_endpoint = 'http://nlc2cmd.sl.res.ibm.com:8000/api/translate' 18 | 19 | 20 | class TELLINA(Agent): 21 | def __init__(self): 22 | super(TELLINA, self).__init__() 23 | 24 | def get_next_action(self, state: State) -> Action: 25 | 26 | # user typed in, in natural language 27 | command = state.command 28 | 29 | try: 30 | 31 | ## Needs to be a post request since service/endpoint is configured for post 32 | endpoint_comeback = requests.post(tellina_endpoint, json={'command': command}).json() 33 | ## tellina endpoint must return a json with 34 | 35 | # tellina response, the cmd for the user NL utterance 36 | response = 'Try >> ' + endpoint_comeback['response'] 37 | # the confidence; the tellina endpoint currently returns 0.0 38 | confidence = float(endpoint_comeback['confidence']) 39 | 40 | return Action( 41 | suggested_command=NOOP_COMMAND, 42 | execute=True, 43 | description=Colorize().info().append(response).to_console(), 44 | confidence=confidence) 45 | 46 | except Exception as ex: 47 | return [ { "text" : "Method failed with status " + str(ex) }, 0.0 ] 48 | -------------------------------------------------------------------------------- /clai/server/plugins/voice/.gitignore: -------------------------------------------------------------------------------- 1 | openai_api.key -------------------------------------------------------------------------------- /clai/server/plugins/voice/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/voice/install_darwin.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | echo "===============================================================" 5 | echo "" 6 | echo " Phase 1: Installing necessary tools" 7 | echo "" 8 | echo "===============================================================" 9 | 10 | if [[ $(command -v brew) == "" ]]; then 11 | echo ">> Installing Hombrew" 12 | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 13 | else 14 | echo ">> Updating Homebrew" 15 | brew update 16 | fi 17 | 18 | # Install ffmpeg 19 | echo ">> Installing ffmpeg" 20 | if brew ls --versions ffmpeg > /dev/null; then 21 | brew ls --versions ffmpeg 22 | else 23 | brew install ffmpeg > /dev/null 24 | fi 25 | 26 | # Install Python3 dependencies 27 | echo ">> Installing python dependencies" 28 | 29 | python3 -m pip install -r requirements.txt 2> /dev/null 30 | -------------------------------------------------------------------------------- /clai/server/plugins/voice/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=voice 3 | description=This skill summarizes console output and converts it into speech 4 | default=no 5 | exclude=Linux OS/390 Z/OS 6 | -------------------------------------------------------------------------------- /clai/server/plugins/voice/priming.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": "bash: clean: command not found", 4 | "output": "command clean not found" 5 | }, 6 | 7 | { 8 | "input": "bash: tae: command not found", 9 | "output": "command tae not found" 10 | }, 11 | { 12 | "input": "tar: testdir: Cannot stat: No such file or directory. tar: Error exit delayed from previous errors", 13 | "output": "no file or directory named testdir" 14 | }, 15 | { 16 | "input": "ls: invalid option -- 'y'", 17 | "output": "Invalid option y for command ls" 18 | }, 19 | { 20 | "input": "script.sh: line 9: syntax error: unexpected end of file", 21 | "output": "unexpected end of file for script.sh at line 9" 22 | }, 23 | { 24 | "input": "find: `./root` : Permission denied", 25 | "output": "permission denied to find command for directory root" 26 | }, 27 | { 28 | "input": "ls: Messages: Operation not permitted", 29 | "output": "permission denied to ls command for directory Messages" 30 | } 31 | ] -------------------------------------------------------------------------------- /clai/server/plugins/voice/requirements.txt: -------------------------------------------------------------------------------- 1 | gTTS 2 | openai -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/README.md: -------------------------------------------------------------------------------- 1 | # zmsgcode 2 | 3 | `Retrieval`   `Support` 4 | 5 | IBM Z error messages can seem daunting to an end-user. The `zmsgcode` skill 6 | tries to make them a little friendlier by automatically displaying a descriptive 7 | help message whenever the console detects any IBM Z style message codes. 8 | 9 | ## Implementation 10 | `zmsgcode` actively monitors the console's standard error stream. Whenever new 11 | data arrives, a regex match is performed on it to identify the style of error 12 | message IDs used by IBM Z operating systems and software. When it detects 13 | one, the following occurs: 14 | 15 | 1. If the stream includes a hexadecimal reason code, `zmsgcode` will call the 16 | `bpxmtext` application to try to get a message description. 17 | 2. If there is no reason code, or if `bpxmtext` doesn't return a message, 18 | `zmsgcode` will try to search the USS publications on the z/OS KnowledgeCenter 19 | website, returning the first result encountered. 20 | 21 | ## Example Usage 22 | 23 | ![zmsgcode](zmsgcode.gif?raw=1) 24 | 25 | The skill will respond to `stderr` if it finds a reasonable solution. 26 | -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/config.ini: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration file for the 'zmsgcode' CLAI skill 3 | # 4 | # Each section in this configuration file denotes a specific 'search provider'. 5 | # Search providers will be searched in the order in which they are encountered 6 | # here. 7 | # 8 | # There is one required parameters per section: 9 | # 10 | # api [REQUIRED] 11 | # URL for the RESTful search provider 12 | # 13 | 14 | # Provider: IBM KnowledgeCenter 15 | # Notes: 16 | # Some installation sites are "dark" (off the Internet), in which case 17 | # they may have a KnowledgeCenter instance being served on their closed LAN. 18 | # To use such a provider, specify the local IP or hostname; an example is 19 | # provided below. 20 | [ibm_kc] 21 | #api=https://192.168.1.100/support/knowledgecenter/v1/search 22 | api=https://www.ibm.com/support/knowledgecenter/v1/search -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "===============================================================" 4 | echo " Phase 1: Installing necessary dependencies" 5 | echo "===============================================================" 6 | # Install Python3 dependencies 7 | if [[ "$1" = "--user" ]]; then 8 | pip3 install --user -r requirements.txt 9 | else 10 | sudo pip3 install -r requirements.txt 11 | fi 12 | 13 | -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=zmsgcode 3 | description=Look up information on IBM z message codes 4 | default=no 5 | exclude=Linux Darwin 6 | -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/requirements.txt: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /clai/server/plugins/zmsgcode/zmsgcode.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/clai/3215e3676b4a0857a56a1e126a052f089be5ff03/clai/server/plugins/zmsgcode/zmsgcode.gif -------------------------------------------------------------------------------- /clai/server/searchlib/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/searchlib/data.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import configparser 9 | from collections import OrderedDict 10 | from typing import List, Dict 11 | 12 | from clai.server.logger import current_logger as logger 13 | from clai.server.searchlib.se_provider import StackExchange 14 | from clai.server.searchlib.kc_provider import KnowledgeCenter 15 | from clai.server.searchlib.man_provider import Manpages 16 | 17 | 18 | class Datastore: 19 | # Instance data members 20 | apis: OrderedDict = {} 21 | 22 | def __init__(self, inifile_path: str): 23 | config = configparser.ConfigParser() 24 | config.read(inifile_path) 25 | 26 | # Get a list of APIs defined in the config file 27 | for section in config.sections(): 28 | if section == "stack_exchange": 29 | self.apis[section] = StackExchange( 30 | section, "Unix StackExchange forums", config[section] 31 | ) 32 | elif section == "ibm_kc": 33 | self.apis[section] = KnowledgeCenter( 34 | section, "IBM KnowledgeCenter", config[section] 35 | ) 36 | elif section == "manpages": 37 | self.apis[section] = Manpages(section, "manpages", config[section]) 38 | else: 39 | raise AttributeError(f"Unsupported service type: '{section}'") 40 | 41 | logger.debug(f"Sections in {inifile_path}: {str(self.apis)}") 42 | 43 | def get_apis(self) -> OrderedDict: 44 | return self.apis 45 | 46 | def search(self, query, service="stack_exchange", size=10, **kwargs) -> List[Dict]: 47 | supported_services = self.apis.keys() 48 | 49 | if service in supported_services: 50 | service_provider = self.apis[service] 51 | res = service_provider.call(query, size, **kwargs) 52 | else: 53 | raise AttributeError(f"service must be one of: {str(supported_services)}") 54 | 55 | return res 56 | -------------------------------------------------------------------------------- /clai/server/searchlib/man_provider.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import List, Dict 9 | 10 | from clai.server.searchlib.providers import Provider 11 | 12 | 13 | class Manpages(Provider): 14 | def __init__(self, name: str, description: str, section: dict): 15 | super().__init__(name, description, section) 16 | self.__log_debug__("Manpages provider initialized") 17 | 18 | def call(self, query: str, limit: int = 1, **kwargs): 19 | self.__log_debug__( 20 | f"call(query={query}, limit={str(limit)}), **kwargs={str(kwargs)})" 21 | ) 22 | 23 | payload = {"text": query, "result_count": limit} 24 | 25 | request = self.__send_post_request__(self.base_uri, params=payload) 26 | if request.status_code == 200: 27 | return request.json() 28 | 29 | return None 30 | 31 | def extract_search_result(self, data: List[Dict]) -> str: 32 | pass 33 | 34 | def get_printable_output(self, data: List[Dict]) -> str: 35 | pass 36 | -------------------------------------------------------------------------------- /clai/server/searchlib/se_provider.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import json 9 | 10 | from typing import List, Dict 11 | from clai.server.searchlib.providers import Provider 12 | 13 | 14 | class StackExchange(Provider): 15 | def __init__(self, name: str, description: str, section: dict): 16 | super().__init__(name, description, section) 17 | self.__log_debug__("UNIX StackExchange provider initialized") 18 | 19 | def call(self, query: str, limit: int = 1, **kwargs): 20 | self.__log_debug__( 21 | f"call(query={query}, limit={str(limit)}), **kwargs={str(kwargs)})" 22 | ) 23 | 24 | payload = {"text": query, "limit": limit} 25 | 26 | request = self.__send_post_request__(self.base_uri, data=json.dumps(payload)) 27 | if request.status_code == 200: 28 | return request.json()["hits"] 29 | 30 | return None 31 | 32 | def extract_search_result(self, data: List[Dict]) -> str: 33 | return data[0]["Answer"] 34 | 35 | def get_printable_output(self, data: List[Dict]) -> str: 36 | lines = [ 37 | f"Post: {data[0]['Content'][:384] + ' ...'}", 38 | f"Answer: {data[0]['Answer'][:256] + ' ...'}", 39 | f"Link: {data[0]['Url']}\n", 40 | ] 41 | 42 | return "\n".join(lines) 43 | -------------------------------------------------------------------------------- /clai/server/server_connector.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import abc 9 | from typing import Callable 10 | 11 | 12 | class ServerConnector(abc.ABC): 13 | @abc.abstractmethod 14 | def create_socket(self, host: str, port: int): 15 | '''This method inialize the connection''' 16 | 17 | @abc.abstractmethod 18 | def loop(self, process_message: Callable[[bytes], str]): 19 | '''This is the loop method for manage the connection''' 20 | -------------------------------------------------------------------------------- /clai/server/state_mapper.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import json 9 | from clai.server.command_message import Action 10 | 11 | 12 | def process_message(data: str) -> Action: 13 | return Action(**json.loads(data)) 14 | -------------------------------------------------------------------------------- /clai/server/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/utilities/gpt3/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/server/web_socket_client_connector.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import asyncio 9 | import websockets 10 | 11 | from clai.server.client_connector import ClientConnector 12 | from clai.server.command_message import StateDTO, Action 13 | from clai.server.state_mapper import process_message 14 | 15 | 16 | # pylint: disable=too-few-public-methods 17 | class WebSocketClientConnector(ClientConnector): 18 | DEFAULT_HOST = "ws://clai-server-test.mybluemix.net" 19 | 20 | def __init__(self, 21 | host: str = DEFAULT_HOST): 22 | self.host = host 23 | 24 | def send(self, message: StateDTO) -> Action: 25 | response = asyncio.get_event_loop() \ 26 | .run_until_complete(self.__send_message(message)) 27 | return response 28 | 29 | async def __send_message(self, message: StateDTO): 30 | async with websockets.connect(self.host) as websocket: 31 | await websocket.send(str(message.json())) 32 | 33 | received_data = await websocket.recv() 34 | return process_message(received_data) 35 | -------------------------------------------------------------------------------- /clai/server/web_socket_server_connector.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import asyncio 9 | from typing import Callable 10 | import websockets 11 | 12 | 13 | from clai.server.server_connector import ServerConnector 14 | from clai.server.logger import current_logger as logger 15 | 16 | 17 | class WebSocketServerConnector(ServerConnector): 18 | def __init__(self): 19 | self.process_message = None 20 | self.server_socket = None 21 | 22 | def create_socket(self, host: str, port: int): 23 | self.server_socket = websockets.serve(self.manage_messages, host, port) 24 | 25 | def loop(self, process_message: Callable[[bytes], str]): 26 | self.process_message = process_message 27 | asyncio.get_event_loop().run_until_complete(self.server_socket) 28 | asyncio.get_event_loop().run_forever() 29 | 30 | # pylint: disable=unused-argument 31 | async def manage_messages(self, websocket, path): 32 | data = await websocket.recv() 33 | logger.info(f" read from the web socket < {data}") 34 | action = self.process_message(data) 35 | print(f"> {action}") 36 | action_to_send = str(action.json()).encode('utf8') 37 | await websocket.send(action_to_send) 38 | -------------------------------------------------------------------------------- /clai/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /clai/tools/anonymizer.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import json 9 | import os 10 | import uuid 11 | from typing import Mapping, Optional 12 | 13 | import clai 14 | 15 | 16 | # pylint: disable=too-few-public-methods 17 | class Anonymizer: 18 | def __init__(self, alternate_path: Optional[str] = None): 19 | self.__cache: Mapping[str, str] = None 20 | self.alternate_path = alternate_path 21 | 22 | def __get_config_path__(self): 23 | if self.alternate_path: 24 | return self.alternate_path 25 | 26 | base_dir = os.path.dirname(clai.tools.__file__) 27 | filename = os.path.join(base_dir, '../../anonymize.json') 28 | return filename 29 | 30 | def anonymize(self, key: str) -> str: 31 | if not self.__cache: 32 | self.__init_cache__() 33 | 34 | if key in self.__cache: 35 | return self.__cache[key] 36 | 37 | return self.__create_anonymize(key) 38 | 39 | def __init_cache__(self): 40 | path = self.__get_config_path__() 41 | 42 | if not os.path.exists(path): 43 | self.__cache = {} 44 | return 45 | 46 | with open(path) as verify_file: 47 | self.__cache = json.load(verify_file) 48 | 49 | def __create_anonymize(self, key: str) -> str: 50 | self.__cache[key] = str(uuid.uuid4()) 51 | with open(self.__get_config_path__(), 'w+') as json_file: 52 | json.dump(self.__cache, json_file) 53 | return self.__cache[key] 54 | -------------------------------------------------------------------------------- /clai/tools/colorize_console.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai import PLATFORM 9 | 10 | class Colorize: 11 | WARNING = "\033[91m" 12 | INFO = "\033[95m" 13 | COMPLETE = "\033[32m" 14 | NORMAL = "\033[0m" 15 | 16 | 17 | if PLATFORM in ('zos', 'os390'): 18 | EMOJI_ROBOT = '@' 19 | EMOJI_CHECK = '[x]' 20 | EMOJI_BOX = '[ ]' 21 | else: 22 | EMOJI_ROBOT = '\U0001F916' 23 | EMOJI_CHECK = '\u2611' 24 | EMOJI_BOX = '\u25FB' 25 | 26 | def __init__(self): 27 | self._text_complete = "" 28 | 29 | def append(self, text): 30 | self._text_complete += text 31 | return self 32 | 33 | def warning(self): 34 | self._text_complete += self.WARNING 35 | return self 36 | 37 | def info(self): 38 | self._text_complete += self.INFO 39 | return self 40 | 41 | def complete(self): 42 | self._text_complete += self.COMPLETE 43 | return self 44 | 45 | def normal(self): 46 | self._text_complete += self.NORMAL 47 | return self 48 | 49 | def emoji(self, emoji_code): 50 | self._text_complete += emoji_code 51 | return self 52 | 53 | def to_console(self): 54 | self._text_complete += self.NORMAL 55 | return self._text_complete 56 | -------------------------------------------------------------------------------- /clai/tools/console_helper.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.tools.colorize_console import Colorize 9 | 10 | 11 | def print_complete(text): 12 | print(Colorize() 13 | .complete() 14 | .append(text) 15 | .to_console() 16 | ) 17 | 18 | 19 | def print_error(text): 20 | print(Colorize() 21 | .warning() 22 | .append(text) 23 | .to_console() 24 | ) 25 | -------------------------------------------------------------------------------- /clai/tools/docker_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # pylint: disable=protected-access,broad-except 9 | from time import sleep 10 | 11 | MAX_SIZE_STDOUT = 5000000 12 | 13 | def wait_server_is_started(): 14 | sleep(2) 15 | 16 | 17 | def read(socket, chunk_readed=None): 18 | data = '' 19 | try: 20 | socket.output._sock.recv(1) 21 | while True: 22 | # note that os.read does not work 23 | # because it does not TLS-decrypt 24 | # but returns the low-level encrypted data 25 | # one must use "socket.recv" instead 26 | data_bytes = socket.output._sock.recv(4096) 27 | if not data_bytes: 28 | break 29 | 30 | chunk = data_bytes.decode('utf8', errors='ignore') 31 | if chunk.endswith(']# '): 32 | if data: 33 | break 34 | else: 35 | data += chunk 36 | 37 | if chunk_readed: 38 | chunk_readed(chunk) 39 | 40 | data = data[-MAX_SIZE_STDOUT:] 41 | 42 | except Exception as exception: 43 | print(f'error: {exception}') 44 | return data 45 | 46 | 47 | def execute_cmd(container, command): 48 | socket = container.exec_run(cmd="bash -l", stdin=True, tty=True, privileged=True, socket=True) 49 | 50 | wait_server_is_started() 51 | 52 | command_to_exec = command + '\n' 53 | socket.output._sock.send(command_to_exec.encode()) 54 | 55 | data = read(socket) 56 | 57 | sleep(1) 58 | socket.output._sock.send(b"exit\n") 59 | return str(data) 60 | -------------------------------------------------------------------------------- /clai/tools/process_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import os 9 | 10 | 11 | def check_if_process_running() -> bool: 12 | return os.system('ps -Ao args | grep "[c]lai-run" > /dev/null 2>&1') == 0 13 | -------------------------------------------------------------------------------- /configPlugins.json: -------------------------------------------------------------------------------- 1 | {"selected": {"user": ["nlc2cmd"]}, "default": ["gpt3"], "default_orchestrator": "max_orchestrator", "installed": [], "report_enable": false} 2 | -------------------------------------------------------------------------------- /develop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker-compose run clai bash -c "cd /zclai && python3 develop.py install --path /zclai && bash" 3 | docker-compose rm -f -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | # Filename: docker-compose.yaml 8 | # Purpose: for isolated development 9 | # Usage: `docker-compose run clai bash -c "cd /zclai && python3 develop.py install --path /zclai && bash"` 10 | 11 | version: '3' 12 | services: 13 | clai: 14 | build: 15 | context: . 16 | dockerfile: Dockerfile.Dev 17 | volumes: 18 | - .:/clai 19 | -------------------------------------------------------------------------------- /docs/Installation.md: -------------------------------------------------------------------------------- 1 | ## Installing Docker 2 | 3 | > **Recommended**: Follow the Docker getting started tutorials online [[link](https://docs.docker.com/get-started/)] to familiarize yourself with Docker.  4 | 5 | 6 | ### Installing Docker For MacOS 7 | 1. Download Docker from [Docker Desktop for Mac](https://hub.docker.com/editions/community/docker-ce-desktop-mac). 8 | 2. Sign in to your Docker account or register for a new account. Registration is free and it only requires a valid email address. 9 | 3. Once you have successfully downloaded the `Docker.dmg` installer file, open it by double-clicking on it and then drag the 10 | Moby-the-whale icon to the Application folder. Installation will require system privileges. 11 | 4. Once installed, double-click `Docker.app` in the Applications folder to start Docker. 12 | 13 | ### Installing Docker For Linux 14 | 15 | You need `docker` and `docker.io` installed on your system. 16 | 17 | > For Debian-based systems, type: 18 | ``` 19 | sudo apt-get install docker docker.io 20 | ``` 21 | 22 | > For Red Hat or CentOS based system, type: 23 | ``` 24 | sudo yum install docker-ce 25 | ``` 26 | 27 | > For Fedora-based system (version 22 or earlier), type: 28 | ``` 29 | sudo yum install docker 30 | ``` 31 | 32 | > For Fedora-based system (version 23 or later), type: 33 | ``` 34 | sudo dnf install docker 35 | ``` 36 | 37 | ## Installing Python 3.6 38 | 39 | Python 3.6 (or higher) is required to run CLAI. Download and run the Python installer package for your operating system 40 | from [https://www.python.org/downloads/](https://www.python.org/downloads/). 41 | 42 | ## Installing Homebrew with fswatch for MacOS 43 | 44 | To install Homebrew, execute the following command in your console: 45 | 46 | ``` 47 | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 48 | ``` 49 | 50 | When prompted, press return to continue, and enter your credentials to begin installation. 51 | Once Homebrew is successfully installed, execute the following command to install fswatch. 52 | 53 | ``` 54 | brew install fswatch 55 | ``` 56 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | [`Home`](https://clai-home.mybluemix.net/)   [`More`](Overview.md)   [`API`](../clai/server/plugins/)   [`Community`](http://ibm.biz/clai-slack)   [`FAQs`](FAQ.md)   [`Troubleshooting`](Troubleshooting.md)   [`Installation`](Installation.md)   [`Whitepaper`](https://arxiv.org/abs/2002.00762)   [`Blog`](https://www.ibm.com/blogs/research/2020/02/bringing-ai-to-the-command-line/)   [`Feedback`](http://ibm.biz/clai-survey) 4 | 5 | This directory contains help with installation, troubleshooting, and quicklinks to documentation. 6 | -------------------------------------------------------------------------------- /docs/Troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | ## Known issue with Python and Bash on the CLAI-enabled Shell 4 | 5 | Currently, we do not support going into a recursive Bash session or the interactive python shell from inside a CLAI-enabled terminal. If you type in `>> bash` your current shell window will freeze, and if you move into the python shell, you will not be able to see the stdout and stderr until you come out of it. 6 | 7 | ## Known issue with parentheses in commands 8 | 9 | The CLAI server crashes when commands have parentheses: e.g. `(ls | grep hi) | grep .`. We intend to support more and more complex invocations such as these in future releases. 10 | 11 | ## Installation Error with Homebrew 12 | 13 | ``` 14 | Error: Running Homebrew as root is extremely dangerous and no longer supported. 15 | As Homebrew does not drop privileges on installation you would be giving all 16 | build scripts full access to your system 17 | ``` 18 | 19 | You may not use Homebrew during the installation of any component if it requires sudo access... you will notice this :point_up: error. Brew no longer supports `sudo`. Installation scripts for individual skills do not use sudo and hence do support brew (instructions that require sudo, such as `pip`, need to be invoked with sudo inside the installation script). 20 | 21 | ## Python / Pip not found 22 | 23 | ### Fedora 24 | 25 | Fedora does not have Python3 and Pip3 installed by default. If you do not have Python installed we recomend you execute: 26 | 27 | ```commandline 28 | >> sudo dnf install python37 29 | >> sudo dnf install python3-devel 30 | ``` 31 | 32 | You can test that everything is correctly installed by checking the version with: 33 | 34 | ```commandline 35 | >> python3 -V 36 | Python 3.7.5 37 | >> pip3 -V 38 | pip 19.0.3 from /usr/lib/python3.7/site-packages/pip (python 3.7) 39 | ``` 40 | 41 | Sometimes Pip3 is redirected to a different path and an older version. For fixing that, invoke: 42 | 43 | ```commandline 44 | >> sudo rm -rf 45 | ``` 46 | 47 | ### Ubuntu 48 | 49 | Similar to Fedora, you may need to install Python3 and Pip3 50 | 51 | ```commandline 52 | >> sudo apt update 53 | >> sudo apt install python3.6 54 | >> sudo apt install python3-pip 55 | ``` 56 | -------------------------------------------------------------------------------- /emulator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | python3 -m pip install -r requirements_emulator.txt --ignore-installed 4 | pushd ./bin 5 | export PYTHONPATH=./..:$PYTHONPATH && python3 emulator.py 6 | popd 7 | -------------------------------------------------------------------------------- /known-issues.md: -------------------------------------------------------------------------------- 1 | # Known Issues 2 | 3 | ## Fedora 4 | 5 | Fedora by default doesn't have Python3 and Pip3 installed. And the install process need install pythondev for install some packages. 6 | 7 | If you does not have Python installed we recommend to you execute 8 | 9 | ```commandline 10 | sudo dnf install python37 11 | sudo dnf install python3-devel 12 | ``` 13 | 14 | After that test that everything is correctly installed checking the version with 15 | 16 | ```commandline 17 | >> python3 -V 18 | Python 3.7.5 19 | >> pip3 -V 20 | pip 19.0.3 from /usr/lib/python3.7/site-packages/pip (python 3.7) 21 | ``` 22 | 23 | Sometimes pip3 is redirected to other path and old version. For fixing that invoke 24 | 25 | ```commandline 26 | sudo rm -rf 27 | ``` 28 | 29 | ## z/OS 30 | 31 | There are numerous problems with the (multiple competing) ports of Python to 32 | z/OS. All versions of z/OS Python routinely regress, often introducing, 33 | resolving, then later re-introducing character encoding problems. Furthermore, 34 | an install process that worked on one level of Pip may not work with a 35 | subsequent level of Pip. Its a mess. 36 | 37 | ### "Known Good" Python Configurations 38 | 39 | The best results we've had in installing CLAI on z/OS were with IBM Open 40 | Enterprise Python for z/OS 3.8, using Pip3 >= 20.0.0. 41 | 42 | The following additional configurations are known to work to some degree or 43 | another: 44 | 45 | + IzODA Python 3.6, but only with a backlevel Pip3 (verified with Pip3 v9.0.1) 46 | 47 | With any of the above configurations, you can perform installation using the 48 | instructions in [README.md](README.md), or you can run: 49 | 50 | ```commandline 51 | (bash-4.3)USERID@ZOSYS:~> cd clai 52 | (bash-4.3)USERID@ZOSYS:~/clai> make clean 53 | (bash-4.3)USERID@ZOSYS:~/clai> make install 54 | ``` 55 | 56 | ### Known Problems 57 | 58 | #### General 59 | 60 | + The path to the env program must be `/usr/bin/env` 61 | - If this is not the case, your system administrator will need to create a symbolic link: `ln -s /bin/env /usr/bin/env` 62 | 63 | #### IzODA Python 3.6 64 | 65 | + In order for install/uninstall to work properly your `.bashrc` and 66 | `.bash_profile` files must be tagged either `ISO8859-1` or `IBM-1047` 67 | beforehand 68 | + The `zmsgcode` plugin fails to work due to an SSL validation failure 69 | -------------------------------------------------------------------------------- /my_agent/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /my_agent/manifest.properties: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | name=my agent 3 | description=This is an agent 4 | default=no -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | amplitude-python==0.13 2 | atomicwrites==1.3.0 3 | attrs==19.1.0 4 | importlib-metadata==0.20 5 | more-itertools==7.0.0 6 | numpy==1.17.2; sys_platform != 'zos' 7 | psutil==5.6.6; sys_platform != 'zos' 8 | pydantic==0.27 9 | requests==2.22.0 10 | # spacy==2.2.0.dev18; sys_platform != 'zos' 11 | # https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz#egg=en_core_web_sm; sys_platform != 'zos' 12 | websockets 13 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pylint==2.3.1 2 | isort==4.3.20 3 | mccabe==0.6.1 4 | -------------------------------------------------------------------------------- /requirements_emulator.txt: -------------------------------------------------------------------------------- 1 | pytest-docker-tools==0.2.0 2 | Pillow==7.1.1 -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | elasticsearch==7.0.5 2 | beautifulsoup4 3 | pluggy 4 | pytest==4.6.1 5 | 6 | # pytest "Add ons" 7 | astroid==2.2.5 ; sys_platform != 'zos' 8 | pytest-docker-tools==0.2.0 9 | pytest-mock==1.10.4 10 | -------------------------------------------------------------------------------- /requirements_utilities.txt: -------------------------------------------------------------------------------- 1 | openai -------------------------------------------------------------------------------- /scripts/fileExist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | flags="" 3 | PLUGIN=$1 4 | CLAI_PATH=$2 5 | 6 | # Check for user passed args 7 | while test $# != 0 8 | do 9 | case "$1" in 10 | --user) 11 | USER_INSTALL=true 12 | flags="$flags --user" 13 | ;; 14 | # add more flags here 15 | *) 16 | esac 17 | shift 18 | done 19 | 20 | 21 | install_darwin () { 22 | code=1 23 | pushd $CLAI_PATH/clai/server/plugins/$PLUGIN 24 | if [ -f $CLAI_PATH/clai/server/plugins/$PLUGIN/install_darwin.sh ]; then 25 | eval "sh install_darwin.sh $flags" 26 | code=$? 27 | else 28 | if [ -f $CLAI_PATH/clai/server/plugins/$PLUGIN/install.sh ]; then 29 | eval "sh install.sh $flags" 30 | code=$? 31 | fi 32 | fi 33 | popd 34 | 35 | return $? 36 | } 37 | 38 | install_linux () { 39 | code=1 40 | cd $CLAI_PATH/clai/server/plugins/$PLUGIN 41 | eval "pwd" 42 | if [ -f $CLAI_PATH/clai/server/plugins/$PLUGIN/install_linux.sh ]; then 43 | eval "sh install_linux.sh $flags" 44 | code=$? 45 | else 46 | if [ -f $CLAI_PATH/clai/server/plugins/$PLUGIN/install.sh ]; then 47 | eval "sh install.sh $flags" 48 | code=$? 49 | fi 50 | fi 51 | 52 | return $code 53 | } 54 | 55 | 56 | install_plugin () { 57 | UNAME=$(uname -s) 58 | if [[ "$UNAME" == "Darwin"* ]]; then 59 | install_darwin $PLUGIN 60 | else 61 | install_linux $PLUGIN 62 | fi 63 | return $? 64 | } 65 | 66 | if [ -z "$CLAI_PATH" ]; then 67 | echo "Error: please pass the clai bin path using the --path flag" 68 | exit 1 69 | fi 70 | 71 | install_plugin $PLUGIN 72 | if [[ $? == 0 ]]; then 73 | echo "Installed plugins dependencies $CLAI_PATH/clai/server/plugins/$1/install.sh" 74 | else 75 | echo "The plugin doesn't have dependencies $CLAI_PATH/clai/server/plugins/$1/install.sh" 76 | fi 77 | -------------------------------------------------------------------------------- /scripts/installOrchestrator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | CLAI_PATH=$2 3 | 4 | install_darwin () { 5 | code=1 6 | pushd $CLAI_PATH/clai/server/orchestration/patterns/$1 7 | if [ -f $CLAI_PATH/clai/server/orchestration/patterns/$1/install_darwin.sh ]; then 8 | eval "sh install_darwin.sh" 9 | code=$? 10 | else 11 | if [ -f $CLAI_PATH/clai/server/orchestration/patterns/$1/install.sh ]; then 12 | eval "sh install.sh" 13 | code=$? 14 | fi 15 | fi 16 | popd 17 | 18 | return $? 19 | } 20 | 21 | install_linux () { 22 | code=1 23 | cd $CLAI_PATH/clai/server/orchestration/patterns/$1 24 | eval "pwd" 25 | if [ -f $CLAI_PATH/clai/server/orchestration/patterns/$1/install_linux.sh ]; then 26 | eval "sh install_linux.sh" 27 | code=$? 28 | else 29 | if [ -f $CLAI_PATH/clai/server/orchestration/patterns/$1/install.sh ]; then 30 | eval "sh install.sh" 31 | code=$? 32 | fi 33 | fi 34 | 35 | return $code 36 | } 37 | 38 | install_plugin () { 39 | UNAME=$(uname -s) 40 | if [[ "$UNAME" == "Darwin"* ]]; then 41 | install_darwin $1 42 | else 43 | install_linux $1 44 | fi 45 | return $? 46 | } 47 | 48 | if [ -z "$2" ]; then 49 | echo "Error: please pass the clai bin path as the second argument" 50 | exit 1 51 | fi 52 | 53 | install_plugin $1 54 | if [[ $? == 0 ]]; then 55 | echo "Installed orchestrator dependencies $CLAI_PATH/clai/server/plugins/$1/install.sh" 56 | else 57 | echo "The orchestrator don't have dependencies $CLAI_PATH/clai/server/plugins/$1/install.sh" 58 | fi 59 | -------------------------------------------------------------------------------- /scripts/saveFilesChanges.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval '($CLAI_PATH/bin/fswatchlog "$1" "$2" >/dev/null &) > /dev/null' 4 | 5 | exit 1 -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /test/io_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | import builtins 9 | 10 | 11 | def spy_print(mocker): 12 | return mocker.spy(builtins, 'print') 13 | 14 | 15 | def mock_input_console(mocker, value): 16 | mocker.patch('builtins.input', return_value=value) 17 | -------------------------------------------------------------------------------- /test/mock_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from typing import List 9 | 10 | from clai.server.agent import Agent 11 | from clai.server.agent_datasource_executor import AgentDatasourceExecutor 12 | from clai.server.agent_executor import AgentExecutor 13 | from clai.server.command_message import State, Action 14 | 15 | 16 | # pylint: disable=too-few-public-methods 17 | class MockExecutor(AgentExecutor): 18 | def execute_agents(self, command: State, agents: List[Agent]) -> List[Action]: 19 | return list(map(lambda agent: self.execute(command, agent), agents)) 20 | 21 | @staticmethod 22 | def execute(command: State, agent: Agent) -> Action: 23 | action = agent.execute(command) 24 | if not action: 25 | action = Action() 26 | return action 27 | 28 | 29 | class MockAgentDatasourceExecutor(AgentDatasourceExecutor): 30 | def execute(self, load_agent, pkg_name): 31 | load_agent(pkg_name) 32 | -------------------------------------------------------------------------------- /test/sample_zcmds/mvs.txt: -------------------------------------------------------------------------------- 1 | DISPLAY 2 | D R,R 3 | D A,L 4 | D A,L,USERID=userid 5 | D A,INIT* 6 | D TS,L 7 | D J,L 8 | D J,L,USERID=userid 9 | 10 | D U,,OFFLINE 11 | D U,,ONLINE 12 | D U,,ALLOC 13 | D U,IPLVOL 14 | D U,DASD,ONLINE 15 | D U,VOL=volser 16 | D U,,,3333,10 17 | D U,,ALLOC,/3333 18 | 19 | D IPLINFO 20 | D IPLINFO,COUPLE 21 | D IPLINFO,DIAG 22 | D IPLINFO,GRS 23 | D IPLINFO,GRS,GRSCNF 24 | D IPLINFO,GRS,GRSRNL 25 | D IPLINFO,ZAAPZIIP 26 | D IPLINFO,ZAAPZIIP,STATE 27 | D PARMLIB 28 | D PARMLIB,ERRORS 29 | D PROD,REG 30 | D M=(CPU,DEV,HIGH,STOR) 31 | D M=STOR(E) 32 | D LLA 33 | D PROG,APF 34 | D PROG,LNKLST 35 | D PROG,LPA 36 | 37 | D GRS,C 38 | D GRS,DELAY 39 | D GRS,ENQ,C 40 | D GRS,LATCH,C 41 | 42 | D XCF,POLICY,TYPE=CFRM 43 | D XCF,COUPLE 44 | D XCF,COUPLE,TYPE=CFRM 45 | D XCF,CF,CFNAME=cfname 46 | D XCF,STR,STRNAME=strname 47 | 48 | D CF,CFNAME=cfname 49 | D WLM 50 | D WLM,SYSTEM=sysname 51 | D WLM,SYSTEMS 52 | D OMVS,O 53 | D OMVS,P 54 | D OMVS,L 55 | D OMVS,SER 56 | D OMVS,W 57 | 58 | D SLIP 59 | D SLIP=id 60 | D SLIP,PER 61 | D TRACE,COMP=ALL 62 | D TRACE,WTR=ALL 63 | D TRACE,TT 64 | 65 | D HIS 66 | D SMF,S 67 | D SMF,O 68 | D SMF,M 69 | D SMS,OPTIONS 70 | D SMS,LOG(ALL) 71 | D SMS,TRANVSAM 72 | D SMS,TRANVSAM,ALL 73 | D SMS,SMSVSAM 74 | D SMS,SMSVSAM,ALL 75 | 76 | D LOGREC,ALL 77 | D LOGGER,STATUS 78 | D LOGGER,L 79 | D LOGGER,CONN,SYSPLEX 80 | 81 | $DJ1-32767 82 | $D SPOOL 83 | $DA 84 | $DA,ALL 85 | $DI,1-9999 86 | 87 | SEND 88 | SE 'msg',USER=(userid) 89 | 90 | REPLY 91 | R x,resp 92 | 93 | CANCEL 94 | C U=userid 95 | C jobname 96 | C started task 97 | MODIFY 98 | F name,command 99 | START 100 | S name 101 | STOP 102 | P name 103 | FORCE 104 | FORCE name or userid 105 | 106 | DUMP 107 | DUMPDS 108 | SLIP 109 | SLIP SET,ID=id,..... 110 | TRACE 111 | 112 | CONFIG 113 | SET 114 | SETETR 115 | SETPROG 116 | SETSMF 117 | SETSMS 118 | SETXCF 119 | VARY 120 | V xxxx,ONLINE 121 | V xxxx,OFFLINE 122 | V SMS,SMSVSAM,MONDS(data set name) 123 | MOUNT 124 | UNLOAD 125 | 126 | CONTROL 127 | RESET 128 | ROUTE -------------------------------------------------------------------------------- /test/sample_zcmds/tso.txt: -------------------------------------------------------------------------------- 1 | DISPLAY 2 | TSO LISTCAT LVL(userid.foo) 3 | TSO ISPVCALL STATUS 4 | TSO LISTALC 5 | TSO LISTDS dsname or partial 6 | TSO LISTBC 7 | TSO LISTBC NOMAIL 8 | TSO HELP 9 | TSO HELP ALLOCATE 10 | TSO STATUS 11 | TSO LISTUSER (userid) 12 | TSO LISTGRP * | (groupname) 13 | TSO LISTDSD DATASET(dsname) 14 | TSO HELP LISTUSER 15 | 16 | SEND 17 | TSO SEND "hello operator" 18 | TSO SEND "hello user" USER(userid) LOGON 19 | 20 | EXEC 21 | TSO EXEC "dsn(mbr)" "parm1" 22 | 23 | LINK 24 | 25 | CALL 26 | 27 | SUBMIT 28 | TSO SUBMIT (ds) 29 | 30 | CANCEL 31 | TSO CANCEL (jobname) 32 | 33 | ALLOCATE 34 | TSO ALLOCATE a new ds 35 | TSO ALLOCATE ds to a dd 36 | 37 | FREE 38 | TSO FREE(da) 39 | 40 | EDIT 41 | TSO EDIT 42 | 43 | COPY 44 | TSO SMCOPY FROMDATASET(dsn) TODATASET(dsn) 45 | 46 | RENAME 47 | TSO RENAME oldds newds 48 | 49 | DELETE 50 | DELETE dsn SCRATCH NONVSAM 51 | 52 | TRANSMIT 53 | 54 | RECEIVE 55 | 56 | TSOLIB 57 | 58 | ALTLIB -------------------------------------------------------------------------------- /test/sample_zcmds/uss.txt: -------------------------------------------------------------------------------- 1 | DISPLAY 2 | ls 3 | ls -al 4 | ls -al 5 | df -P 6 | df -S 7 | df -v 8 | du 9 | du 10 | cd pathname 11 | pwd 12 | dircmp pathname1 pathname2 13 | pathchk pathname 14 | cmp filename1 filename2 15 | diff filename1 filename2 16 | chtag -p * 17 | 18 | find / -name "*.py" 19 | grep -i "string" * 20 | egrep use grep -E 21 | fgrep use grep -F 22 | 23 | file filename 24 | cat filename 25 | head -n 20 filename 26 | tail -n 15 filename 27 | more filename 28 | date 29 | echo "string" 30 | man ls 31 | 32 | uptime 33 | uname -rsv 34 | ps -ef 35 | top 36 | jobs 37 | who 38 | whoami 39 | id 40 | zlsof -d 41 | 42 | tso tsocmd 43 | tsocmd authtsocmd 44 | 45 | SEND 46 | talk - not useful bc 2 sided setup 47 | 48 | EDIT 49 | oedit fn.ft 50 | obrowse fn.ft 51 | vi fn.ft 52 | 53 | HELP 54 | help ls 55 | man ls 56 | 57 | SUBMIT 58 | submit fn.ft 59 | 60 | COPY 61 | cp 62 | 63 | RENAME 64 | mv 65 | 66 | DELETE 67 | rm 68 | rmdir 69 | unmount 70 | 71 | CREATE 72 | mkdir 73 | mount 74 | 75 | OTHER 76 | docker --version -------------------------------------------------------------------------------- /test/state_mother.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.server.command_message import State 9 | 10 | ANY_ID = '1' 11 | ANY_INPUT_COMMAND = "command" 12 | ANY_NAME = 'any_name' 13 | ANY_ROUTE = './any_folder' 14 | ANY_URL = 'http://ipv4.download.thinkbroadband.com/5MB.zip' 15 | 16 | 17 | def clai_plugins_state(): 18 | return State( 19 | command_id=ANY_ID, 20 | user_name=ANY_NAME, 21 | command="clai skills" 22 | ) 23 | 24 | 25 | def clai_power_state(): 26 | return State( 27 | command_id=ANY_ID, 28 | user_name=ANY_NAME, 29 | command="clai auto" 30 | ) 31 | 32 | 33 | def clai_power_disabled_state(): 34 | return State( 35 | command_id=ANY_ID, 36 | user_name=ANY_NAME, 37 | command="clai manual" 38 | ) 39 | 40 | 41 | ANY_COMMAND_MESSAGE = State( 42 | command_id=ANY_ID, 43 | user_name=ANY_NAME, 44 | command=ANY_INPUT_COMMAND 45 | ) 46 | 47 | 48 | def command_state(): 49 | return State( 50 | command_id=ANY_ID, 51 | user_name=ANY_NAME, 52 | command="command" 53 | ) 54 | 55 | 56 | COMMAND_AGENT_STATE = State( 57 | command_id=ANY_ID, 58 | user_name=ANY_NAME, 59 | command="clai command" 60 | ) 61 | 62 | COMMAND_NAME_AGENT_STATE = State( 63 | command_id=ANY_ID, 64 | user_name=ANY_NAME, 65 | command='clai "demo agent" command' 66 | ) 67 | 68 | CLAI_INSTALL_STATE_FOLDER = State( 69 | command_id=ANY_ID, 70 | user_name=ANY_NAME, 71 | command=f"clai install {ANY_ROUTE}" 72 | ) 73 | 74 | CLAI_INSTALL_STATE_URL = State( 75 | command_id=ANY_ID, 76 | user_name=ANY_NAME, 77 | command=f"clai install {ANY_URL}" 78 | ) 79 | 80 | 81 | def clai_select_state(plugin_to_select): 82 | return State( 83 | command_id=ANY_ID, 84 | user_name=ANY_NAME, 85 | command=f"clai activate {plugin_to_select}" 86 | ) 87 | 88 | 89 | def clai_unselect_state(plugin_to_select): 90 | return State( 91 | command_id=ANY_ID, 92 | user_name=ANY_NAME, 93 | command=f"clai unselect {plugin_to_select}" 94 | ) 95 | -------------------------------------------------------------------------------- /test/test_restore_history.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from clai.restore_history import __remove_clai_history__ 9 | 10 | 11 | def test_should_remove_when_not_commands(): 12 | lines = [] 13 | 14 | new_lines = __remove_clai_history__(lines, "ls") 15 | 16 | assert new_lines == [] 17 | 18 | 19 | def test_should_remove_the_list_if_only_one_command(): 20 | lines = ["ls"] 21 | 22 | new_lines = __remove_clai_history__(lines, "ls") 23 | 24 | assert new_lines == ["ls"] 25 | 26 | 27 | def test_should_remove_last_command_if_have_two_repeat(): 28 | lines = ["pwd", "ls", "ls"] 29 | 30 | new_lines = __remove_clai_history__(lines, "ls") 31 | 32 | assert new_lines == ["pwd", "ls"] 33 | 34 | 35 | def test_should_remove_nothing_if_have_only_the_command(): 36 | lines = ["pwd", "ls"] 37 | 38 | new_lines = __remove_clai_history__(lines, "ls") 39 | 40 | assert new_lines == ["pwd", "ls"] 41 | 42 | 43 | def test_should_remove_dirty_until_the_command(): 44 | lines = ["pwd", "ls", "pwd", "cd"] 45 | 46 | new_lines = __remove_clai_history__(lines, "ls") 47 | 48 | assert new_lines == ["pwd", "ls"] 49 | 50 | 51 | def test_should_not_remove_if_command_not_exist(): 52 | lines = ["pwd", "ls"] 53 | 54 | new_lines = __remove_clai_history__(lines, "ls -la") 55 | 56 | assert new_lines == ["pwd", "ls"] 57 | 58 | 59 | def test_should_remove_dirt_if_it_is_the_first_comment(): 60 | lines = ["clai", ":"] 61 | 62 | new_lines = __remove_clai_history__(lines, "clai") 63 | 64 | assert new_lines == ["clai"] 65 | -------------------------------------------------------------------------------- /test_integration/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | -------------------------------------------------------------------------------- /test_integration/conftest.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # pylint: disable=invalid-name,no-value-for-parameter 9 | import os 10 | 11 | from pytest_docker_tools import build, container 12 | 13 | 14 | def get_base_path(): 15 | root_path = os.getcwd() 16 | print(root_path) 17 | if 'test_integration' in root_path: 18 | return '../' 19 | 20 | return '.' 21 | 22 | 23 | my_clai_installed_image = build( 24 | path=get_base_path(), 25 | dockerfile='./test_integration/docker/centos/Dockerfile', 26 | rm=True 27 | ) 28 | 29 | my_clai_module = container( 30 | image='{my_clai_installed_image.id}', 31 | scope='module' 32 | ) 33 | 34 | 35 | def pytest_generate_tests(metafunc): 36 | if "command" in metafunc.fixturenames: 37 | commands = getattr(metafunc.cls, 'get_commands_to_execute')(metafunc.cls) 38 | commands_expected = getattr(metafunc.cls, 'get_commands_expected')(metafunc.cls) 39 | metafunc.parametrize(["command", 'command_expected'], list(zip(commands, commands_expected))) 40 | -------------------------------------------------------------------------------- /test_integration/contract_skills.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # pylint: disable=no-self-use 9 | import pytest 10 | 11 | from clai.tools.docker_utils import execute_cmd 12 | 13 | 14 | class ContractSkills: 15 | def is_auto_mode(self): 16 | return True 17 | 18 | def get_skill_name(self): 19 | raise NotImplementedError('You should provide the commands to execute.') 20 | 21 | def get_commands_to_execute(self): 22 | raise NotImplementedError('You should provide the commands to execute.') 23 | 24 | def get_commands_expected(self): 25 | raise NotImplementedError('You should provide the commands expected.') 26 | 27 | @pytest.mark.dependency() 28 | def test_install(self, my_clai_module): 29 | if self.is_auto_mode(): 30 | execute_cmd(my_clai_module, "clai auto") 31 | skill_name = self.get_skill_name() 32 | 33 | execute_cmd(my_clai_module, "clai deactivate gpt3") 34 | 35 | command_select = f"clai activate {skill_name}" 36 | command_executed = execute_cmd(my_clai_module, command_select) 37 | 38 | assert f"\x1b[32m {skill_name} (Installed)" in command_executed, \ 39 | f'Skill {skill_name} not found installed. Output: {command_executed}' 40 | 41 | @pytest.mark.dependency(depends=['test_install']) 42 | def test_skill_values(self, my_clai_module, command, command_expected): 43 | command_executed = execute_cmd(my_clai_module, command) 44 | assert command_expected in command_executed, \ 45 | f'Expected: {command_expected}, Received: {command_executed}' 46 | -------------------------------------------------------------------------------- /test_integration/test_skill_fixit.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from test_integration.contract_skills import ContractSkills 9 | 10 | 11 | class TestSkillFixit(ContractSkills): 12 | 13 | def get_skill_name(self): 14 | return 'fixit' 15 | 16 | def get_commands_to_execute(self): 17 | return ['clean', 'puthon'] 18 | 19 | def get_commands_expected(self): 20 | return ['clear', 'python'] 21 | -------------------------------------------------------------------------------- /test_integration/test_skill_helpme.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from test_integration.contract_skills import ContractSkills 9 | 10 | 11 | class TestSkillHelpme(ContractSkills): 12 | 13 | def get_skill_name(self): 14 | return 'helpme' 15 | 16 | def get_commands_to_execute(self): 17 | return ['pwd', 'apt-get install emacs'] 18 | 19 | def get_commands_expected(self): 20 | return ['/opt/IBM/clai', 'apt'] 21 | -------------------------------------------------------------------------------- /test_integration/test_skill_howdoi.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from test_integration.contract_skills import ContractSkills 9 | 10 | 11 | class TestSkillHowDoI(ContractSkills): 12 | 13 | def is_auto_mode(self): 14 | return False 15 | 16 | def get_skill_name(self): 17 | return 'howdoi' 18 | 19 | def get_commands_to_execute(self): 20 | return ['pwd'] #['pwd', 'clai "howdoi" when to use sudo vs su?', 21 | #'clai "howdoi" find out disk usage per user?', 22 | #'clai "howdoi" How to process gz files?'] 23 | 24 | def get_commands_expected(self): 25 | return ['/opt/IBM/clai'] #, 'su', 'df', 'gzip'] 26 | -------------------------------------------------------------------------------- /test_integration/test_skill_manpageexplorer.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from test_integration.contract_skills import ContractSkills 9 | 10 | 11 | class TestSkillManPageExplorer(ContractSkills): 12 | 13 | def is_auto_mode(self): 14 | return False 15 | 16 | def get_skill_name(self): 17 | return 'man page explorer' 18 | 19 | def get_commands_to_execute(self): 20 | return ['pwd'] #, 'clear screen?'] 21 | 22 | def get_commands_expected(self): 23 | return ['/opt/IBM/clai'] #, 'man clear'] 24 | -------------------------------------------------------------------------------- /test_integration/test_skill_nlc2cmd.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from test_integration.contract_skills import ContractSkills 9 | 10 | 11 | class TestSkillNlc2cmd(ContractSkills): 12 | 13 | def get_skill_name(self): 14 | return 'nlc2cmd' 15 | 16 | def get_commands_to_execute(self): 17 | return ['show me how to add tags for ibmcloud', 18 | 'show billing costs', 19 | 'copy file to PDS member', 20 | 'view mvs file', 21 | 'extract files from an archive', 22 | 'grep for all files with "clai" in this directory, show me details and line numbers'] 23 | 24 | def get_commands_expected(self): 25 | return ['Try >> ibmcloud resource tag-attach --tag-names TAG --resource-name NAME', 26 | 'Try >> ibmcloud billing account-usage [-d YYYY-MM]', 27 | 'Try >> OGET [pathname] mvs_data_set_name(member_name)', 28 | 'Try >> obrowse -r xx [file]', 29 | 'Try >> tar -xf ', 30 | 'Try >> grep -rnv "clai" '] 31 | -------------------------------------------------------------------------------- /test_integration/test_skill_tellina.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | from test_integration.contract_skills import ContractSkills 9 | 10 | 11 | class TestSkillTellina(ContractSkills): 12 | 13 | def get_skill_name(self): 14 | return 'tellina' 15 | 16 | def get_commands_to_execute(self): 17 | return ['pwd', 18 | 'clai "tellina" exit terminal', 19 | 'clai "tellina" show me all files'] 20 | 21 | def get_commands_expected(self): 22 | return ['/opt/IBM/clai', 23 | 'exit', 24 | 'find .'] 25 | -------------------------------------------------------------------------------- /uninstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | flags="" 3 | 4 | # Check for user passed args 5 | while test $# != 0 6 | do 7 | case "$1" in 8 | --user) 9 | USER_INSTALL=true 10 | flags="$flags --user" 11 | ;; 12 | # add more flags here 13 | *) flags="$flags $1" 14 | esac 15 | shift 16 | done 17 | 18 | die () { 19 | echo -e $1; 20 | exit $2; 21 | } 22 | 23 | is_sh () { 24 | value=$(ps -o args= -p "$$" | cut -f 1 -d " ") 25 | test "sh" = $value 26 | return 27 | } 28 | 29 | command_exists () { 30 | type "$1" &> /dev/null ; 31 | } 32 | 33 | if is_sh ; then 34 | die "\n Please don't invoke with sh, to uninstall use ./uninstall.sh" 35 | fi 36 | 37 | if [ "$USER_INSTALL" != true ]; then 38 | if [ "$EUID" -ne 0 ]; then 39 | die "\n Please run as sudo." 1 40 | fi 41 | fi 42 | 43 | if ! command_exists python3 ; then 44 | die "\n Sorry you need to have Python3 installed. Please install it and rerun this script." 1 45 | fi 46 | 47 | if ps -Ao args -u `whoami` | grep "[c]lai-run" &> /dev/null ; then 48 | if [ ! $(uname) == 'OS/390' ] && [ ! $(uname) == 'z/OS' ]; then 49 | running_process=$(ps -Ao args -u $(whoami) | grep "[c]lai-run" | head -1) 50 | pkill -f "${running_process}" 51 | else 52 | clai_pid=$(ps -e -o pid,args -u `whoami` | grep "[c]lai-run" | head -1 | awk '{print $1}') 53 | clai_subpids=$(ps -e -o pid,ppid -u `whoami` | awk '$2=='"$clai_pid"'' | awk '{printf "%s ",$1}') 54 | kill -9 $clai_pid $clai_subpids 55 | fi 56 | fi 57 | 58 | eval "python3 uninstall.py $flags" 59 | -------------------------------------------------------------------------------- /usersInstalled.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /utils/.bash_profile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # Sample .bash_profile file for a z/OS USS ID that will run CLAI 9 | 10 | # Invokes .bashrc 11 | if [ -f $HOME/.bashrc ] ; then 12 | source $HOME/.bashrc 13 | fi 14 | # EOF 15 | -------------------------------------------------------------------------------- /utils/.bashrc: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # Sample .bashrc file for a z/OS USS ID that will run CLAI 9 | # 10 | # Instructions: set the following environment variables as is appropriate for 11 | # your system configuration. If nothing else, you definitely need to change the 12 | # value of PYTHON_HOME. 13 | 14 | # Respect file tags for ASCII/EBCDIC stuff. 15 | export _BPXK_AUTOCVT=ON 16 | export _CEE_RUNOPTS="FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)" 17 | 18 | # Uncomment this if you need to specify the name of a particular TCP/IP stack to 19 | # be used; needed when your system configuration uses multiple TCP/IP stacks. 20 | #export _BPXK_SETIBMOPT_TRANSPORT=TCP342 21 | 22 | # Configure Python 23 | export PYTHON_HOME=/path/to/python 24 | export LIBPATH=$PYTHON_HOME/lib:$LIBPATH 25 | export FFI_LIB=$PYTHON_HOME/lib/ffi 26 | export MANPATH=$PYTHON_HOME/share/man:$MANPATH 27 | export PKG_CONFIG_PATH=$PYTHON_HOME/lib/pkgconfig:$PYTHON_HOME/share/pkgconfig 28 | export CURL_CA_BUNDLE=$PYTHON_HOME/etc/ssl/cacert.pem 29 | 30 | # Configure PATH environment variable 31 | export PATH=$HOME/.local/bin:$PYTHON_HOME/bin:$PATH 32 | 33 | # Uncomment the following line to set the TCP/IP port that CLAI uses to some 34 | # value other than the default. Replace "8xxx" with a unique port number. 35 | #export CLAI_PORT=8xxx 36 | 37 | # Uncomment the following line if you want to write CLAI logfile to 38 | # /tmp/USERID instead of the default logfile path 39 | #export CLAI_LOG_FILE="/tmp/$(whoami | tr '[:upper:]' '[:lower:]').log" 40 | -------------------------------------------------------------------------------- /utils/.profile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2020 IBM. All Rights Reserved. 3 | # 4 | # See LICENSE.txt file in the root directory 5 | # of this source tree for licensing information. 6 | # 7 | 8 | # Sample bash profile for a z/OS USS ID that will run CLAI 9 | # 10 | # Instructions: Uncomment the `export SHELL` statement that is appropriate for 11 | # your system configuration 12 | 13 | export SHELL=/bin/bash 14 | #export SHELL=/local/bash 15 | #export SHELL=/Voyager/anaconda/bin/bash 16 | #export SHELL=/usr/lpp/IBM/izoda/anaconda/bin/bash 17 | exec $SHELL --login -------------------------------------------------------------------------------- /utils/README.md: -------------------------------------------------------------------------------- 1 | # Profile setup: 2 | 3 | Copy the _.profile_, _.bashrc_, and _.bash_profile_ files into your home 4 | directory on z/OS in the USS subsystem. 5 | -------------------------------------------------------------------------------- /utils/getMakeType.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2020 IBM. All Rights Reserved. 4 | # 5 | # See LICENSE.txt file in the root directory 6 | # of this source tree for licensing information. 7 | # 8 | 9 | read arg 10 | makestr=$($arg -V 2> /dev/null) 11 | 12 | if [ -n "$makestr" ]; then 13 | echo "Makefile.uss" 14 | else 15 | echo "Makefile.gnu" 16 | fi 17 | 18 | exit 0 -------------------------------------------------------------------------------- /utils/getVersionInfo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2020 IBM. All Rights Reserved. 4 | # 5 | # See LICENSE.txt file in the root directory 6 | # of this source tree for licensing information. 7 | # 8 | 9 | read versionFile fieldName 10 | cat $versionFile | grep $fieldName | awk '{print $3}' | sed 's/\"//g' 11 | exit 0 --------------------------------------------------------------------------------