├── .codeclimate.yml ├── .coveragerc ├── .dockerignore ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── AUTHORS ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.test ├── LICENSE ├── MAINTAINERS ├── Makefile ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── bin └── vent ├── docs ├── img │ └── vent-logo.png └── source │ ├── build_plugin.rst │ ├── conf.py │ ├── contributing.rst │ ├── core_and_plugins.rst │ ├── index.rst │ ├── initialization.rst │ ├── modules.rst │ ├── new_release.rst │ ├── plugin_config.rst │ ├── quickstart.rst │ ├── system_command.rst │ ├── troubleshoot.rst │ ├── user_data.rst │ ├── vent.api.rst │ ├── vent.core.file_drop.rst │ ├── vent.core.network_tap.ncontrol.rst │ ├── vent.core.network_tap.rst │ ├── vent.core.rmq_es_connector.rst │ ├── vent.core.rq_dashboard.rst │ ├── vent.core.rq_worker.rst │ ├── vent.core.rst │ ├── vent.helpers.rst │ ├── vent.menus.rst │ └── vent.rst ├── healthcheck ├── hc.py └── requirements.txt ├── renovate.json ├── setup.py ├── test-requirements.txt ├── tests ├── __init__.py ├── menu │ ├── __init__.py │ └── test_menu.py └── unit │ ├── __init__.py │ ├── test_api_image.py │ ├── test_api_repository.py │ ├── test_api_system.py │ ├── test_api_tools.py │ ├── test_helpers_meta.py │ ├── test_helpers_paths.py │ └── test_helpers_templates.py └── vent ├── __init__.py ├── api ├── __init__.py ├── image.py ├── repository.py ├── system.py └── tools.py ├── core ├── __init__.py ├── file_drop │ ├── Dockerfile │ ├── __init__.py │ ├── file_drop.py │ ├── healthcheck │ │ ├── hc.py │ │ └── requirements.txt │ ├── requirements.txt │ ├── test_file_drop.py │ └── vent.template ├── rabbitmq │ ├── Dockerfile │ └── vent.template ├── redis │ ├── Dockerfile │ └── vent.template ├── rq_worker │ ├── Dockerfile │ ├── __init__.py │ ├── healthcheck │ │ ├── hc.py │ │ └── requirements.txt │ ├── requirements.txt │ ├── settings.py │ ├── test_watch.py │ ├── vent.template │ └── watch.py └── syslog │ ├── Dockerfile │ ├── syslog-ng.conf │ └── vent.template ├── extras ├── elasticsearch │ ├── Dockerfile │ └── vent.template ├── gonet │ ├── Dockerfile │ └── main.go ├── network_tap │ ├── Dockerfile │ ├── README.md │ ├── __init__.py │ ├── healthcheck │ │ ├── hc.py │ │ └── requirements.txt │ ├── ncapture │ │ ├── Dockerfile │ │ ├── VERSION │ │ ├── requirements.txt │ │ ├── run.sh │ │ ├── send_message.py │ │ ├── test_ncapture.sh │ │ └── vent.template │ ├── ncontrol │ │ ├── __init__.py │ │ ├── ncontrol.py │ │ ├── paths.py │ │ ├── requirements.txt │ │ ├── routes.py │ │ └── test_ncontrol.py │ ├── startup.sh │ └── vent.template ├── rmq_es_connector │ ├── Dockerfile │ ├── __init__.py │ ├── healthcheck │ │ ├── hc.py │ │ └── requirements.txt │ ├── requirements.txt │ ├── rmq_es_connector.py │ ├── test_rmq_es_connector.py │ └── vent.template └── rq_dashboard │ ├── Dockerfile │ ├── __init__.py │ ├── healthcheck │ ├── hc.py │ └── requirements.txt │ ├── rq_dash_settings.py │ ├── run.sh │ ├── test_rq_dashboard.py │ └── vent.template ├── helpers ├── __init__.py ├── errors.py ├── logs.py ├── meta.py ├── paths.py └── templates.py ├── menu.py └── menus ├── __init__.py ├── add.py ├── add_options.py ├── backup.py ├── choose_tools.py ├── del_instances.py ├── editor.py ├── help.py ├── inventory.py ├── inventory_forms.py ├── main.py ├── services.py ├── tools.py ├── tutorial_forms.py └── tutorials.py /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | pep8: 4 | enabled: true 5 | duplication: 6 | enabled: true 7 | config: 8 | languages: 9 | - python 10 | fixme: 11 | enabled: true 12 | radon: 13 | enabled: true 14 | ratings: 15 | paths: 16 | - "**.py" 17 | exclude_paths: 18 | - docs/ 19 | - tests/ 20 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = setup.py 3 | */conftest.py 4 | */suplemon/* 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.gz 3 | *.iso 4 | *.swp 5 | *.pyc 6 | *.tar 7 | *.zip 8 | .cache 9 | .DS_Store 10 | .git 11 | .vent 12 | build 13 | core 14 | dist 15 | images 16 | plugins 17 | tests/__pycache__ 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .eggs/ 2 | .tox/ 3 | *.egg-info 4 | *.bkp 5 | *.bak 6 | *.gz 7 | *.iso 8 | *.pyc 9 | *.swo 10 | *.swp 11 | *.tar 12 | *.zip 13 | .cache 14 | .coverage 15 | .DS_Store 16 | .pytest_cache 17 | .vent 18 | /build/ 19 | dist 20 | images 21 | plugins 22 | tests/__pycache__ 23 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.0.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: check-case-conflict 10 | - id: check-json 11 | - id: pretty-format-json 12 | args: ['--autofix'] 13 | - id: double-quote-string-fixer 14 | - id: check-yaml 15 | - repo: https://github.com/asottile/reorder_python_imports 16 | rev: v1.3.4 17 | hooks: 18 | - id: reorder-python-imports 19 | - repo: https://github.com/pre-commit/mirrors-autopep8 20 | rev: v1.4.3 21 | hooks: 22 | - id: autopep8 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | 3 | services: 4 | - redis 5 | - elasticsearch 6 | language: python 7 | cache: pip 8 | python: 9 | - "3.6" 10 | before_script: 11 | - sleep 10 12 | - echo -e "https://github.com/cyberreboot/vent:\n rabbitmq:" >> ~/.vent_startup.yml 13 | before_install: 14 | - sudo apt-get update 15 | - sudo apt-get install docker-ce 16 | jobs: 17 | include: 18 | - stage: test 19 | script: make test-menu && bash <(curl -s https://codecov.io/bash) 20 | - script: make test-unit && bash <(curl -s https://codecov.io/bash) 21 | - script: make test-plugins && bash <(curl -s https://codecov.io/bash) 22 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This file lists all individuals having contributed content to the repository. 2 | # 3 | # For a list of active project maintainers, see the MAINTAINERS file. 4 | # 5 | Alice Chang 6 | Arpit Solanki 7 | arpit1997 8 | Arun Pattni 9 | blake 10 | bpagon 11 | bpagon13 12 | bpagon13 13 | Cece Hedrick 14 | cglewis 15 | Charlie Lewis 16 | Charlie Lewis 17 | Chiraag Prafullchandra 18 | Chiraag Prafullchandra 19 | Chiraag Prafullchandra 20 | Cody 21 | cominixo01 22 | Danny Padilla 23 | Edgard Mota 24 | gb-hacktoberfest <> 25 | gb-hacktoberfest 26 | James Day 27 | Jeff Wang 28 | Jeff Wang 29 | Jeffrey Wang 30 | JJ 31 | JJ Ben-Joseph 32 | Joe Wonohadidjojo 33 | Joe Wonohadidjojo 34 | Joecakes4u 35 | Klaudiusz Dembler 36 | lilchurro 37 | lilchurro 38 | root 39 | rashley-iqt 40 | Samuel Gaist 41 | Shaurya 42 | The Codacy Badger 43 | the-zebulan 44 | Tranquilled 45 | Usman Iqbal 46 | Wonohadidjojo, Joe 47 | xaver.fabian 48 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at clewis@iqt.org. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Vent 2 | 3 | Want to hack on Vent? Awesome! Here are instructions to get you started. 4 | They are probably not perfect, please let us know if anything feels 5 | wrong or incomplete. 6 | 7 | ## Contribution guidelines 8 | 9 | ### Pull requests are always welcome 10 | 11 | We are always thrilled to receive pull requests, and do our best to 12 | process them as fast as possible. Not sure if that typo is worth a pull 13 | request? Do it! We will appreciate it. 14 | 15 | If your pull request is not accepted on the first try, don't be 16 | discouraged! If there's a problem with the implementation, hopefully you 17 | received feedback on what to improve. 18 | 19 | We're trying very hard to keep Vent lean and focused. We don't want it 20 | to do everything for everybody. This means that we might decide against 21 | incorporating a new feature. However, there might be a way to implement 22 | that feature *on top of* Vent. 23 | 24 | ### Create issues... 25 | 26 | Any significant improvement should be documented as [a github 27 | issue](https://github.com/CyberReboot/vent/issues) before anybody 28 | starts working on it. 29 | 30 | ### ...but check for existing issues first! 31 | 32 | Please take a moment to check that an issue doesn't already exist 33 | documenting your bug report or improvement proposal. If it does, it 34 | never hurts to add a quick "+1" or "I have this problem too". This will 35 | help prioritize the most common problems and requests. 36 | 37 | ### Conventions 38 | 39 | Fork the repo and make changes on your fork in a feature branch. 40 | 41 | Make sure you include relevant updates or additions to documentation and 42 | tests when creating or modifying features. 43 | 44 | Pull requests descriptions should be as clear as possible and include a 45 | reference to all the issues that they address. 46 | 47 | Code review comments may be added to your pull request. Discuss, then make the 48 | suggested modifications and push additional commits to your feature branch. Be 49 | sure to post a comment after pushing. The new commits will show up in the pull 50 | request automatically, but the reviewers will not be notified unless you 51 | comment. 52 | 53 | Before the pull request is merged, make sure that you squash your commits into 54 | logical units of work using `git rebase -i` and `git push -f`. After every 55 | commit the test suite should be passing. Include documentation changes in the 56 | same commit so that a revert would remove all traces of the feature or fix. 57 | 58 | Commits that fix or close an issue should include a reference like `Closes #XXX` 59 | or `Fixes #XXX`, which will automatically close the issue when merged. 60 | 61 | Add your name to the AUTHORS file, but make sure the list is sorted and your 62 | name and email address match your git configuration. The AUTHORS file is 63 | regenerated occasionally from the git commit history, so a mismatch may result 64 | in your changes being overwritten. 65 | 66 | ## Decision process 67 | 68 | ### How are decisions made? 69 | 70 | Short answer: with pull requests to the Vent repository. 71 | 72 | All decisions affecting Vent, big and small, follow the same 3 steps: 73 | 74 | * Step 1: Open a pull request. Anyone can do this. 75 | 76 | * Step 2: Discuss the pull request. Anyone can do this. 77 | 78 | * Step 3: Accept or refuse a pull request. A maintainer does this. 79 | 80 | 81 | ### How can I become a maintainer? 82 | 83 | * Step 1: learn the code inside out 84 | * Step 2: make yourself useful by contributing code, bugfixes, support etc. 85 | 86 | Don't forget: being a maintainer is a time investment. Make sure you will have time to make yourself available. 87 | You don't have to be a maintainer to make a difference on the project! 88 | 89 | ### What are a maintainer's responsibility? 90 | 91 | It is every maintainer's responsibility to: 92 | 93 | * 1) Deliver prompt feedback and decisions on pull requests. 94 | * 2) Be available to anyone with questions, bug reports, criticism etc. on Vent. 95 | 96 | ### How is this process changed? 97 | 98 | Just like everything else: by making a pull request :) 99 | 100 | *Derivative work from [Moby](https://github.com/moby/moby/blob/master/CONTRIBUTING.md).* 101 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " 3 | 4 | RUN apk --no-cache add --update \ 5 | curl \ 6 | git \ 7 | python3 \ 8 | py3-pip && \ 9 | pip3 install --no-cache-dir --upgrade pip==19.1 10 | 11 | # healthcheck 12 | COPY healthcheck /healthcheck 13 | RUN pip3 install --no-cache-dir -r /healthcheck/requirements.txt 14 | ENV FLASK_APP /healthcheck/hc.py 15 | HEALTHCHECK --interval=15s --timeout=15s \ 16 | CMD curl --silent --fail http://localhost:5000/healthcheck || exit 1 17 | 18 | COPY . /vent 19 | RUN pip3 install --no-cache-dir /vent 20 | 21 | RUN mkdir /root/.vent 22 | VOLUME ["/root/.vent"] 23 | 24 | ENV VENT_CONTAINERIZED true 25 | 26 | CMD (flask run > /dev/null 2>&1) & (vent) 27 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM elasticsearch:5-alpine 2 | FROM rabbitmq:3-management 3 | FROM redis:alpine 4 | FROM ubuntu:19.04 5 | LABEL maintainer="Charlie Lewis " 6 | 7 | RUN apt-get update && apt-get install -y \ 8 | apt-transport-https \ 9 | ca-certificates \ 10 | curl \ 11 | git \ 12 | make \ 13 | python3 \ 14 | python3-pip \ 15 | software-properties-common 16 | 17 | RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - 18 | RUN add-apt-repository \ 19 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 20 | $(lsb_release -cs) \ 21 | stable" 22 | 23 | RUN apt-get update && apt-get install -y docker-ce 24 | RUN pip3 install --upgrade pip==19.1 25 | 26 | RUN echo "https://github.com/cyberreboot/vent:\n rabbitmq:" >> ~/.vent_startup.yml 27 | ADD . /vent 28 | WORKDIR /vent 29 | RUN pip3 install -r test-requirements.txt 30 | RUN python3 setup.py install 31 | 32 | ENTRYPOINT ["pytest"] 33 | CMD ["-l", "-s", "-v", "--cov=.", "--cov-report", "term-missing"] 34 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Charlie Lewis 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OS=`uname -s` 2 | PIP=$(shell which pip3 || echo "pip3") 3 | 4 | build: clean 5 | @echo 6 | @echo "checking dependencies" 7 | @echo 8 | env 9 | docker version || true 10 | $(PIP) -V 11 | $(PIP) install -r test-requirements.txt 12 | python3 setup.py install 13 | 14 | docs: docs_clean 15 | @echo "generating documentation" 16 | sphinx-apidoc -f -o docs/source/ vent/ \ 17 | vent/core/file_drop/test_file_drop.py \ 18 | vent/core/network_tap/ncontrol/test_ncontrol.py \ 19 | vent/core/rmq_es-_connector/test_rmq_es_connector.py \ 20 | vent/core/rq_dashboard/rq_dash_settings.py \ 21 | vent/core/rq_dashboard/test_rq_dashboard.py \ 22 | vent/core/rq_worker/settings.py \ 23 | vent/core/rq_worker/test_rq_worker.py 24 | 25 | docs_clean: 26 | @echo "deleting all vent rst files" 27 | rm -rf docs/source/v*.rst 28 | 29 | gpu: build 30 | @if hash nvidia-docker 2>/dev/null; then \ 31 | echo "nvidia-docker found"; \ 32 | docker pull nvidia/cuda:8.0-runtime; \ 33 | if [ "${OS}" = "Linux" ] ; then \ 34 | if [ -f /etc/redhat-release ] ; then \ 35 | sudo sed -i '/ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR/c\ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR -l :3476' /usr/lib/systemd/system/nvidia-docker.service; \ 36 | sudo systemctl daemon-reload; \ 37 | sudo systemctl start nvidia-docker; \ 38 | elif [ -f /etc/debian_version ] ; then \ 39 | sudo sed -i '/ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR/c\ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR -l :3476' /lib/systemd/system/nvidia-docker.service; \ 40 | sudo systemctl daemon-reload; \ 41 | sudo systemctl restart nvidia-docker; \ 42 | fi \ 43 | fi \ 44 | else \ 45 | if [ "${OS}" = "Linux" ] ; then \ 46 | docker pull nvidia/cuda:8.0-runtime; \ 47 | if [ -f /etc/redhat-release ] ; then \ 48 | wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker-1.0.1-1.x86_64.rpm && \ 49 | sudo rpm -i /tmp/nvidia-docker*.rpm && rm /tmp/nvidia-docker*.rpm; \ 50 | sudo sed -i '/ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR/c\ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR -l :3476' /usr/lib/systemd/system/nvidia-docker.service; \ 51 | sudo systemctl daemon-reload; \ 52 | sudo systemctl start nvidia-docker; \ 53 | elif [ -f /etc/debian_version ] ; then \ 54 | wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb && \ 55 | sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb; \ 56 | sudo sed -i '/ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR/c\ExecStart=\/usr\/bin\/nvidia-docker-plugin -s $$SOCK_DIR -l :3476' /lib/systemd/system/nvidia-docker.service; \ 57 | sudo systemctl daemon-reload; \ 58 | sudo systemctl restart nvidia-docker; \ 59 | else \ 60 | wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1_amd64.tar.xz && \ 61 | sudo tar --strip-components=1 -C /usr/bin -xvf /tmp/nvidia-docker*.tar.xz && rm /tmp/nvidia-docker*.tar.xz && \ 62 | sudo -b nohup nvidia-docker-plugin > /tmp/nvidia-docker.log; \ 63 | echo "!!! Nvidia-docker-plugin needs to be configured to bind to all interfaces !!!"; \ 64 | fi \ 65 | else \ 66 | echo "unable to install nvidia-docker on this host"; \ 67 | fi \ 68 | fi 69 | 70 | clean: 71 | rm -rf .cache 72 | rm -rf .coverage 73 | rm -rf .vent 74 | rm -rf vent.egg-info 75 | rm -rf vent.iso 76 | rm -rf dist 77 | rm -rf build 78 | rm -rf plugins 79 | rm -rf core 80 | find . -name "*.pyc" -type f -delete 81 | find . -name "__pycache__" -delete 82 | $(PIP) uninstall -y vent || true 83 | 84 | test: build 85 | pytest -l -s -v --cov=. --cov-report term-missing 86 | 87 | test-menu: build 88 | pytest tests/menu/ -l -s -v --cov=. --cov-report term-missing 89 | 90 | test-unit: build 91 | pytest tests/unit/ -l -s -v --cov=. --cov-report term-missing 92 | 93 | test-plugins: build 94 | pytest vent/ -l -s -v --cov=. --cov-report term-missing 95 | 96 | test-local: test-local-clean clean 97 | docker build -t vent-test -f Dockerfile.test . 98 | docker run -d --name vent-test-redis redis:alpine 99 | docker run -d --name vent-test-rabbitmq rabbitmq:3-management 100 | docker run -d --name vent-test-elasticsearch elasticsearch:2-alpine 101 | docker run -it -v /var/run/docker.sock:/var/run/docker.sock:rw --link vent-test-redis:redis --link vent-test-rabbitmq:localhost --link vent-test-elasticsearch:localhost vent-test 102 | 103 | test-local-clean: 104 | docker rm -f vent-test-redis || true 105 | docker rm -f vent-test-rabbitmq || true 106 | docker rm -f vent-test-elasticsearch || true 107 | 108 | .PHONY: build test 109 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | - [ ] This patch includes any updated documentation of the proposed changes 2 | - [ ] This patch includes tests that verify the proposed changes work as expected 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # vent 3 | 4 | > Network Visibility (an anagram) 5 | 6 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/792bc7e54645427581da66cd6847cc31)](https://www.codacy.com/app/clewis/vent?utm_source=github.com&utm_medium=referral&utm_content=CyberReboot/vent&utm_campaign=badger) 7 | [![Build Status](https://travis-ci.org/CyberReboot/vent.svg?branch=master)](https://travis-ci.org/CyberReboot/vent) 8 | [![Documentation Status](https://readthedocs.org/projects/vent/badge/?version=latest)](http://vent.readthedocs.io/en/latest/?badge=latest) 9 | [![PyPI version](https://badge.fury.io/py/vent.svg)](https://badge.fury.io/py/vent) 10 | [![codecov](https://codecov.io/gh/CyberReboot/vent/branch/master/graph/badge.svg)](https://codecov.io/gh/CyberReboot/vent) 11 | [![Docker Hub Downloads](https://img.shields.io/docker/pulls/cyberreboot/vent-ncapture.svg)](https://hub.docker.com/u/cyberreboot) 12 | 13 | ![Vent Logo](/docs/img/vent-logo.png) 14 | 15 | ## Overview 16 | 17 | vent is a library that includes a CLI designed to serve as a general platform for analyzing network traffic. Built with some basic functionality, vent serves as a user-friendly platform to build custom `plugins` that perform user-defined processing on incoming network data. See this blog post - [Introducing vent](https://blog.cyberreboot.org/introducing-vent-1d883727b624) 18 | 19 | ## Dependencies 20 | 21 | ``` bash 22 | docker>=1.13.1 23 | git 24 | make (if building from source) 25 | pip3 26 | python3.6.x 27 | ``` 28 | 29 | ## Installation 30 | 31 | ### Option 1: Running inside of a Docker container 32 | 33 | ``` bash 34 | docker run -it -v /var/run/docker.sock:/var/run/docker.sock cyberreboot/vent 35 | ``` 36 | 37 | ### Option 2: Installing directly 38 | 39 | ``` bash 40 | pip3 install vent 41 | ``` 42 | 43 | ### Option 3: Getting the source and building 44 | 45 | ``` bash 46 | git clone https://github.com/CyberReboot/vent.git 47 | cd vent 48 | ``` 49 | 50 | Root/sudo users can simply run `make` to compile and install the platform. Users with limited permissions or require user-local installation can use the following: 51 | 52 | ``` bash 53 | sudo env "PATH=$PATH" make 54 | ``` 55 | 56 | _Note - If you already have `docker-py` installed on your machine, you may need to_ `pip uninstall docker-py` _first. `vent` will install `docker-py` as part of the installation process, however there are known incompatibilities of `docker-py` with older versions._ 57 | 58 | ## Running 59 | 60 | ``` bash 61 | vent 62 | ``` 63 | 64 | ## Plugins 65 | 66 | vent supports custom `plugins` that perform user-defined processing on incoming data. 67 | 68 | vent is filetype-agnostic in that the plugins installed within your specific vent instance determine what type of files your instance supports. Simply create your `plugins`, point vent to them & install them, and drop a file in vent to begin processing! 69 | 70 | The [vent-plugins](https://github.com/CyberReboot/vent-plugins) repository showcases a number of example plugins and contains details on how to create your own. 71 | 72 | ## Documentation 73 | 74 | Want to read the documentation for vent? Great! You can find it [here](https://vent.readthedocs.io/en/latest/?badge=latest) 75 | 76 | ## Contributing to vent 77 | 78 | Want to contribute? Awesome! Issue a pull request or see more details [here](https://github.com/CyberReboot/vent/blob/master/CONTRIBUTING.md). 79 | 80 | See [this](https://media.readthedocs.org/pdf/npyscreen/latest/npyscreen.pdf) for a crash course on npyscreen: the TUI used by Vent! 81 | -------------------------------------------------------------------------------- /bin/vent: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import sys 4 | 5 | from vent.menu import VentApp 6 | 7 | try: 8 | app = VentApp() 9 | app.run() 10 | except KeyboardInterrupt: 11 | os.system('reset') 12 | os.system('stty sane') 13 | try: 14 | sys.exit(0) 15 | except SystemExit: 16 | os._exit(0) 17 | -------------------------------------------------------------------------------- /docs/img/vent-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/docs/img/vent-logo.png -------------------------------------------------------------------------------- /docs/source/build_plugin.rst: -------------------------------------------------------------------------------- 1 | .. _customventplugin-label: 2 | 3 | Custom Vent Plugins 4 | ############################ 5 | 6 | Building custom vent plugins is an easy process. 7 | 8 | Each custom vent plugin needs at least a ``Dockerfile`` and a ``vent.template``. 9 | Read more about ``Dockerfile`` `here`_. 10 | 11 | .. _here: https://docs.docker.com/engine/reference/builder/ 12 | 13 | 14 | .. _venttemplate-label: 15 | 16 | Vent.template Files 17 | =================== 18 | Vent template files are what Vent uses to build images into something it recognizes. 19 | Listed here are sections and their various options. 20 | 21 | Look below for examples of a ``vent.template`` file. 22 | 23 | -docker 24 | ------- 25 | All possible options (except network_mode and remove) and their explanations are the same as the parameters for the python `Docker container run command`_. 26 | 27 | .. _Docker container run command: https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run 28 | 29 | 30 | -gpu 31 | ---- 32 | Vent plugins can run on solely on GPU if specified (extra ``on``). This section sets the 33 | settings for GPU processing. At the moment, **only NVIDIA GPUs are supported**. 34 | 35 | *dedicated* 36 | Should the plugin be the only process running on the GPU? If it should be, set the 37 | option to ``yes``. ``no`` by default. 38 | 39 | *device* 40 | If you know the GPU device's number, you can set it here. 41 | 42 | *enabled* 43 | Tell Vent to run the plugin on GPU 44 | 45 | *mem_mb* 46 | The amount of RAM(in MB) a plugin requires. 47 | 48 | 49 | -info 50 | ----- 51 | All metadata about the custom plugin goes under this section. 52 | 53 | *groups* 54 | the group a plugin belongs to. 55 | 56 | *name* 57 | the plugin's name. 58 | 59 | 60 | -service 61 | -------- 62 | Appending info to an exposed port for a running service, multiple exposed ports 63 | can be done for the same service (up to 9) by incrementing the value of 1 at 64 | the end of each settings to correspond to the order of the ports exposed for 65 | that service. 66 | 67 | *uri_prefix1* 68 | Services a tool exposes that need a more specific URI at the beginning 69 | 70 | *uri_postfix1* 71 | Services a tool exposes that need a more specific URI at the end 72 | 73 | *uri_user1* 74 | Services a tool exposes that need a username 75 | 76 | *uri_pw1* 77 | Services a tool exposes that need a password 78 | 79 | 80 | -settings 81 | --------- 82 | Options that specify how the plugin will run. 83 | 84 | *ext_types* 85 | Whatever this option is set to is the file extension that Vent will run this plugin on. 86 | For example, ``ext_types = jpg`` means Vent will run this plugin if a ``.jpg`` 87 | file is placed in ``File Drop``. 88 | 89 | *priority* 90 | Set the order in which tools get started. If a tool belongs to more than one 91 | group, it's possible to specify priorities for each group. The priorities are 92 | comma separated and in the same order that the groups are specified. 93 | 94 | *process_base* 95 | Files put in ``File Drop`` by a user or process outside of Vent will get 96 | processed. This option is set to ``yes`` by default. 97 | 98 | *process_from_tool* 99 | Allows to specify if the tool should process files that are outputs from 100 | other tools specifically. 101 | 102 | *instances* 103 | Allows you to specify how many instantiations of a tool you want running. 104 | For example, you could have two instances of rq_worker running. This would 105 | create two rq_worker containers, which would be useful for scaling larger 106 | amounts of work. 107 | 108 | | 109 | 110 | Example Custom Vent Plugin 111 | ========================== 112 | Let's create a simple Vent plugin that uses ``cat`` to output the contents of a 113 | ``.example`` file. Let's create a ``vent_plugin_example`` directory and enter it. 114 | 115 | First, let's create a simple ``bash`` script that will ``cat`` the contents of a 116 | file. 117 | 118 | .. code-block:: bash 119 | :caption: cat.sh 120 | 121 | #!bin/bash 122 | cat $1 123 | 124 | 125 | Next, a ``Dockerfile`` is needed so let's make it. 126 | 127 | .. code-block:: bash 128 | :caption: Dockerfile 129 | 130 | FROM ubuntu:latest 131 | ADD cat.sh . 132 | ENTRYPOINT ["/cat.sh"] 133 | CMD [""] 134 | 135 | 136 | Lastly, a ``vent.template`` file is needed so Vent knows how to use the plugin: 137 | 138 | .. code-block:: bash 139 | :caption: vent.template 140 | 141 | [info] 142 | name = example plugin 143 | groups = example 144 | 145 | [settings] 146 | ext_types = example 147 | 148 | Here's an example of this plugin using GPUs to do work: 149 | 150 | .. code-block:: bash 151 | :caption: vent.template 152 | 153 | [info] 154 | name = example plugin 155 | groups = example 156 | 157 | [settings] 158 | ext_types = example 159 | 160 | [gpu] 161 | enabled = yes 162 | mem_mb = 1024 163 | dedicated = yes 164 | 165 | 166 | We need to add this to either a git repo or the docker hub. Let's use git. 167 | Push the ``vent_plugin_example`` into some repo. 168 | 169 | Let's now add the custom plugin to Vent. From the plugins sub-menu, select 170 | ``Add new plugin`` and enter the fields with whatever repo 171 | ``vent_plugin_example`` was pushed to. After, select the branch, commit and leave 172 | ``build`` to ``True``. Now select ``example_plugin`` and hit ``OK``. Vent will 173 | now build the custom plugin. 174 | 175 | To test, let's create a test file. 176 | 177 | .. code-block:: bash 178 | :caption: test.example 179 | 180 | qwerty 181 | 182 | 183 | Finally, with Vent and the plugin up and running and all core tools added, built, 184 | and running, let's drop ``test.example`` into ``File Drop``. After a few 185 | seconds, the job counter on the main menu of Vent will show that one job is 186 | running, and it'll finish soon after and show one completed job. 187 | 188 | To check that the plugin worked and outputted ``qwerty``, let's check the syslog 189 | container using the command ``docker logs cyberreboot-vent-syslog-master | grep 190 | qwerty``. 191 | 192 | If you see this line, congrats! You have successfully built your first Vent 193 | plugin. 194 | 195 | If the plugin did not function correctly, try rereading the tutorial or check 196 | the :ref:`troubleshooting-label` guide. 197 | 198 | Other examples of custom plugins can be found at `CyberReboot/vent-plugins`_. 199 | 200 | .. _CyberReboot/vent-plugins: https://github.com/CyberReboot/vent-plugins 201 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Vent documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jul 17 12:58:13 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | # If extensions (or modules to document with autodoc) are in another directory, 15 | # add these directories to sys.path here. If the directory is relative to the 16 | # documentation root, use os.path.abspath to make it absolute, like shown here. 17 | # 18 | import os 19 | import sys 20 | sys.path.insert(0, os.path.abspath('../../')) 21 | 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | # 27 | needs_sphinx = '1.1' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = ['sphinx.ext.autodoc', 33 | 'sphinx.ext.coverage', 34 | 'sphinx.ext.viewcode'] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = u'Vent' 50 | copyright = u'2015-2019, IQT Labs LLC' 51 | author = u'Cyber Reboot' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = u'0.10.2.dev' 59 | # The full version, including alpha/beta/rc tags. 60 | release = u'0.10.2.dev' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This patterns also effect to html_static_path and html_extra_path 72 | exclude_patterns = ['modules.rst', 73 | 'vent.rst'] 74 | 75 | # The name of the Pygments (syntax highlighting) style to use. 76 | pygments_style = 'sphinx' 77 | 78 | # If true, `todo` and `todoList` produce output, else they produce nothing. 79 | todo_include_todos = False 80 | 81 | 82 | # -- Options for HTML output ---------------------------------------------- 83 | 84 | # The theme to use for HTML and HTML Help pages. See the documentation for 85 | # a list of builtin themes. 86 | # 87 | #html_theme = 'alabaster' 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # html_theme_options = {} 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | # Custom sidebar templates, must be a dictionary that maps document names 101 | # to template names. 102 | # 103 | # This is required for the alabaster theme 104 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 105 | html_sidebars = { 106 | '**': [ 107 | 'about.html', 108 | 'navigation.html', 109 | 'relations.html', # needs 'show_related': True theme option to display 110 | 'searchbox.html', 111 | ] 112 | } 113 | 114 | 115 | # -- Options for HTMLHelp output ------------------------------------------ 116 | 117 | # Output file base name for HTML help builder. 118 | htmlhelp_basename = 'Ventdoc' 119 | 120 | 121 | # -- Options for LaTeX output --------------------------------------------- 122 | 123 | latex_elements = { 124 | # The paper size ('letterpaper' or 'a4paper'). 125 | # 126 | # 'papersize': 'letterpaper', 127 | 128 | # The font size ('10pt', '11pt' or '12pt'). 129 | # 130 | # 'pointsize': '10pt', 131 | 132 | # Additional stuff for the LaTeX preamble. 133 | # 134 | # 'preamble': '', 135 | 136 | # Latex figure (float) alignment 137 | # 138 | # 'figure_align': 'htbp', 139 | } 140 | 141 | # Grouping the document tree into LaTeX files. List of tuples 142 | # (source start file, target name, title, 143 | # author, documentclass [howto, manual, or own class]). 144 | latex_documents = [ 145 | (master_doc, 'Vent.tex', u'Vent Documentation', 146 | u'Cyber Reboot', 'manual'), 147 | ] 148 | 149 | 150 | # -- Options for manual page output --------------------------------------- 151 | 152 | # One entry per manual page. List of tuples 153 | # (source start file, name, description, authors, manual section). 154 | man_pages = [ 155 | (master_doc, 'vent', u'Vent Documentation', 156 | [author], 1) 157 | ] 158 | 159 | 160 | # -- Options for Texinfo output ------------------------------------------- 161 | 162 | # Grouping the document tree into Texinfo files. List of tuples 163 | # (source start file, target name, title, author, 164 | # dir menu entry, description, category) 165 | texinfo_documents = [ 166 | (master_doc, 'Vent', u'Vent Documentation', 167 | author, 'Vent', 'One line description of project.', 168 | 'Miscellaneous'), 169 | ] 170 | -------------------------------------------------------------------------------- /docs/source/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing to Vent 2 | #################### 3 | 4 | Want to hack on Vent? Awesome! Here are instructions to get you started. 5 | They are probably not perfect, please let us know if anything feels 6 | wrong or incomplete. 7 | 8 | Contribution guidelines 9 | ======================= 10 | 11 | Pull requests are always welcome 12 | ******************************** 13 | 14 | We are always thrilled to receive pull requests, and do our best to 15 | process them as fast as possible. Not sure if that typo is worth a pull 16 | request? Do it! We will appreciate it. 17 | 18 | If your pull request is not accepted on the first try, don't be 19 | discouraged! If there's a problem with the implementation, hopefully you 20 | received feedback on what to improve. 21 | 22 | We're trying very hard to keep Vent lean and focused. We don't want it 23 | to do everything for everybody. This means that we might decide against 24 | incorporating a new feature. However, there might be a way to implement 25 | that feature *on top of* Vent. 26 | 27 | Create issues... 28 | **************** 29 | 30 | Any significant improvement should be documented as `a github 31 | issue`_ before anybody 32 | starts working on it. 33 | 34 | .. _a github issue: https://github.com/CyberReboot/vent/issues 35 | 36 | ...but check for existing issues first! 37 | *************************************** 38 | 39 | Please take a moment to check that an issue doesn't already exist 40 | documenting your bug report or improvement proposal. If it does, it 41 | never hurts to add a quick "+1" or "I have this problem too". This will 42 | help prioritize the most common problems and requests. 43 | 44 | Conventions 45 | *********** 46 | 47 | Fork the repo and make changes on your fork in a feature branch. 48 | 49 | Make sure you include relevant updates or additions to documentation and 50 | tests when creating or modifying features. 51 | 52 | Pull requests descriptions should be as clear as possible and include a 53 | reference to all the issues that they address. 54 | 55 | Code review comments may be added to your pull request. Discuss, then make the 56 | suggested modifications and push additional commits to your feature branch. Be 57 | sure to post a comment after pushing. The new commits will show up in the pull 58 | request automatically, but the reviewers will not be notified unless you 59 | comment. 60 | 61 | Before the pull request is merged, make sure that you squash your commits into 62 | logical units of work using ``git rebase -i`` and ``git push -f``. After every 63 | commit the test suite should be passing. Include documentation changes in the 64 | same commit so that a revert would remove all traces of the feature or fix. 65 | 66 | Commits that fix or close an issue should include a reference like ``Closes #XXX`` 67 | or ``Fixes #XXX``, which will automatically close the issue when merged. 68 | 69 | Add your name to the ``AUTHORS`` file, but make sure the list is sorted and your 70 | name and email address match your git configuration. The ``AUTHORS`` file is 71 | regenerated occasionally from the git commit history, so a mismatch may result 72 | in your changes being overwritten. 73 | 74 | Decision process 75 | ================ 76 | 77 | How are decisions made? 78 | *********************** 79 | 80 | Short answer: with pull requests to the Vent repository. 81 | 82 | All decisions affecting Vent, big and small, follow the same 3 steps: 83 | 84 | - Step 1: Open a pull request. Anyone can do this. 85 | - Step 2: Discuss the pull request. Anyone can do this. 86 | - Step 3: Accept or refuse a pull request. A maintainer does this. 87 | 88 | How can I become a maintainer? 89 | ****************************** 90 | 91 | - Step 1: learn the code inside out 92 | - Step 2: make yourself useful by contributing code, bug fixes, support etc. 93 | 94 | Don't forget: being a maintainer is a time investment. Make sure you will have time to make yourself available. 95 | You don't have to be a maintainer to make a difference on the project! 96 | 97 | What are a maintainer's responsibility? 98 | *************************************** 99 | 100 | It is every maintainer's responsibility to: 101 | 102 | - 1) Deliver prompt feedback and decisions on pull requests. 103 | - 2) Be available to anyone with questions, bug reports, criticism, etc. regarding Vent. 104 | 105 | How is this process changed? 106 | **************************** 107 | 108 | Just like everything else: by making a pull request :) 109 | 110 | Derivative work from `Docker`_ 111 | 112 | .. _Docker: https://github.com/docker/docker/blob/master/CONTRIBUTING.md 113 | -------------------------------------------------------------------------------- /docs/source/core_and_plugins.rst: -------------------------------------------------------------------------------- 1 | Core Tools and Plugins 2 | ###################### 3 | 4 | Core Tool Explanations 5 | ************************** 6 | There are currently nine core tools that Vent uses. 7 | 8 | elasticsearch 9 | ============= 10 | Enables comprehensive text search of syslog. 11 | 12 | file_drop 13 | ========= 14 | Watches the specified directory for any new files. If a new file is added, it is 15 | added to a ``redis`` queue. 16 | 17 | .. _networktap-label: 18 | 19 | network_tap 20 | =========== 21 | A container that will watch a specific nic using ``tcpdump`` to output pcap 22 | files based on what was monitored. Has an interface located in ``System 23 | Commands -> Network Tap Interface`` in the main action menu. 24 | The interface has six available actions: 25 | 26 | - **Create**: Create a new container with a specified nic, tag, interval (in *seconds*), 27 | filter, and iterations. The container is also automatically started on 28 | creation. 29 | - **Delete**: Delete a specified network tap container. Containers *must be stopped* before they 30 | are able to be deleted. 31 | - **List**: Show all network tap containers. Will return container's ID, if the container is 32 | running or not, and the tag provided in ``create``. 33 | - **NICs**: Show all available network interfaces. Will return a list of the 34 | names of the available NICs. Note for ``Docker for Mac`` it will show 35 | available network interfaces on the VM running the Docker daemon, not the 36 | network interface names on the Mac host. 37 | - **Start**: Start a network tap container if it is exited. Will run with the same 38 | options given to the container in ``create``. 39 | - **Stop**: Stop a network tap container. 40 | - **Update**: Update the metadata of a network tap container. 41 | 42 | rabbitmq 43 | ======== 44 | Formats messages received from syslog and sends them to rmq_es_connector. 45 | 46 | redis 47 | ===== 48 | A key/value store that is used for the queuing system that ``file drop`` sends to 49 | and ``rq_worker`` pulls out of. 50 | 51 | rmq_es_connector 52 | ================ 53 | A gateway between the messaging system and ``elasticsearch``. This way, the message 54 | formatting system is not locked to ``rabbitmq``. 55 | 56 | rq_worker 57 | ========= 58 | The tool that takes files from the ``redis`` queue and runs plugins that deal with 59 | those file extensions. 60 | 61 | rq_dashboard 62 | ============ 63 | Management console to look at rq_worker's active queue. 64 | 65 | syslog 66 | ====== 67 | Standard logging system that adheres to the syslog standard. All tool containers send 68 | their information to syslog. If there's some unexpected outputs or a container 69 | isn't running properly, all information will be in this tool's container. 70 | 71 | Access this tool's container with the command: 72 | ``docker logs cyberreboot-vent-syslog-master`` 73 | 74 | | 75 | 76 | Core Tool and Plugin Actions 77 | **************************** 78 | 79 | Short explanations of all actions available in the core tools and plugins sub-menu. 80 | 81 | Add all latest core/plugin tools 82 | ================================ 83 | Clone the latest version of the tool. This will **not** update or 84 | remove any tools that have already been added. 85 | 86 | By default, core tools are cloned from `CyberReboot/vent`_ and plugins, if no 87 | custom repo is specified, are cloned from `CyberReboot/vent-plugins`_. 88 | 89 | .. _CyberReboot/vent: https://github.com/CyberReboot/vent/ 90 | .. _CyberReboot/vent-plugins: https://github.com/CyberReboot/vent-plugins/ 91 | 92 | Build core/plugin tools 93 | ======================= 94 | Build docker images from the Dockerfiles obtained from adding. 95 | 96 | Clean core/plugin tools 97 | ======================= 98 | Stop and remove the chosen tools' containers. 99 | 100 | Configure core/plugin tools 101 | =========================== 102 | Edit a tool's vent.template file found in the tool's respective folder. 103 | Read more about :ref:`venttemplate-label`. 104 | 105 | Disable core/plugin tools 106 | ========================= 107 | Remove chosen tools from menus. For example, let's say there were ten tools but only 108 | five were needed. Disabling the five unneeded tools would stop those tools from 109 | appearing on the other menus. 110 | 111 | Enable core/plugin tools 112 | ======================== 113 | Opposite of disable tools. Enables the tools so they can be seen again. 114 | 115 | Inventory of core/plugin tools 116 | ============================== 117 | Provides meta data regarding currently added core/plugin tools. It tells if a tool is built, 118 | enabled, the name of the image, and the if the tool is currently running. 119 | 120 | Remove core/plugin tools 121 | ======================== 122 | Remove a tool entirely. Any of that tool's containers are also stopped and 123 | deleted. The tool must be added again if it is to be used. 124 | 125 | Start core/plugin tools 126 | ======================= 127 | Start the tools' respective containers. 128 | 129 | Stop core/plugin tools 130 | ====================== 131 | Stop the tools' respective containers. 132 | 133 | Update core/plugin tools 134 | ======================== 135 | Pulls the latest commit of the tool from its repo and builds it. 136 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Vent documentation master file, created by 2 | sphinx-quickstart on Mon Jul 17 12:58:13 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Overview 7 | ======== 8 | Vent is a library that includes a CLI designed to serve as a general platform for analyzing network traffic. Built with some basic functionality, Vent serves as a user-friendly platform to build custom ``plugins`` on to perform user-defined processing on incoming network data. Vent is filetype-agnostic in that the plugins installed within your specific vent instance determine what type of files your instance supports. 9 | 10 | Simply create your ``plugins``, point Vent to them, install them, and drop a file in Vent to begin processing! 11 | 12 | Dependencies 13 | ------------ 14 | :: 15 | 16 | docker >= 1.13.1 17 | make (if building from source) 18 | pip3 19 | python3.6.x 20 | 21 | 22 | Getting Set Up 23 | -------------- 24 | There's two ways to get Vent up and running on your machine: 25 | 26 | 1. Pip:: 27 | 28 | $ pip3 install vent 29 | 30 | 2. Clone the repo:: 31 | 32 | $ git clone --recursive https://github.com/CyberReboot/vent.git 33 | $ cd vent 34 | 35 | 3. Build from source (for sudo/root privileged users):: 36 | 37 | $ make 38 | 39 | Users with limited permissions or require user-local installation can use the following:: 40 | 41 | $ sudo env "PATH=$PATH" make 42 | 43 | .. note:: If you already have ``docker-py`` installed on your machine, you may need to ``pip3 uninstall docker-py`` first. ``vent`` will install ``docker-py`` as part of the installation process. However, there are known incompatibilities of ``docker-py`` with older versions. 44 | 45 | Once installed, it's simply:: 46 | 47 | $ vent 48 | 49 | 50 | Contributing to Vent 51 | ==================== 52 | 53 | Want to contribute? Awesome! Issue a pull request or see more `details here`_. 54 | 55 | See `this`_ for a crash course on npyscreen: the GUI used by Vent! 56 | 57 | .. _details here: https://github.com/CyberReboot/vent/blob/master/CONTRIBUTING.md 58 | .. _this: https://media.readthedocs.org/pdf/npyscreen/latest/npyscreen.pdf 59 | 60 | | 61 | 62 | .. toctree:: 63 | :maxdepth: 2 64 | :caption: Vent Guides 65 | 66 | quickstart 67 | 68 | build_plugin 69 | 70 | plugin_config 71 | 72 | initialization 73 | 74 | troubleshoot 75 | 76 | contributing 77 | 78 | new_release 79 | 80 | 81 | .. toctree:: 82 | :maxdepth: 2 83 | :caption: Vent Internals 84 | 85 | core_and_plugins 86 | 87 | system_command 88 | 89 | user_data 90 | 91 | .. toctree:: 92 | :maxdepth: 1 93 | :caption: Vent Code 94 | 95 | vent.api 96 | 97 | vent.core 98 | 99 | vent.helpers 100 | 101 | vent.menus 102 | 103 | 104 | Indices and tables 105 | ================== 106 | 107 | * :ref:`genindex` 108 | * :ref:`modindex` 109 | * :ref:`search` 110 | -------------------------------------------------------------------------------- /docs/source/initialization.rst: -------------------------------------------------------------------------------- 1 | Vent Initialization 2 | ################### 3 | 4 | This page describes the different actions that vent takes upon 5 | initialization. 6 | 7 | File Structure 8 | ============== 9 | Vent will set up the necessary file structure to store all of the 10 | :ref:`userdata-label` 11 | 12 | Auto-Install 13 | ============ 14 | Vent will detect if there are any vent images already running before an 15 | instance of itself is instantiated, and will automatically install the 16 | tools that correspond to that image without you having to do anything. 17 | 18 | Startup 19 | ======= 20 | Before initialization you can specify a certain set of tools that you want 21 | vent to install once it starts up for the first time. You can do this by 22 | writing a .vent_startup.yml file in your home directory. If you do not know 23 | how YAML syntax works, `watch this quick video`_ for more info. The syntax that 24 | vent uses when parsing the .vent_startup.yml for tools is as follows:: 25 | 26 | repo_name: 27 | tool_name: 28 | 29 | 30 | A thing to note, tool_name here means the name of the directory in which 31 | the Dockerfile and vent.template for the tool can be found. If you have 32 | multiple tools defined in the same directory, then use the syntax @tool_name 33 | to differentiate between the different tools in that directory. A lot of 34 | the additional settings you can configure are the same as what you would do 35 | for :ref:`customventplugin-label`. These additional settings will be used 36 | to update the default settings for that tool (as defined in its vent.template), 37 | they will not overwrite the default settings. Besides those settings, there are 38 | some that are specific just to this startup file: 39 | 40 | .. _watch this quick video: https://www.youtube.com/watch?v=W3tQPk8DNbk 41 | 42 | *build* 43 | If you set build to yes, then the tool will be installed and then built 44 | immediately rather than just installed. 45 | 46 | *start* 47 | If you set start to yes, then the tool will be started immediately. **Note**, 48 | you have to set build to yes for a tool in order for you to be able to set 49 | start to yes as well. 50 | 51 | *branch* 52 | For tools installed from a repo, if you want to install the tools from a 53 | specific branch of the repo, specify it with the branch keyword. By default, 54 | the master branch will be where the tools are installed from. 55 | 56 | *version* 57 | For tools installed from a repo, if you want to install the tools from a specific 58 | commit version, specify it with the version keyword. By default, the version HEAD 59 | will be used to install tools. 60 | 61 | Example Startup 62 | --------------- 63 | With that said, here is an example .vent-startup.yml file that vent could use:: 64 | 65 | https://github.com/cyberreboot/vent: 66 | rabbitmq: 67 | build: yes 68 | start: yes 69 | elasticsearch: 70 | rq_worker: 71 | build: yes 72 | start: yes 73 | settings: 74 | instances: 2 75 | 76 | This would install the tools rabbitmq, elasticsearch, and rq_worker. 77 | Additionally, this would build and start rabbitmq and rq_worker, and 78 | it would set the number of instances in settings to 2 for rq_worker. 79 | Notice how with elasticsearch even though there were no additional 80 | parameters defined it still had the the colon at the end of its 81 | declaration. This is necessary otherwise a parsing error will occur 82 | with YAML. 83 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | vent 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | vent 8 | -------------------------------------------------------------------------------- /docs/source/new_release.rst: -------------------------------------------------------------------------------- 1 | New Vent Releases 2 | ################# 3 | 4 | Get the latest clone of Vent from https://github.com/CyberReboot/vent 5 | 6 | 1. Change the version number 7 | 8 | - ``setup.py`` 9 | - ``docs/source/conf.py`` 10 | 11 | 2. Edit ``CHANGELOG.md`` and include a list of changes that were made. Please 12 | follow previous formatting. 13 | 14 | 3. Run list of authors and put in `AUTHORS` to make sure it is up to date:: 15 | 16 | git log --format='%aN <%aE>' | sort -f | uniq 17 | 18 | 4. Commit the changes, open a PR, and merge into ``master``. 19 | 20 | 5. Now let's upload the release to pypi assuming there's an account with admin 21 | access and a corresponding ``.pypirc``:: 22 | 23 | python3 setup.py sdist upload 24 | 25 | 6. Create a new github release. Tag and release title are the version number. 26 | 27 | 7. Finally, change the version number to the next version number with a ``dev`` 28 | tag. Eg: ``0.4.4.dev``. Commit the version change, make a PR, and merge to ``master``. 29 | -------------------------------------------------------------------------------- /docs/source/plugin_config.rst: -------------------------------------------------------------------------------- 1 | Custom Configurations of Plugins 2 | ################################ 3 | 4 | Say your custom plugin, called ``example_plugin``, has a 5 | ``example_plugin.config`` file that it needs filled out before the plugin is 6 | built into an image. 7 | 8 | .. code-block:: bash 9 | :caption: example_plugin.config 10 | 11 | [Some_Section] 12 | example_option1 = FILL IN 13 | example_option2 = FILL IN 14 | example_option3 = FILL IN 15 | 16 | Vent can now fill in those values for you. 17 | In your home directory, have a file called ``.plugin_config.yml``. 18 | 19 | We'll fill in ``.plugin_config.yml`` with the necessary fields: 20 | 21 | .. code-block:: bash 22 | :caption: .plugin.config.yml 23 | 24 | example_plugin 25 | Some_Section: 26 | example_option1 = 1 27 | example_option2 = 2 28 | example_option3 = 3 29 | 30 | Vent will assume that, in the directory ``example_plugin``, there exists another 31 | directory titled ``config`` that has the actual config file. 32 | The path of our example would be 33 | ``example_plugin/config/example_plugin.config``. 34 | 35 | Using this ``.plugin_config.yml``, you can fill in multiple tools' config files 36 | at the same time. 37 | 38 | If you make a change to your config files and want to rebuild the images using 39 | these new settings, **clean** and then **build** the plugin tools in the ``plugins`` 40 | submenu. Vent will fill in the config files with the new values. 41 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | Vent Quickstart Guide 2 | ##################### 3 | 4 | Getting Vent set up is quick and easy. 5 | 6 | **1.** First, use pip to install the latest *stable* version of Vent: 7 | 8 | - Using pip:: 9 | 10 | pip3 install vent && vent 11 | 12 | 13 | Developer versions of Vent are available but may not be entirely stable. 14 | 15 | - Using Docker:: 16 | 17 | docker pull cyberreboot/vent 18 | docker run -it vent_image_id 19 | 20 | In order to avoid having to use sudo or run docker as root, adding your current 21 | user to the docker group is the recommended way to work. 22 | 23 | - Using Git:: 24 | 25 | git clone https://github.com/CyberReboot/vent 26 | cd vent && make && vent 27 | 28 | **2.** Now that Vent has started, let's add, build, and start the core tools. 29 | 30 | 1. In the main menu, press ``^X`` to open the action menu 31 | 2. Select ``Core Tools`` or press ``c`` 32 | 3. Select ``Add all latest core tools`` or press ``i``. Vent will now clone the 33 | core tools' directories from `CyberReboot/vent`_. 34 | 4. Select ``Build core tools`` from the core tools sub-menu and use the arrow 35 | keys and the Enter key to press ``OK``. It's possible to choose which core 36 | tools are built using the Space key. Boxes with an ``X`` in them have been 37 | selected. Note that building the core tools takes a few minutes. Please 38 | be patient while Vent creates the images. 39 | 5. Once the tool images are built, go back to the core tools sub-menu from 40 | main action menu and select ``Start core tools`` and hit ``OK``. Much like 41 | ``Build core tools``, it's possible to select which core tools are 42 | started. 43 | 44 | .. _CyberReboot/vent: https://github.com/CyberReboot/vent/ 45 | 46 | **3.** The core tools' containers are up and running. Next, let's add some plugins. 47 | 48 | 1. From the action menu, Select ``Plugins`` or press ``p``. 49 | 2. Select ``Add new plugin`` or press ``a``. 50 | 3. For this quick start guide, we will use one of the example plugins 51 | provided from `CyberReboot/vent-plugins`_. So just hit ``OK`` on the form. 52 | 4. Press the Space key to choose the ``master`` branch and hit ``OK``. 53 | 5. Uncheck all the boxes except for ``/tcpdump_hex_parser`` and hit ``OK``. 54 | Depending on the plugin, add times may vary so it is not unusual for long 55 | plugin add times. 56 | 57 | .. _CyberReboot/vent-plugins: https://github.com/CyberReboot/vent-plugins/ 58 | 59 | **4.** Now we have a plugin that can process files with the extension ``.pcap``. 60 | 61 | 1. Now, at the Vent main menu, look for the field ``File Drop``. This is the 62 | folder that Vent watches for new files. 63 | 2. Move or copy a ``.pcap`` file into the path. Vent will recognize this new file 64 | and start ``tcpdump_hex_parser``. Depending on the size of the ``.pcap`` 65 | file, it could take anywhere from a few seconds to minutes. You should see 66 | the ``jobs running`` counter increase by one and, after the plugin is 67 | finished running, the ``completed jobs`` counter will increase by one. 68 | 69 | **5.** Let's look at the results of the plugin using elasticsearch. 70 | 71 | 1. From the action menu, select ``Services Running`` and select ``Core 72 | Services``. 73 | 2. Copy the address next to ``elasticsearch`` into the web browser of choice. 74 | 3. On the main page, there should be a section with ``pcap`` with the results 75 | of the plugin. 76 | 77 | Congrats! Vent is setup and has successfully recognized the pcap file and ran a 78 | plugin that specifically deals with pcaps. You can now remove the 79 | ``tcpdump_hex_parser`` via the ``Plugins`` sub-menu and create and install your own 80 | :ref:`customventplugin-label` 81 | -------------------------------------------------------------------------------- /docs/source/system_command.rst: -------------------------------------------------------------------------------- 1 | System Commands 2 | ############### 3 | 4 | This page describes what the different actions in the ``System Commands`` 5 | sub-menu do. 6 | 7 | Backup 8 | ****** 9 | Saves the ``vent.cfg`` and the ``plugin_manifest``. 10 | 11 | Change vent configuration 12 | ************************* 13 | Edit the ``vent.cfg`` in Vent. The ``vent.cfg`` file can be found in the 14 | directory listed next to ``User Data`` in the main Vent menu. 15 | 16 | Detect GPUs 17 | *********** 18 | Detect GPU devices on the current machine running Vent. Currently works **with 19 | NVIDIA GPUs only** . 20 | 21 | Enable Swarm Mode 22 | ***************** 23 | To Be Implemented. 24 | 25 | Factory reset 26 | ************* 27 | Remove all Vent related images and containers. Removes the ``User Data`` 28 | directory as well. Please back up any necessary information before activating this option. 29 | 30 | Restore 31 | ******* 32 | Restores a backup. 33 | 34 | Upgrade 35 | ******* 36 | To Be Implemented 37 | 38 | Network Tap Interface 39 | ********************* 40 | The interface in which to interact with the ``Network Tap`` core tool. Read more 41 | about :ref:`networktap-label`. Each form will also have more information 42 | regarding the specific action. 43 | -------------------------------------------------------------------------------- /docs/source/troubleshoot.rst: -------------------------------------------------------------------------------- 1 | .. _troubleshooting-label: 2 | 3 | Troubleshooting 4 | ############### 5 | 6 | Basic Troubleshooting 7 | ========================= 8 | Something not working as expected within Vent? Not a problem! 9 | Let's first get some basic possible errors out of the way. 10 | 11 | 1. Is Docker installed and is the daemon running? Vent uses Docker heavily so 12 | it is necessary to have the Docker installed and the daemon running. 13 | 2. Is this the latest version of Vent? Ways to get the latest Vent 14 | version: 15 | 16 | - Using pip:: 17 | 18 | pip3 install vent && vent 19 | 20 | 21 | Pip installs the latest *stable* release of Vent. If a problem still persists, 22 | try using the latest developer build: 23 | 24 | - Using Docker:: 25 | 26 | docker pull cyberreboot/vent 27 | docker run -it vent_image_id 28 | 29 | - Using Git:: 30 | 31 | git clone https://github.com/CyberReboot/vent 32 | cd vent && make && vent 33 | 34 | | 35 | 36 | In-Depth Troubleshooting 37 | ============================ 38 | Still not working? That's fine! Let's get into the nitty gritty and 39 | try to figure out what went wrong. 40 | 41 | 42 | Is it Vent that's causing the problem? 43 | -------------------------------------- 44 | Firstly, let's see if it's Vent that's causing the problems. 45 | Go to the ``User Data`` folder and open up ``vent.log`` with your favorite 46 | text editor. Let's search the key term ``False``. Iterate through the 47 | search results and look for ``Status of some_function: False``. This 48 | tells us that one of Vent's core functions is not performing as 49 | expected. Next to it, there will be an error message explaining what 50 | went wrong. 51 | 52 | If there's a problem with Vent's implementation, please create an issue `here`_. 53 | 54 | .. _here: https://github.com/CyberReboot/vent/issues 55 | 56 | 57 | Is it a custom plugin that's causing the problem? 58 | ------------------------------------------------- 59 | If there's no obvious error messages within ``vent.log``, let's check any 60 | added plugin tools and their containers. 61 | 62 | Run the command ``docker logs cyberreboot-vent-syslog-master``. 63 | This will return all information about all plugin containers and any 64 | information regarding the error should be displayed here. 65 | -------------------------------------------------------------------------------- /docs/source/user_data.rst: -------------------------------------------------------------------------------- 1 | .. _userdata-label: 2 | 3 | User Data 4 | ######### 5 | 6 | The User Data directory is listed on the Vent main menu. Here you can find five 7 | files and two directories here. 8 | 9 | Files 10 | ===== 11 | 12 | plugin_manifest.cfg 13 | ------------------- 14 | Meta data about all built core and plugin tools. 15 | 16 | status.json 17 | ----------- 18 | All data regarding finished jobs is written to this JSON file. This file actually doesn't 19 | exist until a job is processed and **completed**. Each tool will get it's own 20 | JSON entry. If there were two tools that ran and finished, there will be one 21 | entry for each tool. 22 | 23 | vent.cfg 24 | -------- 25 | A configuration file used to customize Vent and its processes. 26 | 27 | -main 28 | ^^^^^ 29 | *files* 30 | The directory of ``File Drop``. For example, if ``files = /opt/vent_files``, 31 | Vent will monitor for any new files in that directory. 32 | 33 | *services_uri* 34 | Override the default POST name for all services. So, let's say Vent has 35 | ``Elasticsearch`` at URL ``0.0.0.0:3772``. 36 | 37 | By adding:: 38 | 39 | services_uri = 5.5.5.5 40 | 41 | under the main section, the URL for ``Elasticsearch`` is now 42 | ``5.5.5.5:3772``. 43 | 44 | -network-mapping 45 | ^^^^^^^^^^^^^^^^ 46 | *nic_name* 47 | The option here can be whatever is desired. The value is the nic that the 48 | plugin tool ``replay_pcap`` will replay to. 49 | 50 | For example:: 51 | 52 | my_nic = enp0s25 53 | 54 | Now the plugin tool ``replay_pcap`` will mirror whatever given pcap files to 55 | ``enp0s25``. 56 | 57 | Currently, only one nic at a time is supported. 58 | 59 | 60 | -nvidia-docker-plugin 61 | ^^^^^^^^^^^^^^^^^^^^^ 62 | *port* 63 | Override port for the Nvidia docker plugin 64 | 65 | *host* 66 | Override the host for the Nvidia docker plugin 67 | 68 | -external-services 69 | ^^^^^^^^^^^^^^^^^^ 70 | Name of tool we want Vent to use externally rather than internally. 71 | For example, if there is already an ``Elasticsearch`` service that a user wants 72 | Vent to use rather than Vent's internal ``Elasticsearch``, we could write:: 73 | 74 | [external-services] 75 | Elasticsearch = {"ip_address": "172.10.5.3", "port": "9200", 76 | "locally_active": "yes"} 77 | 78 | If a port is not specified, then it defaults to core tool's default port. 79 | Also, you can toggle whether you want to use the local docker container 80 | that vent utilizes or the external service by switching locally_active 81 | between yes and no. 82 | 83 | -build-options 84 | ^^^^^^^^^^^^^^ 85 | *use_existing_images* 86 | If set to ``yes`` it will not try to rebuild or pull images that already 87 | exist. 88 | 89 | -groups 90 | ^^^^^^^ 91 | *start_order* 92 | The order in which the tool groups are started in csv format. For example:: 93 | 94 | core,pcap,replay 95 | 96 | means that core tools are started first, then any tool in the ``pcap`` 97 | group, and lastly any tool in the ``replay`` group. Within those groups, 98 | plugin priority will determine the start order of the tools. 99 | 100 | vent.init 101 | --------- 102 | This file only tells Vent if this is the first time Vent is running so the 103 | tutorial can be the first menu listed rather than Vent's main menu. 104 | 105 | vent.log 106 | -------- 107 | Log file that keeps track of the statuses of Vent's functionality. It lists what 108 | action is taking place and shows a list of steps taken to perform that action. 109 | So if something goes wrong, this file is a good place to start to find out what 110 | the problem is. 111 | 112 | 113 | Directories 114 | =========== 115 | 116 | plugins 117 | ---------- 118 | This is a folder that Vent clones to when dealing with tools. So when Vent 119 | adds or builds tools, this is the working directory. 120 | -------------------------------------------------------------------------------- /docs/source/vent.api.rst: -------------------------------------------------------------------------------- 1 | vent.api package 2 | ================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.api.actions module 8 | ----------------------- 9 | 10 | .. automodule:: vent.api.actions 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | vent.api.menu\_helpers module 16 | ----------------------------- 17 | 18 | .. automodule:: vent.api.menu_helpers 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | vent.api.plugin\_helpers module 24 | ------------------------------- 25 | 26 | .. automodule:: vent.api.plugin_helpers 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | vent.api.plugins module 32 | ----------------------- 33 | 34 | .. automodule:: vent.api.plugins 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | vent.api.templates module 40 | ------------------------- 41 | 42 | .. automodule:: vent.api.templates 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: vent.api 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/source/vent.core.file_drop.rst: -------------------------------------------------------------------------------- 1 | vent.core.file\_drop package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.core.file\_drop.file\_drop module 8 | -------------------------------------- 9 | 10 | .. automodule:: vent.core.file_drop.file_drop 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: vent.core.file_drop 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/source/vent.core.network_tap.ncontrol.rst: -------------------------------------------------------------------------------- 1 | vent.core.network\_tap.ncontrol package 2 | ======================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.core.network\_tap.ncontrol.ncontrol module 8 | ----------------------------------------------- 9 | 10 | .. automodule:: vent.core.network_tap.ncontrol.ncontrol 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | vent.core.network\_tap.ncontrol.paths module 16 | -------------------------------------------- 17 | 18 | .. automodule:: vent.core.network_tap.ncontrol.paths 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | vent.core.network\_tap.ncontrol.prestart module 24 | ----------------------------------------------- 25 | 26 | .. automodule:: vent.core.network_tap.ncontrol.prestart 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | vent.core.network\_tap.ncontrol.routes module 32 | --------------------------------------------- 33 | 34 | .. automodule:: vent.core.network_tap.ncontrol.routes 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: vent.core.network_tap.ncontrol 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/source/vent.core.network_tap.rst: -------------------------------------------------------------------------------- 1 | vent.core.network\_tap package 2 | ============================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | vent.core.network_tap.ncontrol 10 | 11 | Module contents 12 | --------------- 13 | 14 | .. automodule:: vent.core.network_tap 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/source/vent.core.rmq_es_connector.rst: -------------------------------------------------------------------------------- 1 | vent.core.rmq\_es\_connector package 2 | ==================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.core.rmq\_es\_connector.rmq\_es\_connector module 8 | ------------------------------------------------------ 9 | 10 | .. automodule:: vent.core.rmq_es_connector.rmq_es_connector 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | vent.core.rmq\_es\_connector.test\_rmq\_es\_connector module 16 | ------------------------------------------------------------ 17 | 18 | .. automodule:: vent.core.rmq_es_connector.test_rmq_es_connector 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: vent.core.rmq_es_connector 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/source/vent.core.rq_dashboard.rst: -------------------------------------------------------------------------------- 1 | vent.core.rq\_dashboard package 2 | =============================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: vent.core.rq_dashboard 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/vent.core.rq_worker.rst: -------------------------------------------------------------------------------- 1 | vent.core.rq\_worker package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.core.rq\_worker.test\_watch module 8 | --------------------------------------- 9 | 10 | .. automodule:: vent.core.rq_worker.test_watch 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | vent.core.rq\_worker.watch module 16 | --------------------------------- 17 | 18 | .. automodule:: vent.core.rq_worker.watch 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: vent.core.rq_worker 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/source/vent.core.rst: -------------------------------------------------------------------------------- 1 | vent.core package 2 | ================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | vent.core.file_drop 10 | vent.core.network_tap 11 | vent.core.rmq_es_connector 12 | vent.core.rq_dashboard 13 | vent.core.rq_worker 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: vent.core 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /docs/source/vent.helpers.rst: -------------------------------------------------------------------------------- 1 | vent.helpers package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.helpers.errors module 8 | -------------------------- 9 | 10 | .. automodule:: vent.helpers.errors 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | vent.helpers.logs module 16 | ------------------------ 17 | 18 | .. automodule:: vent.helpers.logs 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | vent.helpers.meta module 24 | ------------------------ 25 | 26 | .. automodule:: vent.helpers.meta 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | vent.helpers.paths module 32 | ------------------------- 33 | 34 | .. automodule:: vent.helpers.paths 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: vent.helpers 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/source/vent.menus.rst: -------------------------------------------------------------------------------- 1 | vent.menus package 2 | ================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | vent.menus.add module 8 | --------------------- 9 | 10 | .. automodule:: vent.menus.add 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | vent.menus.add\_options module 16 | ------------------------------ 17 | 18 | .. automodule:: vent.menus.add_options 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | vent.menus.backup module 24 | ------------------------ 25 | 26 | .. automodule:: vent.menus.backup 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | vent.menus.choose\_tools module 32 | ------------------------------- 33 | 34 | .. automodule:: vent.menus.choose_tools 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | vent.menus.del\_instances module 40 | -------------------------------- 41 | 42 | .. automodule:: vent.menus.del_instances 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | vent.menus.editor module 48 | ------------------------ 49 | 50 | .. automodule:: vent.menus.editor 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | vent.menus.help module 56 | ---------------------- 57 | 58 | .. automodule:: vent.menus.help 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | vent.menus.inventory module 64 | --------------------------- 65 | 66 | .. automodule:: vent.menus.inventory 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | vent.menus.inventory\_forms module 72 | ---------------------------------- 73 | 74 | .. automodule:: vent.menus.inventory_forms 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | vent.menus.main module 80 | ---------------------- 81 | 82 | .. automodule:: vent.menus.main 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | vent.menus.ntap module 88 | ---------------------- 89 | 90 | .. automodule:: vent.menus.ntap 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | vent.menus.services module 96 | -------------------------- 97 | 98 | .. automodule:: vent.menus.services 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | vent.menus.tools module 104 | ----------------------- 105 | 106 | .. automodule:: vent.menus.tools 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | vent.menus.tutorial\_forms module 112 | --------------------------------- 113 | 114 | .. automodule:: vent.menus.tutorial_forms 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | vent.menus.tutorials module 120 | --------------------------- 121 | 122 | .. automodule:: vent.menus.tutorials 123 | :members: 124 | :undoc-members: 125 | :show-inheritance: 126 | 127 | 128 | Module contents 129 | --------------- 130 | 131 | .. automodule:: vent.menus 132 | :members: 133 | :undoc-members: 134 | :show-inheritance: 135 | -------------------------------------------------------------------------------- /docs/source/vent.rst: -------------------------------------------------------------------------------- 1 | vent package 2 | ============ 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | vent.api 10 | vent.core 11 | vent.helpers 12 | vent.menus 13 | 14 | Submodules 15 | ---------- 16 | 17 | vent.menu module 18 | ---------------- 19 | 20 | .. automodule:: vent.menu 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | 25 | 26 | Module contents 27 | --------------- 28 | 29 | .. automodule:: vent 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | -------------------------------------------------------------------------------- /healthcheck/hc.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from healthcheck import EnvironmentDump 4 | from healthcheck import HealthCheck 5 | 6 | app = Flask(__name__) 7 | 8 | health = HealthCheck(app, '/healthcheck') 9 | envdump = EnvironmentDump(app, '/environment') 10 | 11 | 12 | def application_data(): 13 | return {'maintainer': 'Charlie Lewis', 14 | 'git_repo': 'https://github.com/CyberReboot/vent', 15 | 'app': 'vent'} 16 | 17 | 18 | envdump.add_section('application', application_data) 19 | -------------------------------------------------------------------------------- /healthcheck/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==1.1.1 2 | healthcheck==1.3.3 3 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base", 4 | "docker:enableMajor" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='vent', 5 | version='v0.10.2.dev', 6 | packages=['vent', 'vent.core', 'vent.core.file_drop', 7 | 'vent.core.rq_worker', 'vent.extras.rq_dashboard', 'vent.menus', 8 | 'vent.extras.network_tap', 'vent.extras.network_tap.ncontrol', 9 | 'vent.extras.rmq_es_connector', 'vent.helpers', 'vent.api'], 10 | install_requires=['docker>=4.0.2', 'npyscreen>=4.10.5', 'pyyaml>=5.1.2'], 11 | scripts=['bin/vent'], 12 | license='Apache License 2.0', 13 | author='arpit', 14 | author_email='', 15 | maintainer='Charlie Lewis', 16 | maintainer_email='clewis@iqt.org', 17 | description=('A library that includes a CLI designed to serve as a' 18 | ' platform to collect and analyze data across a flexible set' 19 | ' of tools and technologies.'), 20 | keywords='docker containers platform collection analysis tools devops', 21 | url='https://github.com/CyberReboot/vent', 22 | ) 23 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | docker==4.1.0 2 | elasticsearch==7.1.0 3 | falcon==2.0.0 4 | falcon-cors==1.1.7 5 | npyscreen==4.10.5 6 | pika==1.1.0 7 | pytest==5.3.2 8 | pytest-cov==2.8.1 9 | python-magic==0.4.15 10 | pyyaml==5.2 11 | redis==3.3.11 12 | rq==1.1.0 13 | sphinx==2.3.1 14 | urllib3==1.25.7 15 | watchdog==0.9.0 16 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/tests/__init__.py -------------------------------------------------------------------------------- /tests/menu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/tests/menu/__init__.py -------------------------------------------------------------------------------- /tests/menu/test_menu.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import curses 3 | 4 | import npyscreen 5 | 6 | from vent.helpers.paths import PathDirs 7 | from vent.menu import VentApp 8 | from vent.menus.main import MainForm 9 | 10 | npyscreen.TEST_SETTINGS['CONTINUE_AFTER_TEST_INPUT'] = False 11 | 12 | 13 | def run_menu(test_input): 14 | """ Actually run the menu and process any input """ 15 | # initialize tutorial 16 | paths = PathDirs() 17 | first_time = paths.ensure_file(paths.init_file) 18 | assert first_time[0] == True 19 | 20 | npyscreen.TEST_SETTINGS['TEST_INPUT'] = test_input 21 | 22 | A = VentApp() 23 | try: 24 | A.run(fork=False) 25 | except npyscreen.ExhaustedTestInput as e: 26 | pass 27 | 28 | 29 | def test_menu(): 30 | """ Run menu tests """ 31 | CTRL_Q = '^Q' 32 | CTRL_T = '^T' 33 | CTRL_X = '^X' 34 | CTRL_V = '^V' 35 | ENTER = curses.ascii.CR 36 | TAB = curses.ascii.TAB 37 | LEFT = curses.KEY_LEFT 38 | RIGHT = curses.KEY_RIGHT 39 | DOWN = curses.KEY_DOWN 40 | SPACE = curses.ascii.SP 41 | BACKSPACE = curses.ascii.BS 42 | 43 | # go through help menus 44 | run_menu([ENTER, CTRL_T, CTRL_X, 'p', 'a', 45 | ENTER, ENTER, ENTER, ENTER, ENTER, ENTER, ENTER, ENTER, ENTER]) 46 | 47 | # go to help menu and leave again 48 | run_menu([ENTER, CTRL_T, RIGHT, ENTER]) 49 | 50 | # configure - quit in the middle of add 51 | # run_menu([ENTER, CTRL_X, 'c', 't', SPACE, TAB, SPACE, TAB, SPACE, TAB, 52 | # SPACE, TAB, SPACE, TAB, SPACE, TAB, SPACE, TAB, TAB, SPACE, TAB, 53 | # SPACE, TAB, TAB, ENTER, DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, 54 | # DOWN, DOWN, DOWN, DOWN, DOWN, LEFT, BACKSPACE, '3', TAB, TAB, 55 | # ENTER, ENTER, TAB, ENTER, ENTER, TAB, ENTER, CTRL_Q]) 56 | # configure - instances add (add an instance of rq_worker) 57 | # run_menu([ENTER, CTRL_X, 'c', 't', SPACE, TAB, SPACE, TAB, SPACE, TAB, 58 | # SPACE, TAB, SPACE, TAB, SPACE, TAB, SPACE, TAB, TAB, SPACE, TAB, 59 | # SPACE, TAB, TAB, ENTER, DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, 60 | # DOWN, DOWN, DOWN, DOWN, DOWN, LEFT, BACKSPACE, '3', TAB, TAB, 61 | # ENTER, ENTER, TAB, ENTER, ENTER, TAB, ENTER, TAB, TAB, ENTER]) 62 | # configure - quit in the middle of delete 63 | # run_menu([ENTER, CTRL_X, 'c', 't', SPACE, TAB, SPACE, TAB, SPACE, TAB, 64 | # SPACE, TAB, SPACE, TAB, SPACE, TAB, SPACE, TAB, TAB, SPACE, TAB, 65 | # SPACE, TAB, SPACE, TAB, TAB, ENTER, DOWN, DOWN, DOWN, DOWN, DOWN, 66 | # DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, LEFT, BACKSPACE, '2', 67 | # TAB, TAB, ENTER, ENTER, TAB, ENTER, CTRL_Q]) 68 | # configure - instances delete (delete an instance of file_drop) 69 | # run_menu([ENTER, CTRL_X, 'c', 't', SPACE, TAB, SPACE, TAB, SPACE, TAB, 70 | # SPACE, TAB, SPACE, TAB, SPACE, TAB, SPACE, TAB, TAB, SPACE, TAB, 71 | # SPACE, TAB, SPACE, TAB, TAB, ENTER, DOWN, DOWN, DOWN, DOWN, DOWN, 72 | # DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, DOWN, LEFT, BACKSPACE, '2', 73 | # TAB, TAB, ENTER, ENTER, TAB, ENTER, SPACE, TAB, TAB, ENTER]) 74 | 75 | # go through the plugins menus 76 | run_menu([ENTER, CTRL_X, 'p', 'a', TAB, TAB, TAB, TAB, TAB, TAB, TAB, TAB, 77 | RIGHT, ENTER, SPACE, TAB, TAB, TAB, TAB, TAB, TAB, TAB, ENTER, 78 | SPACE, TAB, SPACE, TAB, SPACE, TAB, TAB, SPACE, TAB, SPACE, TAB, 79 | TAB, ENTER, ENTER, ENTER]) 80 | cmds = [ENTER, CTRL_X, 'p', 'a', TAB, TAB, TAB, 'alpine', TAB, TAB, TAB, 81 | TAB, TAB, TAB, ENTER, ENTER, ENTER] 82 | cmds += (43 * [BACKSPACE]) 83 | cmds += [TAB, TAB, TAB, BACKSPACE, BACKSPACE, BACKSPACE, BACKSPACE, 84 | BACKSPACE, BACKSPACE, TAB, TAB, TAB, TAB, TAB, TAB, ENTER, ENTER, 85 | ENTER, CTRL_Q] 86 | run_menu(cmds) 87 | run_menu([ENTER, CTRL_X, 'p', 'a', TAB, TAB, TAB, 'alpine', TAB, 'alpine', 88 | TAB, TAB, TAB, TAB, TAB, ENTER, ENTER, ENTER, TAB, TAB, ENTER, 89 | ENTER, ENTER]) 90 | run_menu([ENTER, CTRL_X, 'p', 'a', CTRL_T, CTRL_T, TAB, TAB, TAB, TAB, TAB, 91 | TAB, TAB, TAB, ENTER]) 92 | run_menu([ENTER, CTRL_X, 'p', 'i', CTRL_T]) 93 | run_menu([ENTER, CTRL_X, 'p', 's', TAB, TAB, RIGHT, ENTER, ENTER, ENTER, 94 | ENTER, ENTER]) 95 | run_menu([ENTER, CTRL_X, 'p', 's', TAB, TAB, ENTER]) 96 | run_menu([ENTER, CTRL_X, 'p', 's', CTRL_Q]) 97 | run_menu([ENTER, CTRL_X, 'p', 's', CTRL_T]) 98 | # services running - plugin services 99 | run_menu([ENTER, CTRL_X, 'p', 'p', RIGHT, ENTER, ENTER, ENTER]) 100 | run_menu([ENTER, CTRL_X, 'p', 'p', ENTER]) 101 | run_menu([ENTER, CTRL_X, 'p', 'p', CTRL_Q]) 102 | run_menu([ENTER, CTRL_X, 'p', 'p', CTRL_T]) 103 | run_menu([ENTER, CTRL_X, 'p', 'r', TAB, TAB, RIGHT, ENTER, ENTER, ENTER]) 104 | run_menu([ENTER, CTRL_X, 'p', 'r', ENTER]) 105 | run_menu([ENTER, CTRL_X, 'p', 'r', CTRL_Q]) 106 | run_menu([ENTER, CTRL_X, 'p', 'r', CTRL_T]) 107 | 108 | # go through the services running menus 109 | run_menu([ENTER, CTRL_X, 's', 'e', CTRL_T]) 110 | 111 | # go through the system commands menus 112 | # causes .coverage file to not exist 113 | # run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 'r', TAB, RIGHT, 114 | # ENTER, ENTER, ENTER]) 115 | # system commands - backup 116 | run_menu([ENTER, CTRL_X, 'y', 'b', ENTER, ENTER]) 117 | # system commands - configure - cancel 118 | run_menu([ENTER, CTRL_X, 'y', 'c', TAB, ENTER, ENTER, ENTER]) 119 | # system commands - configure - ok 120 | run_menu([ENTER, CTRL_X, 'y', 'c', TAB, TAB, ENTER, ENTER, ENTER]) 121 | run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 'g', ENTER, ENTER]) 122 | run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 's']) 123 | run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 'u']) 124 | run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 'b', ENTER, ENTER]) 125 | run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 't', SPACE, TAB, 126 | ENTER]) 127 | run_menu([ENTER, CTRL_X, DOWN, DOWN, DOWN, DOWN, ENTER, 't', SPACE, TAB, 128 | TAB, ENTER, ENTER, ENTER]) 129 | 130 | # go through the tutorials menus 131 | run_menu([ENTER, CTRL_X, 't', 'v', 'b', RIGHT, ENTER]) 132 | run_menu([ENTER, CTRL_X, 't', 'v', 't', RIGHT, ENTER]) 133 | run_menu([ENTER, CTRL_X, 't', 'v', 's', RIGHT, ENTER]) 134 | run_menu([ENTER, CTRL_X, 't', 'c', 'c', RIGHT, ENTER]) 135 | run_menu([ENTER, CTRL_X, 't', 'p', 'a', RIGHT, ENTER]) 136 | run_menu([ENTER, CTRL_X, 't', 'f', 'a', RIGHT, ENTER]) 137 | run_menu([ENTER, CTRL_X, 't', 's', 't', RIGHT, ENTER]) 138 | 139 | # exit 140 | # causes .coverage file to not exist 141 | # run_menu([ENTER, CTRL_Q]) 142 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_api_image.py: -------------------------------------------------------------------------------- 1 | from vent.api.image import Image 2 | from vent.api.system import System 3 | 4 | 5 | def test_update(): 6 | """ Test the update class """ 7 | image = Image(System().manifest) 8 | image.update('foo') 9 | -------------------------------------------------------------------------------- /tests/unit/test_api_repository.py: -------------------------------------------------------------------------------- 1 | from vent.api.repository import Repository 2 | from vent.api.system import System 3 | 4 | 5 | def test_add(): 6 | """ Test the add function """ 7 | repo = Repository(System().manifest) 8 | repo.add('https://github.com/cyberreboot/poseidon') 9 | 10 | 11 | def test_update(): 12 | """ Test the update function """ 13 | repo = Repository(System().manifest) 14 | repo.update('foo') 15 | -------------------------------------------------------------------------------- /tests/unit/test_api_system.py: -------------------------------------------------------------------------------- 1 | from vent.api.system import System 2 | 3 | 4 | def test_configure(): 5 | """ Test the configure function """ 6 | system = System() 7 | system.configure() 8 | 9 | 10 | def test_gpu(): 11 | """ Test the gpu function """ 12 | system = System() 13 | system.gpu() 14 | 15 | 16 | def test_history(): 17 | """ Test the history function """ 18 | system = System() 19 | system.history() 20 | 21 | 22 | def test_restore(): 23 | """ Test the restore function """ 24 | system = System() 25 | system.restore('foo') 26 | 27 | 28 | def test_rollback(): 29 | """ Test the rollback function """ 30 | system = System() 31 | system.rollback() 32 | 33 | 34 | def test_stop(): 35 | """ Test the stop function """ 36 | system = System() 37 | system.stop() 38 | 39 | 40 | def test_upgrade(): 41 | """ Test the upgrade function """ 42 | system = System() 43 | system.upgrade() 44 | -------------------------------------------------------------------------------- /tests/unit/test_api_tools.py: -------------------------------------------------------------------------------- 1 | from vent.api.tools import Tools 2 | 3 | 4 | def test_configure(): 5 | """ Test the configure function """ 6 | tools = Tools() 7 | tools.configure('foo') 8 | 9 | 10 | def test_new(): 11 | """ Test the new function """ 12 | tools = Tools() 13 | tools.new('image', 'redis') 14 | tools.new('core', '') 15 | tools.new('repo', 'https://github.com/cyberreboot/vent') 16 | 17 | 18 | def test_inventory(): 19 | """ Test the inventory function """ 20 | tools = Tools() 21 | tools.inventory() 22 | 23 | 24 | def test_stop(): 25 | """ Test the stop function """ 26 | tools = Tools() 27 | tools.stop('https://github.com/cyberreboot/vent', 'rabbitmq') 28 | 29 | 30 | def test_remove(): 31 | """ Test the remove function """ 32 | tools = Tools() 33 | tools.remove('https://github.com/cyberreboot/vent', 'rabbitmq') 34 | -------------------------------------------------------------------------------- /tests/unit/test_helpers_meta.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | import docker 5 | import requests 6 | 7 | from vent.helpers.meta import Containers 8 | from vent.helpers.meta import Cpu 9 | from vent.helpers.meta import Docker 10 | from vent.helpers.meta import Gpu 11 | from vent.helpers.meta import Images 12 | from vent.helpers.meta import Jobs 13 | from vent.helpers.meta import ManifestTools 14 | from vent.helpers.meta import ParsedSections 15 | from vent.helpers.meta import Services 16 | from vent.helpers.meta import System 17 | from vent.helpers.meta import Timestamp 18 | from vent.helpers.meta import Uptime 19 | from vent.helpers.meta import Version 20 | from vent.helpers.paths import PathDirs 21 | 22 | 23 | def test_run_containers(): 24 | """ Run some containers for testing purposes """ 25 | d_client = docker.from_env() 26 | d_client.containers.run('alpine:latest', 'tail -f /etc/passwd', 27 | detach=True, labels=['vent', 'vent-plugins']) 28 | 29 | 30 | def test_version(): 31 | """ Test the version function """ 32 | version = Version() 33 | # currently broken due to something that urllib is doing 34 | assert version.startswith('v') 35 | 36 | 37 | def test_system(): 38 | """ Test the system function """ 39 | system = System() 40 | assert system != '' 41 | 42 | 43 | def test_docker(): 44 | """ Test the docker function """ 45 | docker = Docker() 46 | assert isinstance(docker, dict) 47 | assert isinstance(docker['server'], dict) 48 | os.environ['DOCKER_MACHINE_NAME'] = 'foo' 49 | docker = Docker() 50 | assert isinstance(docker, dict) 51 | assert docker['type'] == 'docker-machine' 52 | del os.environ['DOCKER_MACHINE_NAME'] 53 | os.environ['DOCKER_HOST'] = 'foo' 54 | docker = Docker() 55 | assert isinstance(docker, dict) 56 | assert docker['type'] == 'remote' 57 | del os.environ['DOCKER_HOST'] 58 | 59 | 60 | def test_containers(): 61 | """ Test the containers function """ 62 | containers = Containers() 63 | containers = Containers(vent=False) 64 | assert isinstance(containers, list) 65 | 66 | 67 | def test_images(): 68 | """ Test the images function """ 69 | images = Images() 70 | images = Images(vent=False) 71 | assert isinstance(images, list) 72 | 73 | 74 | def test_tools(): 75 | """ Test the tools function """ 76 | tools = ManifestTools() 77 | assert isinstance(tools, list) 78 | 79 | 80 | def test_services(): 81 | """ Test the services function """ 82 | services = Services(True) 83 | assert isinstance(services, list) 84 | services = Services(False) 85 | assert isinstance(services, list) 86 | services = Services(True, vent=False) 87 | assert isinstance(services, list) 88 | 89 | 90 | def test_timestamp(): 91 | """ Test the timestamp function """ 92 | timestamp = Timestamp() 93 | assert isinstance(timestamp, str) 94 | 95 | 96 | def test_uptime(): 97 | """ Test the uptime function """ 98 | uptime = Uptime() 99 | assert isinstance(uptime, str) 100 | 101 | 102 | def test_cpu(): 103 | """ Test the cpu function """ 104 | cpu = Cpu() 105 | assert isinstance(cpu, str) 106 | 107 | 108 | def test_gpu(): 109 | """ Test the gpu function """ 110 | gpu = Gpu() 111 | assert isinstance(gpu, tuple) 112 | gpu = Gpu(pull=True) 113 | assert isinstance(gpu, tuple) 114 | 115 | 116 | def test_jobs(): 117 | """ Test the jobs function """ 118 | jobs = Jobs() 119 | assert isinstance(jobs, tuple) 120 | path = PathDirs() 121 | status = path.host_config() 122 | assert isinstance(status, tuple) 123 | assert status[0] 124 | # run test job 125 | with open('/opt/vent_files/foo.matrix', 'w') as f: 126 | f.write('24,23\n10,22') 127 | pcap = 'https://s3.amazonaws.com/tcpreplay-pcap-files/test.pcap' 128 | r = requests.get(pcap, stream=True) 129 | 130 | if r.status_code == 200: 131 | with open('/opt/vent_files/foo.pcap', 'wb') as f: 132 | r.raw.decode_content = True 133 | shutil.copyfileobj(r.raw, f) 134 | services = Services(True) 135 | assert isinstance(services, list) 136 | jobs = Jobs() 137 | assert isinstance(jobs, tuple) 138 | 139 | 140 | def test_parsed_sections(): 141 | """ Test the ParsedSections function """ 142 | test_val = '[docker]\nenvironment = ["TEST_VAR=5", "RANDOM_VAR=10"]\ntest = yes' 143 | template_dict = ParsedSections(test_val) 144 | assert isinstance(template_dict, dict) 145 | assert len(template_dict) == 1 146 | assert 'docker' in template_dict 147 | assert len(template_dict['docker']) == 2 148 | assert 'environment' in template_dict['docker'] 149 | assert template_dict['docker']['environment'] == '["TEST_VAR=5", "RANDOM_VAR=10"]' 150 | assert 'test' in template_dict['docker'] 151 | assert template_dict['docker']['test'] == 'yes' 152 | -------------------------------------------------------------------------------- /tests/unit/test_helpers_paths.py: -------------------------------------------------------------------------------- 1 | from vent.helpers.paths import PathDirs 2 | 3 | 4 | def test_pathdirs(): 5 | """ Test the pathdirs class """ 6 | path = PathDirs() 7 | path.host_config() 8 | 9 | 10 | def test_ensure_file(): 11 | """ Test the ensure_file function """ 12 | paths = PathDirs() 13 | status = paths.ensure_file(paths.init_file) 14 | assert isinstance(status, tuple) 15 | assert status[0] == True 16 | -------------------------------------------------------------------------------- /tests/unit/test_helpers_templates.py: -------------------------------------------------------------------------------- 1 | from vent.helpers.templates import Template 2 | 3 | 4 | def test_options(): 5 | """ Test the options function """ 6 | instance = Template() 7 | instance.options('foo') 8 | 9 | 10 | def test_option(): 11 | """ Test the option function """ 12 | instance = Template() 13 | instance.option('foo', 'bar') 14 | 15 | 16 | def test_add_option(): 17 | """ Test the add_option function """ 18 | instance = Template() 19 | instance.add_option('foo', 'bar') 20 | instance.add_option('foo', 'bar') 21 | instance.add_option('bad', 'x') 22 | 23 | 24 | def test_del_option(): 25 | """ Test the del_option function """ 26 | instance = Template() 27 | instance.del_option('foo', 'bar') 28 | instance.add_option('foo', 'bar2') 29 | instance.del_option('foo', 'bar2') 30 | instance.del_option('foo', 'bar2') 31 | 32 | 33 | def test_del_section(): 34 | """ Test the del_section function """ 35 | instance = Template() 36 | instance.del_section('foo') 37 | instance.add_option('foo', 'bar') 38 | instance.del_section('foo') 39 | 40 | 41 | def test_set_option(): 42 | """ Test the set_option function """ 43 | instance = Template() 44 | instance.set_option('bad', 'x', 'x') 45 | -------------------------------------------------------------------------------- /vent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/__init__.py -------------------------------------------------------------------------------- /vent/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/api/__init__.py -------------------------------------------------------------------------------- /vent/api/image.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import docker 4 | 5 | from vent.helpers.logs import Logger 6 | from vent.helpers.templates import Template 7 | 8 | 9 | class Image: 10 | 11 | def __init__(self, manifest): 12 | self.manifest = manifest 13 | self.d_client = docker.from_env() 14 | self.logger = Logger(__name__) 15 | 16 | def add(self, image, link_name, tag=None, registry=None, groups=None): 17 | status = (True, None) 18 | try: 19 | pull_name = image 20 | org = '' 21 | name = image 22 | if '/' in image: 23 | org, name = image.split('/') 24 | else: 25 | org = 'official' 26 | if not tag: 27 | tag = 'latest' 28 | if not registry: 29 | registry = 'docker.io' 30 | if not link_name: 31 | link_name = name 32 | if not groups: 33 | groups = '' 34 | full_image = registry + '/' + image + ':' + tag 35 | image = self.d_client.images.pull(full_image) 36 | section = ':'.join([registry, org, name, '', tag]) 37 | namespace = org + '/' + name 38 | 39 | # set template section and options for tool at version and branch 40 | template = Template(template=self.manifest) 41 | template.add_section(section) 42 | template.set_option(section, 'name', name) 43 | template.set_option(section, 'pull_name', pull_name) 44 | template.set_option(section, 'namespace', namespace) 45 | template.set_option(section, 'path', '') 46 | template.set_option(section, 'repo', registry + '/' + org) 47 | template.set_option(section, 'branch', '') 48 | template.set_option(section, 'version', tag) 49 | template.set_option(section, 'last_updated', 50 | str(datetime.utcnow()) + ' UTC') 51 | template.set_option(section, 'image_name', 52 | image.attrs['RepoTags'][0]) 53 | template.set_option(section, 'type', 'registry') 54 | template.set_option(section, 'link_name', link_name) 55 | template.set_option(section, 'commit_id', '') 56 | template.set_option(section, 'built', 'yes') 57 | template.set_option(section, 'image_id', 58 | image.attrs['Id'].split(':')[1][:12]) 59 | template.set_option(section, 'groups', groups) 60 | 61 | # write out configuration to the manifest file 62 | template.write_config() 63 | status = (True, 'Successfully added ' + full_image) 64 | except Exception as e: # pragma: no cover 65 | self.logger.error( 66 | 'Failed to add image because: {0}'.format(str(e))) 67 | status = (False, str(e)) 68 | return status 69 | 70 | def update(self, image): 71 | # TODO 72 | return 73 | -------------------------------------------------------------------------------- /vent/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/core/__init__.py -------------------------------------------------------------------------------- /vent/core/file_drop/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="file_drop" \ 5 | vent.groups="core,files" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | RUN apk add --update \ 10 | curl \ 11 | python3 \ 12 | py3-pip \ 13 | && rm -rf /var/cache/apk/* 14 | 15 | RUN pip3 install --no-cache-dir --upgrade pip==19.1 16 | 17 | # healthcheck 18 | COPY healthcheck /healthcheck 19 | RUN pip3 install --no-cache-dir -r /healthcheck/requirements.txt 20 | ENV FLASK_APP /healthcheck/hc.py 21 | HEALTHCHECK --interval=15s --timeout=15s \ 22 | CMD curl --silent --fail http://localhost:5000/healthcheck || exit 1 23 | 24 | COPY . /file-drop 25 | RUN pip3 install --no-cache-dir -r /file-drop/requirements.txt 26 | 27 | WORKDIR /file-drop 28 | 29 | CMD (flask run > /dev/null 2>&1) & (python3 /file-drop/file_drop.py) 30 | -------------------------------------------------------------------------------- /vent/core/file_drop/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/core/file_drop/__init__.py -------------------------------------------------------------------------------- /vent/core/file_drop/file_drop.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import time 5 | import uuid 6 | 7 | import pika 8 | from redis import Redis 9 | from redis import StrictRedis 10 | from rq import Queue 11 | from watchdog.events import PatternMatchingEventHandler 12 | from watchdog.observers.polling import PollingObserver 13 | 14 | 15 | class GZHandler(PatternMatchingEventHandler): 16 | """ 17 | Handles when an event on the directory being watched happens that matches 18 | the values in patterns 19 | """ 20 | 21 | patterns = ['*'] 22 | # want to ignore certain pcap files from splitter as they contain junk 23 | ignore_patterns = ['*-miscellaneous*'] 24 | 25 | # don't want to process files in on_modified for files that have already 26 | # been created and processed 27 | created_files = set() 28 | try: 29 | # let jobs run for up to one day 30 | q = Queue(connection=Redis(host='redis'), default_timeout=86400) 31 | r = StrictRedis(host='redis', port=6379, db=0) 32 | except Exception as e: # pragma: no cover 33 | print('Unable to connect to redis:', str(e)) 34 | 35 | def process(self, event): 36 | """ 37 | event.event_type 38 | 'modified' | 'created' | 'moved' | 'deleted' 39 | event.is_directory 40 | True | False 41 | event.src_path 42 | path/to/observed/file 43 | """ 44 | uid = str(uuid.uuid4()) 45 | hostname = os.environ.get('VENT_HOST') 46 | if not hostname: 47 | hostname = '' 48 | 49 | try: 50 | # TODO should directories be treated as bulk paths to send to a 51 | # plugin? 52 | if not event.is_directory: 53 | spath = event.src_path 54 | # wait for long copies to finish 55 | historicalSize = -1 56 | while (historicalSize != os.path.getsize(spath)): 57 | historicalSize = os.path.getsize(spath) 58 | time.sleep(0.1) 59 | 60 | spath = str(spath) 61 | if spath.startswith('/files/trace_'): 62 | key = spath.split('_')[1] 63 | # Rabbit settings 64 | exchange = 'topic-poseidon-internal' 65 | exchange_type = 'topic' 66 | routing_key = 'poseidon.algos.decider' 67 | 68 | message = {} 69 | if os.path.getsize(spath) == 0: 70 | print( 71 | 'file drop ignoring empty file: {0}'.format(spath)) 72 | message[key] = { 73 | 'valid': False, 'source': 'file_drop', 'plugin': 'ncapture', 'file': spath} 74 | else: 75 | message[key] = { 76 | 'valid': True, 'source': 'file_drop', 'plugin': 'ncapture', 'file': spath} 77 | message = json.dumps(message) 78 | 79 | # Send Rabbit message 80 | try: 81 | connection = pika.BlockingConnection( 82 | pika.ConnectionParameters(host='rabbit') 83 | ) 84 | 85 | channel = connection.channel() 86 | channel.exchange_declare( 87 | exchange=exchange, exchange_type=exchange_type 88 | ) 89 | channel.basic_publish(exchange=exchange, 90 | routing_key=routing_key, 91 | body=message) 92 | connection.close() 93 | except Exception as e: 94 | print('failed to send rabbit message because: ' + 95 | str(e)) 96 | 97 | if os.path.getsize(spath) == 0: 98 | return 99 | 100 | # check if the file was already queued and ignore 101 | exists = False 102 | print(uid + ' started ' + spath) 103 | jobs = self.r.keys(pattern='rq:job*') 104 | for job in jobs: 105 | print(uid + ' ***') 106 | description = self.r.hget( 107 | job, 'description').decode('utf-8') 108 | print(uid + ' ' + description) 109 | if description.startswith("watch.file_queue('"): 110 | description = description.split( 111 | "watch.file_queue('")[1] 112 | print(uid + ' ' + 113 | description.split('_')[1][:-2]) 114 | print(uid + ' ' + spath) 115 | if description.split('_')[1][:-2] == spath: 116 | print(uid + ' true') 117 | exists = True 118 | print(uid + ' ***') 119 | 120 | if not exists: 121 | # !! TODO this should be a configuration option in the 122 | # vent.template 123 | print(uid + " let's queue it " + spath) 124 | # let jobs be queued for up to 30 days 125 | self.q.enqueue('watch.file_queue', 126 | hostname + '_' + spath, 127 | ttl=2592000) 128 | print(uid + ' end ' + spath) 129 | except Exception as e: # pragma: no cover 130 | print('file drop error: ' + str(e)) 131 | 132 | def on_created(self, event): 133 | self.created_files.add(event.src_path) 134 | self.process(event) 135 | 136 | def on_modified(self, event): 137 | # don't perform any action if file was already created or file is 138 | # deleted 139 | if (event.src_path not in self.created_files and 140 | os.path.exists(event.src_path)): 141 | # add to created files because the file was moved into directory, 142 | # which is what should be creating it, but some OS's treat it as a 143 | # modification with docker mounts 144 | self.created_files.add(event.src_path) 145 | self.process(event) 146 | 147 | 148 | if __name__ == '__main__': # pragma: no cover 149 | args = None 150 | if len(sys.argv) > 1: 151 | args = sys.argv[1:] 152 | 153 | # TODO: counter-intuitively inotify observer uses a lot of resources 154 | # for directories that have a lot of existing files. Rather than 155 | # just adding more watchers, we should encourage archiving of old 156 | # pcaps and use a polling observer instead so that file_drop 157 | # doesn't silently break. 158 | observer = PollingObserver() 159 | observer.schedule(GZHandler(), path=args[0] if args else '/files', 160 | recursive=True) 161 | observer.start() 162 | 163 | try: 164 | while True: 165 | time.sleep(1) 166 | except KeyboardInterrupt: # pragma: no cover 167 | observer.stop() 168 | 169 | observer.join() 170 | -------------------------------------------------------------------------------- /vent/core/file_drop/healthcheck/hc.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from healthcheck import EnvironmentDump 4 | from healthcheck import HealthCheck 5 | 6 | app = Flask(__name__) 7 | 8 | health = HealthCheck(app, '/healthcheck') 9 | envdump = EnvironmentDump(app, '/environment') 10 | 11 | 12 | def application_data(): 13 | return {'maintainer': 'Charlie Lewis', 14 | 'git_repo': 'https://github.com/CyberReboot/vent', 15 | 'app': 'file_drop'} 16 | 17 | 18 | envdump.add_section('application', application_data) 19 | -------------------------------------------------------------------------------- /vent/core/file_drop/healthcheck/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==1.1.1 2 | healthcheck==1.3.3 3 | six==1.13.0 4 | -------------------------------------------------------------------------------- /vent/core/file_drop/requirements.txt: -------------------------------------------------------------------------------- 1 | pika==1.1.0 2 | redis==3.3.11 3 | rq==1.1.0 4 | watchdog==0.9.0 5 | -------------------------------------------------------------------------------- /vent/core/file_drop/test_file_drop.py: -------------------------------------------------------------------------------- 1 | from redis import Redis 2 | from redis import StrictRedis 3 | from rq import Queue 4 | 5 | from .file_drop import GZHandler 6 | 7 | 8 | def test_file_drop_GZHandler(): 9 | """ Tests the GZZHandler for file drop """ 10 | a = GZHandler() 11 | 12 | class Event: 13 | """ Creates a mock event object for tests """ 14 | event_type = None 15 | src_path = None 16 | is_directory = None 17 | q = None 18 | r = None 19 | 20 | def __init__(self, event_type, src_path, is_directory): 21 | """ initializes necessary variables for the object """ 22 | self.event_type = event_type 23 | self.src_path = src_path 24 | self.is_directory = is_directory 25 | self.q = Queue(connection=Redis(host='localhost'), 26 | default_timeout=86400) 27 | self.r = StrictRedis(host='localhost', port=6379, db=0) 28 | 29 | b = Event('created', '/dev/null', False) 30 | c = Event('modified', '/etc/hosts', False) 31 | a.process(b) 32 | a.process(b) 33 | a.process(b) 34 | a.on_created(b) 35 | a.on_modified(c) 36 | -------------------------------------------------------------------------------- /vent/core/file_drop/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = FileDrop 3 | groups = core,files 4 | 5 | [docker] 6 | environment = ["PYTHONUNBUFFERED=0", "VENT_HOST=`hostname`"] 7 | links = {"Redis":"redis", "RabbitMQ":"rabbit"} 8 | -------------------------------------------------------------------------------- /vent/core/rabbitmq/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rabbitmq:3-management-alpine 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="rabbitmq" \ 5 | vent.groups="core,messages,syslog" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | RUN apk add --update curl && rm -rf /var/cache/apk/* 10 | 11 | HEALTHCHECK --interval=15s --timeout=15s \ 12 | CMD curl --silent --fail http://localhost:15672/ || exit 1 13 | -------------------------------------------------------------------------------- /vent/core/rabbitmq/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = RabbitMQ 3 | groups = core,messages,syslog 4 | 5 | [settings] 6 | priority = 1 7 | 8 | [docker] 9 | ports = {'15672/tcp': ('127.0.0.1', None), '5672/tcp': ('0.0.0.0', 5672)} 10 | -------------------------------------------------------------------------------- /vent/core/redis/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM redis:5-alpine 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="redis" \ 5 | vent.groups="core" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | HEALTHCHECK --interval=15s --timeout=15s \ 10 | CMD redis-cli ping 11 | CMD redis-server --appendonly yes 12 | -------------------------------------------------------------------------------- /vent/core/redis/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = Redis 3 | groups = core 4 | 5 | [settings] 6 | instances = 1 7 | priority = 3 8 | 9 | [docker] 10 | volumes = {'/opt/redis': {'bind': '/data', 'mode': 'rw'}} 11 | -------------------------------------------------------------------------------- /vent/core/rq_worker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="rq_worker" \ 5 | vent.groups="core,files" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | RUN apk add --update \ 10 | curl \ 11 | git \ 12 | libmagic \ 13 | python3 \ 14 | py3-pip \ 15 | && rm -rf /var/cache/apk/* 16 | 17 | RUN pip3 install --no-cache-dir --upgrade pip==19.1 18 | 19 | # healthcheck 20 | COPY healthcheck /healthcheck 21 | RUN pip3 install --no-cache-dir -r /healthcheck/requirements.txt 22 | ENV FLASK_APP /healthcheck/hc.py 23 | HEALTHCHECK --interval=15s --timeout=15s \ 24 | CMD curl --silent --fail http://localhost:5000/healthcheck || exit 1 25 | 26 | COPY requirements.txt requirements.txt 27 | RUN pip3 install --no-cache-dir -r requirements.txt 28 | RUN pip3 install --no-cache-dir git+git://github.com/cyberreboot/vent.git@master 29 | COPY *.py / 30 | 31 | CMD (flask run > /dev/null 2>&1) & (rqworker -c settings) 32 | -------------------------------------------------------------------------------- /vent/core/rq_worker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/core/rq_worker/__init__.py -------------------------------------------------------------------------------- /vent/core/rq_worker/healthcheck/hc.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from healthcheck import EnvironmentDump 4 | from healthcheck import HealthCheck 5 | 6 | app = Flask(__name__) 7 | 8 | health = HealthCheck(app, '/healthcheck') 9 | envdump = EnvironmentDump(app, '/environment') 10 | 11 | 12 | def application_data(): 13 | return {'maintainer': 'Charlie Lewis', 14 | 'git_repo': 'https://github.com/CyberReboot/vent', 15 | 'app': 'rq_worker'} 16 | 17 | 18 | envdump.add_section('application', application_data) 19 | -------------------------------------------------------------------------------- /vent/core/rq_worker/healthcheck/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==1.1.1 2 | healthcheck==1.3.3 3 | six==1.13.0 4 | -------------------------------------------------------------------------------- /vent/core/rq_worker/requirements.txt: -------------------------------------------------------------------------------- 1 | docker==4.1.0 2 | python-magic==0.4.15 3 | redis==3.3.11 4 | rq==1.1.0 5 | vent==0.10.1 6 | -------------------------------------------------------------------------------- /vent/core/rq_worker/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | REDIS_HOST = os.environ['REMOTE_REDIS_HOST'] 3 | REDIS_PORT = os.environ['REMOTE_REDIS_PORT'] 4 | -------------------------------------------------------------------------------- /vent/core/rq_worker/test_watch.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from .watch import file_queue 5 | from .watch import gpu_queue 6 | from vent.helpers.paths import PathDirs 7 | 8 | 9 | def test_settings(): 10 | """ Tests settings """ 11 | os.environ['REMOTE_REDIS_HOST'] = 'localhost' 12 | os.environ['REMOTE_REDIS_PORT'] = '6379' 13 | from . import settings 14 | 15 | 16 | def test_file_queue(): 17 | """ Tests simulation of new file """ 18 | path_dirs = PathDirs() 19 | images = file_queue('/tmp/foo') 20 | assert not images[0] 21 | images = file_queue('host_/tmp/foo', 22 | template_path=path_dirs.base_dir, 23 | r_host='localhost') 24 | assert isinstance(images, tuple) 25 | assert images[0] 26 | assert isinstance(images[1], list) 27 | images = file_queue('host_/tmp/foo.matrix', 28 | template_path=path_dirs.base_dir, 29 | r_host='localhost') 30 | assert isinstance(images, tuple) 31 | assert images[0] 32 | assert isinstance(images[1], list) 33 | 34 | 35 | def test_gpu_queue(): 36 | """ Tests simulation of gpu job """ 37 | options = json.dumps({'configs': {'devices': ['foo0', 'bar', 'baz3'], 'gpu_options': { 38 | 'device': '0'}}, 'labels': {}, 'image': 'alpine:latest'}) 39 | status = gpu_queue(options) 40 | assert isinstance(status, tuple) 41 | -------------------------------------------------------------------------------- /vent/core/rq_worker/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = RQ-Worker 3 | groups = core,files 4 | 5 | [settings] 6 | instances = 2 7 | priority = 6 8 | 9 | [docker] 10 | environment = ["REMOTE_REDIS_HOST=redis", "REMOTE_REDIS_PORT=6379", "PYTHONUNBUFFERED=0"] 11 | volumes = {'/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'}} 12 | links = {"Redis":"redis"} 13 | -------------------------------------------------------------------------------- /vent/core/syslog/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stretch-slim 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="syslog" \ 5 | vent.groups="core,logging,syslog" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | 10 | RUN apt-get update -qq && apt-get install -y syslog-ng telnet && \ 11 | apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 12 | 13 | COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf 14 | EXPOSE 514 15 | 16 | HEALTHCHECK --interval=15s --timeout=15s \ 17 | CMD logger -p cron.info "Testing health of syslog" 18 | 19 | ENTRYPOINT ["/usr/sbin/syslog-ng", "-F", "-f", "/etc/syslog-ng/syslog-ng.conf"] 20 | -------------------------------------------------------------------------------- /vent/core/syslog/syslog-ng.conf: -------------------------------------------------------------------------------- 1 | @version: 3.8 2 | 3 | options { 4 | use_dns(no); 5 | keep_hostname(yes); 6 | create_dirs(yes); 7 | ts_format(iso); 8 | log_fifo_size(5000); 9 | }; 10 | 11 | source s_net { 12 | tcp(ip(0.0.0.0) 13 | log-iw-size(50000) 14 | port(514) 15 | max-connections(50)); 16 | udp(ip(0.0.0.0) 17 | port(514)); 18 | unix-stream("/var/run/syslog-ng/syslog-ng.sock" max-connections(50)); 19 | }; 20 | 21 | destination logfiles { 22 | file("/var/log/syslog-ng/$YEAR/$MONTH/$DAY/$PROGRAM.log"); 23 | file("/dev/stdout"); 24 | }; 25 | 26 | destination d_amqp { 27 | amqp( 28 | vhost("/") 29 | host("rabbitmq") 30 | port(5672) 31 | exchange("topic_recs") 32 | exchange_type("topic") 33 | routing_key("syslog.$PROGRAM.`VENT_HOST`.$UNIXTIME") 34 | body("${MESSAGE}") 35 | retries(3) 36 | persistent(yes) 37 | username("guest") 38 | password("guest") 39 | value-pairs( 40 | scope("selected-macros" "nv-pairs" "sdata") 41 | ) 42 | ); 43 | }; 44 | 45 | log { 46 | source(s_net); 47 | destination(logfiles); 48 | destination(d_amqp); 49 | }; 50 | -------------------------------------------------------------------------------- /vent/core/syslog/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = Syslog 3 | groups = core,logging,syslog 4 | 5 | [service] 6 | uri_prefix1 = tcp:// 7 | 8 | [settings] 9 | instances = 1 10 | priority = 2,1,2 11 | 12 | [docker] 13 | environment = ["VENT_HOST=`hostname`"] 14 | volumes = {'/var/log/vent': {'bind': '/var/log/syslog-ng', 'mode': 'rw'}} 15 | ports = {'514/tcp': ('127.0.0.1', 514)} 16 | links = {"RabbitMQ":"rabbitmq"} 17 | -------------------------------------------------------------------------------- /vent/extras/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/elasticsearch/elasticsearch:7.5.1 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="elasticsearch" \ 5 | vent.groups="search" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | HEALTHCHECK --interval=15s --timeout=15s \ 10 | CMD curl --silent --fail http://localhost:9200/_cluster/health || exit 1 11 | -------------------------------------------------------------------------------- /vent/extras/elasticsearch/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = Elasticsearch 3 | groups = search 4 | 5 | [service] 6 | uri_prefix1 = http:// 7 | 8 | [settings] 9 | instances = 1 10 | priority = 10 11 | 12 | [docker] 13 | #command = -Des.network.host=0.0.0.0 14 | ports = {'9200/tcp': ('0.0.0.0', None)} 15 | -------------------------------------------------------------------------------- /vent/extras/gonet/Dockerfile: -------------------------------------------------------------------------------- 1 | # build stage 2 | FROM golang:alpine AS builder 3 | LABEL maintainer="Charlie Lewis " 4 | COPY main.go /app/main.go 5 | WORKDIR /app 6 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o gonet . 7 | 8 | # final stage 9 | FROM scratch 10 | LABEL maintainer="Charlie Lewis " \ 11 | vent="" 12 | COPY --from=builder /app/gonet /gonet 13 | CMD ["/gonet"] 14 | -------------------------------------------------------------------------------- /vent/extras/gonet/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | ) 7 | 8 | func main() { 9 | 10 | interfaces, err := net.Interfaces() 11 | check(err) 12 | 13 | for _, iface := range interfaces { 14 | fmt.Println(iface.Name) 15 | } 16 | 17 | } 18 | 19 | func check(e error) { 20 | if e != nil { 21 | panic(e) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /vent/extras/network_tap/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="network_tap" \ 5 | vent.groups="files,network" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | COPY ncontrol/requirements.txt requirements.txt 10 | COPY healthcheck /healthcheck 11 | 12 | RUN apk add --update \ 13 | curl \ 14 | gcc \ 15 | git \ 16 | linux-headers \ 17 | musl-dev \ 18 | python3 \ 19 | python3-dev \ 20 | && pip3 install --no-cache-dir --upgrade pip==19.1 \ 21 | && pip3 install --no-cache-dir -r requirements.txt \ 22 | && pip3 install --no-cache-dir -r /healthcheck/requirements.txt \ 23 | && apk del \ 24 | gcc \ 25 | git \ 26 | linux-headers \ 27 | musl-dev \ 28 | python3-dev \ 29 | && rm -rf /var/cache/apk/* 30 | 31 | # healthcheck 32 | ENV FLASK_APP /healthcheck/hc.py 33 | HEALTHCHECK --interval=15s --timeout=15s \ 34 | CMD curl --silent --fail http://localhost:5000/healthcheck || exit 1 35 | 36 | COPY . /network-tap 37 | WORKDIR /network-tap 38 | 39 | EXPOSE 8080 40 | 41 | CMD (flask run > /dev/null 2>&1) & (/network-tap/startup.sh) 42 | -------------------------------------------------------------------------------- /vent/extras/network_tap/README.md: -------------------------------------------------------------------------------- 1 | Once this tool is started, it can be interacted with through it's RESTful interface. To find the port it's exposed on, you can go into the shell and search for it using docker commands. 2 | 3 | Once you have the IP and port it's running on, you can make a POST request that looks like this: 4 | 5 | ``` 6 | http://192.168.99.100:32815/create 7 | ``` 8 | 9 | Along with a payload that looks like this: 10 | 11 | ``` 12 | {"nic":"eth1","id":"foo","interval":"300","filter":"","iters":"-1"} 13 | ``` 14 | for a continuously running filter, or like: 15 | ``` 16 | {"nic":"eth1","id":"foo","interval":"300","filter":"","iters":"1"} 17 | ``` 18 | for a filter to run `1` capture. 19 | 20 | In the payload, the `nic` will be the network controller you want to capture on, the `id` can be any unique value, the `interval` is the time in seconds to cut up the captures into for processing, the `filter` is for limiting what gets captured off the network controller, if it's an empty string as in this example, there is no filter applied, and the `iters` is for specifying the number of captures to make (if this is 0 or less then the collector will run until killed, otherwise it will make this many captures). 21 | -------------------------------------------------------------------------------- /vent/extras/network_tap/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/extras/network_tap/__init__.py -------------------------------------------------------------------------------- /vent/extras/network_tap/healthcheck/hc.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from healthcheck import EnvironmentDump 4 | from healthcheck import HealthCheck 5 | 6 | app = Flask(__name__) 7 | 8 | health = HealthCheck(app, '/healthcheck') 9 | envdump = EnvironmentDump(app, '/environment') 10 | 11 | 12 | def application_data(): 13 | return {'maintainer': 'Charlie Lewis', 14 | 'git_repo': 'https://github.com/CyberReboot/vent', 15 | 'app': 'network_tap'} 16 | 17 | 18 | envdump.add_section('application', application_data) 19 | -------------------------------------------------------------------------------- /vent/extras/network_tap/healthcheck/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==1.1.1 2 | healthcheck==1.3.3 3 | six==1.13.0 4 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="ncapture" \ 5 | vent.groups="collection,hidden,network,network_tap_child" 6 | 7 | ENV BUILDDEPS="autoconf automake bison build-base flex gcc git libtool libpcap-dev libpcap linux-headers musl-dev python3-dev yaml-dev" 8 | WORKDIR /tmp 9 | 10 | # TODO: libwdcap currently requires openssl 1.0.2 11 | RUN apk add --update $BUILDDEPS \ 12 | bash \ 13 | coreutils \ 14 | python3 \ 15 | && rm -rf /var/cache/apk/* \ 16 | && mkdir /src \ 17 | && cd /src \ 18 | && git clone https://github.com/wanduow/wandio.git \ 19 | && git clone https://github.com/LibtraceTeam/libtrace.git \ 20 | && git clone https://github.com/openssl/openssl -b OpenSSL_1_0_2s \ 21 | && git clone https://github.com/wanduow/libwdcap.git \ 22 | && cd /src/wandio \ 23 | && ./bootstrap.sh \ 24 | && ./configure \ 25 | && make && make install \ 26 | && cd /src/libtrace \ 27 | && ./bootstrap.sh \ 28 | && ./configure \ 29 | && make && make install \ 30 | && cd /src/openssl \ 31 | && ./config --prefix=/usr/local --openssldir=/usr/local/openssl \ 32 | && make && make install \ 33 | && cd /src/libwdcap \ 34 | && ./bootstrap.sh \ 35 | && ./configure --disable-shared \ 36 | && make && make install \ 37 | && cd examples \ 38 | && g++ -fpermissive -o tracecapd tracecapd.c -L/usr/local/lib -Wl,-Bstatic -ltrace -lwdcap -Wl,-Bdynamic -lpcap -lssl -lcrypto -lwandio -lyaml \ 39 | && cp tracecapd /usr/local/bin \ 40 | && rm -rf /src \ 41 | && apk del $BUILDDEPS \ 42 | && apk add \ 43 | libstdc++ \ 44 | libgcc \ 45 | libpcap \ 46 | yaml 47 | 48 | VOLUME /tmp 49 | COPY . /tmp 50 | 51 | RUN pip3 install --no-cache-dir -r requirements.txt 52 | 53 | CMD ["/tmp/run.sh"] 54 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/VERSION: -------------------------------------------------------------------------------- 1 | 0.10.2.dev 2 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/requirements.txt: -------------------------------------------------------------------------------- 1 | pika==1.1.0 2 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | URI="$1" 4 | INTERVAL="$2" 5 | ID="$3" 6 | ITERS="$4" 7 | FILTER="$5" 8 | # TODO: migrate above static args to getopt style. 9 | 10 | # check if filter has '' surrounding it 11 | if [[ "$FILTER" =~ ^\'.*\'$ ]]; then 12 | FILTER=${FILTER:1:${#FILTER}-2} 13 | fi 14 | 15 | # See https://github.com/wanduow/libwdcap for full flag documentation. 16 | # Set CryptoPAn IP anonymization (https://en.wikipedia.org/wiki/Crypto-PAn) with -a (default none). 17 | ANON="none" 18 | # Set checksum updating for anonymization. 19 | CSUM="check" 20 | # Set number of app payload size to keep in bytes. 21 | PAYS="4" 22 | # Set number of DNS payload size to keep in bytes 23 | DPAYS="12" 24 | 25 | OUT_PATH="/files/" 26 | 27 | while getopts "a:c:d:s:" arg; do 28 | case $arg in 29 | a) 30 | ANON=$OPTARG 31 | ;; 32 | c) 33 | CSUM=$OPTARG 34 | ;; 35 | d) 36 | DPAYS=$OPTARG 37 | ;; 38 | o) 39 | OUT_PATH=$OPTARG 40 | ;; 41 | s) 42 | PAYS=$OPTARG 43 | ;; 44 | esac 45 | done 46 | 47 | CAPTMP=$(mktemp -d) 48 | 49 | make_pcap_name() { 50 | local id=$1 51 | local dt=$(date '+%Y-%m-%d_%H_%M_%S') 52 | echo trace_${id}_${dt}.pcap 53 | } 54 | 55 | run_tracecapd() { 56 | local uri=$1 57 | local name=$2 58 | local interval=$3 59 | local filter=$4 60 | 61 | local dwconf=${CAPTMP}/dw.yaml 62 | local ppconf=${CAPTMP}/pp.yaml 63 | 64 | # default to interface URI if no prefix. 65 | # See https://wand.net.nz/trac/libtrace/wiki/SupportedTraceFormats. 66 | if [[ ! "$uri" =~ .+":".+ ]]; then 67 | uri="int:$uri" 68 | fi 69 | 70 | echo -e "format: pcapfile\nnamingscheme: ${name}\ncompressmethod: none\nrotationperiod: day\n" > $dwconf 71 | echo -e "anon: $ANON\nchecksum: $CSUM\npayload: $PAYS\ndnspayload: $DPAYS\n" > $ppconf 72 | $(timeout -k2 ${interval}s tracecapd -t 1 -c $dwconf -p $ppconf -s $uri -f "$filter") 73 | } 74 | 75 | run_capture() { 76 | local uri=$1 77 | local id=$2 78 | local interval=$3 79 | local filter=$4 80 | local out_path=$5 81 | 82 | local name=$(make_pcap_name $id) 83 | run_tracecapd $uri $name $interval "$filter" 84 | mv $name $out_path; 85 | python3 send_message.py $out_path/$name; 86 | } 87 | 88 | # if ITERS is non-negative then do the capture ITERS times 89 | if [ $ITERS -gt "0" ]; then 90 | COUNTER=0 91 | while [ $COUNTER -lt $ITERS ]; do 92 | run_capture $URI $ID $INTERVAL "$FILTER" $OUT_PATH 93 | let COUNTER=COUNTER+1; 94 | done 95 | else # else do the capture until killed 96 | while true 97 | do 98 | run_capture $URI $ID $INTERVAL "$FILTER" $OUT_PATH 99 | done 100 | fi 101 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/send_message.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sends message to RabbitMQ to notify capture is complete and written to a file 3 | 4 | Created on 27 November 2019 5 | @author: Charlie Lewis 6 | """ 7 | import argparse 8 | import datetime 9 | import json 10 | import os 11 | 12 | import pika 13 | 14 | 15 | def connect_rabbit(host='messenger', port=5672, queue='task_queue'): 16 | params = pika.ConnectionParameters(host=host, port=port) 17 | connection = pika.BlockingConnection(params) 18 | channel = connection.channel() 19 | channel.queue_declare(queue=queue, durable=True) 20 | return channel 21 | 22 | 23 | def send_rabbit_msg(msg, channel, exchange='', routing_key='task_queue'): 24 | channel.basic_publish(exchange=exchange, 25 | routing_key=routing_key, 26 | body=json.dumps(msg), 27 | properties=pika.BasicProperties(delivery_mode=2,)) 28 | print(' [X] %s UTC %r %r' % (str(datetime.datetime.utcnow()), 29 | str(msg['id']), str(msg['file_path']))) 30 | 31 | 32 | def get_version(): 33 | version = '' 34 | with open('VERSION', 'r') as f: 35 | for line in f: 36 | version = line.strip() 37 | return version 38 | 39 | 40 | def get_path(paths): 41 | path = None 42 | try: 43 | path = paths[0] 44 | except Exception as e: 45 | print('No path provided: {0}, quitting'.format(str(e))) 46 | return path 47 | 48 | 49 | def parse_args(parser): 50 | parser.add_argument('paths', nargs='*') 51 | parsed_args = parser.parse_args() 52 | return parsed_args 53 | 54 | 55 | if __name__ == '__main__': # pragma: no cover 56 | parsed_args = parse_args(argparse.ArgumentParser()) 57 | path = get_path(parsed_args.paths) 58 | uid = '' 59 | if 'id' in os.environ: 60 | uid = os.environ['id'] 61 | if 'external_host' in os.environ: 62 | external_host = os.environ['external_host'] 63 | else: 64 | external_host = 'messenger' 65 | if os.environ.get('rabbit', False) == 'true': 66 | try: 67 | channel = connect_rabbit(host=external_host) 68 | body = {'id': uid, 'type': 'metadata', 'file_path': path, 69 | 'data': '', 'file_type': 'pcap_strip', 70 | 'results': {'tool': 'ncapture', 'version': get_version()}} 71 | send_rabbit_msg(body, channel) 72 | except Exception as e: 73 | print(str(e)) 74 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/test_ncapture.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # smoke test for ncapture worker 4 | # requires tcpdump and tshark to be installed. 5 | 6 | URI=lo 7 | IP=127.0.0.1 8 | SIZE=1000 9 | MAXCAPLEN=50 10 | 11 | TMPDIR=$(mktemp -d) 12 | 13 | docker build -f Dockerfile . -t cyberreboot/vent-ncapture 14 | echo starting ncapture 15 | docker run --privileged --net=host --cap-add=NET_ADMIN -v $TMPDIR:/files -t cyberreboot/vent-ncapture /tmp/run.sh $URI 15 test 1 "host $IP and icmp" -d 12 -s 4 -a none -c none -o /files/ || exit 1 & 16 | echo waiting for pcap 17 | while [ "$(find $TMPDIR -prune -empty)" ] ; do 18 | ping -q -n -i 0.1 -s $SIZE -c 10 $IP > /dev/null 19 | echo -n . 20 | done 21 | tcpdump -n -r $TMPDIR/*pcap greater $SIZE > $TMPDIR/greater.txt || exit 1 22 | if [ ! -s $TMPDIR/greater.txt ] ; then 23 | echo "FAIL: no packets with original size $SIZE" 24 | exit 1 25 | fi 26 | CAPLEN=$(capinfos -l $TMPDIR/*cap|grep -E 'Packet size limit:\s+inferred: [0-9]+ bytes'|grep -o -E '[0-9]+') 27 | if [ "$CAPLEN" == "" ] ; then 28 | echo "FAIL: capture length not limited" 29 | exit 1 30 | fi 31 | if [ "$CAPLEN" -gt $MAXCAPLEN ] ; then 32 | echo "FAIL: capture length $CAPLEN over limit (payload not stripped?)" 33 | exit 1 34 | fi 35 | echo ok 36 | 37 | rm -rf $TMPDIR 38 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncapture/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = NCapture 3 | groups = collection,hidden,network,network_tap_child 4 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncontrol/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/extras/network_tap/ncontrol/__init__.py -------------------------------------------------------------------------------- /vent/extras/network_tap/ncontrol/ncontrol.py: -------------------------------------------------------------------------------- 1 | import falcon 2 | from falcon_cors import CORS 3 | 4 | from .routes import routes 5 | 6 | 7 | cors = CORS(allow_all_origins=True) 8 | api = application = falcon.API(middleware=[cors.middleware]) 9 | 10 | r = routes() 11 | for route in r: 12 | api.add_route(route, r[route]) 13 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncontrol/requirements.txt: -------------------------------------------------------------------------------- 1 | docker==4.1.0 2 | falcon==2.0.0 3 | falcon-cors==1.1.7 4 | gevent==1.4.0 5 | gunicorn==20.0.4 6 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncontrol/routes.py: -------------------------------------------------------------------------------- 1 | def routes(): 2 | from .paths import CreateR, DeleteR, InfoR, NICsR, ListR, StartR, StopR 3 | p = paths() 4 | create_r = CreateR() 5 | delete_r = DeleteR() 6 | info_r = InfoR() 7 | nics_r = NICsR() 8 | list_r = ListR() 9 | start_r = StartR() 10 | stop_r = StopR() 11 | funcs = [create_r, 12 | delete_r, 13 | info_r, 14 | list_r, 15 | nics_r, 16 | start_r, 17 | stop_r] 18 | return dict(list(zip(p, funcs))) 19 | 20 | 21 | def paths(): 22 | return ['/create', 23 | '/delete', 24 | '/info', 25 | '/list', 26 | '/nics', 27 | '/start', 28 | '/stop'] 29 | -------------------------------------------------------------------------------- /vent/extras/network_tap/ncontrol/test_ncontrol.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import docker 3 | import falcon 4 | import pytest 5 | from falcon import testing 6 | 7 | from .ncontrol import api 8 | 9 | 10 | @pytest.fixture 11 | def client(): 12 | return testing.TestClient(api) 13 | 14 | 15 | def test_create_r(client): 16 | """ tests the restful endpoint: create """ 17 | # test create 18 | payload = {'id': 'foo', 'interval': '60', 'filter': '', 'nic': 'eth1'} 19 | r = client.simulate_post('/create', json=payload) 20 | assert r.status == '200 OK' 21 | r = client.simulate_post('/create', json={'id': 'foo', 22 | 'interval': '60', 23 | 'iters': '1', 24 | 'filter': '', 25 | 'nic': 'eth1'}) 26 | assert r.status == '200 OK' 27 | r = client.simulate_post('/create', json={}) 28 | assert r.status == '200 OK' 29 | r = client.simulate_post('/create', json={'nic': 'eth1'}) 30 | assert r.status == '200 OK' 31 | r = client.simulate_post('/create', json={'nic': 'eth1', 'id': 'foo'}) 32 | assert r.status == '200 OK' 33 | r = client.simulate_post( 34 | '/create', json={'nic': 'eth1', 'id': 'foo', 'interval': '61'}) 35 | assert r.status == '200 OK' 36 | r = client.simulate_post('/create', json={'id': 'foo', 37 | 'interval': '60', 38 | 'filter': '', 39 | 'metadata': '{"foo": "bar"}', 40 | 'iters': '1', 41 | 'nic': 'eth1'}) 42 | assert r.status == '200 OK' 43 | 44 | 45 | def test_info_r(client): 46 | """ tests the restful endpoint: info """ 47 | # test info 48 | r = client.simulate_get('/info') 49 | assert r.status == '200 OK' 50 | 51 | 52 | def test_list_r(client): 53 | """ tests the restful endpoint: list """ 54 | # test list 55 | r = client.simulate_get('/list') 56 | assert r.status == '200 OK' 57 | 58 | 59 | def test_nics_r(client): 60 | """ tests the restful endpoint: nics """ 61 | # test nics 62 | r = client.simulate_get('/nics') 63 | assert r.status == '200 OK' 64 | 65 | 66 | def test_stop_r(client): 67 | """ tests the restful endpoint: stop """ 68 | # create some container and start it 69 | d = docker.from_env() 70 | d.images.pull('alpine') 71 | test_cont = d.containers.create('alpine') 72 | 73 | # test stop 74 | r = client.simulate_post('/stop', json={}) 75 | assert r.status == '200 OK' 76 | r = client.simulate_post('/stop', json={'id': test_cont.attrs['Id']}) 77 | assert r.status == '200 OK' 78 | r = client.simulate_post('/stop', json={'id': []}) 79 | assert r.status == '200 OK' 80 | 81 | 82 | def test_start_r(client): 83 | """ tests the restful endpoint: start """ 84 | # create some container 85 | d = docker.from_env() 86 | d.images.pull('alpine') 87 | test_cont = d.containers.create('alpine') 88 | 89 | # test start 90 | r = client.simulate_post('/start', json={}) 91 | assert r.status == '200 OK' 92 | r = client.simulate_post('/start', json={'id': test_cont.attrs['Id']}) 93 | assert r.status == '200 OK' 94 | r = client.simulate_post('/start', json={'id': []}) 95 | assert r.status == '200 OK' 96 | 97 | 98 | def test_delete_r(client): 99 | """ tests the restful endpoint: delete """ 100 | # create some container and start it 101 | d = docker.from_env() 102 | d.images.pull('alpine') 103 | test_cont = d.containers.create('alpine') 104 | 105 | # test delete 106 | r = client.simulate_post('/delete', json={}) 107 | assert r.status == '200 OK' 108 | r = client.simulate_post('/delete', json={'id': test_cont.attrs['Id']}) 109 | assert r.status == '200 OK' 110 | r = client.simulate_post('/delete', json={'id': []}) 111 | assert r.status == '200 OK' 112 | -------------------------------------------------------------------------------- /vent/extras/network_tap/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | gunicorn -b :8080 -k gevent -w 4 --reload ncontrol.ncontrol 3 | -------------------------------------------------------------------------------- /vent/extras/network_tap/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = NetworkTap 3 | groups = files,network 4 | 5 | [service] 6 | uri_prefix1 = http:// 7 | 8 | [settings] 9 | priority = 11 10 | 11 | [docker] 12 | environment = ["PYTHONUNBUFFERED=0"] 13 | volumes = {'/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'}} 14 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="rmq_es_connector" \ 5 | vent.groups="connector" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | RUN apk add --update \ 10 | curl \ 11 | python3 \ 12 | py3-pip \ 13 | && rm -rf /var/cache/apk/* 14 | 15 | RUN pip3 install --no-cache-dir --upgrade pip==19.1 16 | 17 | # healthcheck 18 | COPY healthcheck /healthcheck 19 | RUN pip3 install --no-cache-dir -r /healthcheck/requirements.txt 20 | ENV FLASK_APP /healthcheck/hc.py 21 | HEALTHCHECK --interval=15s --timeout=15s \ 22 | CMD curl --silent --fail http://localhost:5000/healthcheck || exit 1 23 | 24 | COPY requirements.txt /vent/requirements.txt 25 | RUN pip3 install --no-cache-dir -r /vent/requirements.txt 26 | COPY rmq_es_connector.py /vent/rmq_es_connector.py 27 | 28 | CMD (flask run > /dev/null 2>&1) & (python3 /vent/rmq_es_connector.py "#") 29 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/extras/rmq_es_connector/__init__.py -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/healthcheck/hc.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from healthcheck import EnvironmentDump 4 | from healthcheck import HealthCheck 5 | 6 | app = Flask(__name__) 7 | 8 | health = HealthCheck(app, '/healthcheck') 9 | envdump = EnvironmentDump(app, '/environment') 10 | 11 | 12 | def application_data(): 13 | return {'maintainer': 'Charlie Lewis', 14 | 'git_repo': 'https://github.com/CyberReboot/vent', 15 | 'app': 'rmq_es_connector'} 16 | 17 | 18 | envdump.add_section('application', application_data) 19 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/healthcheck/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==1.1.1 2 | healthcheck==1.3.3 3 | six==1.13.0 4 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/requirements.txt: -------------------------------------------------------------------------------- 1 | elasticsearch==7.1.0 2 | pika==1.1.0 3 | urllib3==1.25.7 4 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/rmq_es_connector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import os 4 | import sys 5 | import time 6 | import uuid 7 | 8 | import pika 9 | from elasticsearch import Elasticsearch 10 | 11 | 12 | class RmqEs(): 13 | """ 14 | opens a connection to rabbitmq and receives messages based on the provided 15 | binding keys and then takes those messages and sends them to an 16 | elasticsearch index 17 | """ 18 | es_conn = None 19 | es_host = None 20 | # get custom set port or else use default port 21 | es_port = int(os.getenv('ELASTICSEARCH_CUSTOM_PORT', 9200)) 22 | rmq_host = None 23 | # get custom set port or else use default port 24 | rmq_port = int(os.getenv('RABBITMQ_CUSTOM_PORT', 5672)) 25 | channel = None 26 | queue_name = None 27 | 28 | def __init__(self, es_host='elasticsearch', rmq_host='rabbitmq'): 29 | """ initialize host information """ 30 | self.es_host = es_host 31 | self.rmq_host = rmq_host 32 | 33 | def connections(self, wait): 34 | """ 35 | wait for connections to both rabbitmq and elasticsearch to be made 36 | before binding a routing key to a channel and sending messages to 37 | elasticsearch 38 | """ 39 | while wait: 40 | try: 41 | params = pika.ConnectionParameters(host=self.rmq_host, 42 | port=self.rmq_port) 43 | connection = pika.BlockingConnection(params) 44 | self.channel = connection.channel() 45 | self.channel.exchange_declare(exchange='topic_recs', 46 | exchange_type='topic') 47 | 48 | result = self.channel.queue_declare() 49 | self.queue_name = result.method.queue 50 | self.es_conn = Elasticsearch([{'host': self.es_host, 51 | 'port': self.es_port}]) 52 | wait = False 53 | print('connected to rabbitmq and elasticsearch...') 54 | except Exception as e: # pragma: no cover 55 | print(str(e)) 56 | print('waiting for connection to rabbitmq...' + str(e)) 57 | time.sleep(2) 58 | wait = True 59 | 60 | def callback(self, ch, method, properties, body): 61 | """ 62 | callback triggered on rabiitmq message received and sends it to 63 | an elasticsearch index 64 | """ 65 | index = method.routing_key.split('.')[1] 66 | index = index.replace('/', '-') 67 | failed = False 68 | body = str(body) 69 | try: 70 | doc = json.loads(body) 71 | except Exception as e: # pragma: no cover 72 | try: 73 | body = body.strip().replace('"', '\"') 74 | body = '{"log":"' + body + '"}' 75 | doc = json.loads(body) 76 | except Exception as e: # pragma: no cover 77 | failed = True 78 | 79 | if not failed: 80 | try: 81 | self.es_conn.index(index=index, 82 | doc_type=method.routing_key.split('.')[1], 83 | id=method.routing_key + '.' + 84 | str(uuid.uuid4()), 85 | body=doc) 86 | except Exception as e: # pragma: no cover 87 | print('Failed to send document to elasticsearch because: ' + str(e)) 88 | 89 | def start(self): 90 | """ start the channel listener and start consuming messages """ 91 | self.connections(True) 92 | 93 | binding_keys = sys.argv[1:] 94 | if not binding_keys: 95 | print(sys.stderr, 96 | 'Usage: {0!s} [binding_key]...'.format(sys.argv[0])) 97 | sys.exit(0) 98 | 99 | for binding_key in binding_keys: 100 | self.channel.queue_bind(exchange='topic_recs', 101 | queue=self.queue_name, 102 | routing_key=binding_key) 103 | 104 | def consume(self): # pragma: no cover 105 | """ start consuming rabbitmq messages """ 106 | print(' [*] Waiting for logs. To exit press CTRL+C') 107 | self.channel.basic_consume(self.queue_name, self.callback) 108 | self.channel.start_consuming() 109 | 110 | 111 | if __name__ == '__main__': # pragma: no cover 112 | rmq_es = RmqEs() 113 | rmq_es.start() 114 | rmq_es.consume() 115 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/test_rmq_es_connector.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from .rmq_es_connector import RmqEs 6 | 7 | 8 | class Method(): 9 | """ create mock method object """ 10 | routing_key = None 11 | 12 | def __init__(self, routing_key='foo.bar'): 13 | self.routing_key = routing_key 14 | 15 | 16 | def test_rmq_es_connector_connections(): 17 | """ tests the connections function """ 18 | rmq_es = RmqEs() 19 | rmq_es.connections(False) 20 | 21 | 22 | def test_rmq_es_connector_callback(): 23 | """ tests the callback function """ 24 | rmq_es = RmqEs() 25 | method = Method() 26 | rmq_es.callback(None, method, None, '[]') 27 | rmq_es.callback(None, method, None, '[]') 28 | method = Method(routing_key='syslog.foo') 29 | rmq_es.callback(None, method, None, '[]') 30 | method = Method(routing_key='dshell_netflow.foo') 31 | rmq_es.callback(None, method, None, '[]') 32 | method = Method(routing_key='hex_flow.foo') 33 | rmq_es.callback(None, method, None, '[]') 34 | rmq_es.callback(None, method, None, "asdf * '[]'") 35 | -------------------------------------------------------------------------------- /vent/extras/rmq_es_connector/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = RMQ-ES-Connector 3 | groups = connector 4 | 5 | [settings] 6 | instances = 1 7 | priority = 20 8 | 9 | [docker] 10 | environment = ["PYTHONUNBUFFERED=0"] 11 | links = {"RabbitMQ":"rabbitmq", "Elasticsearch":"elasticsearch"} 12 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.11 2 | LABEL maintainer="Charlie Lewis " \ 3 | vent="" \ 4 | vent.name="rq_dashboard" \ 5 | vent.groups="monitoring" \ 6 | vent.repo="https://github.com/cyberreboot/vent" \ 7 | vent.type="repository" 8 | 9 | RUN apk add --update \ 10 | curl \ 11 | git \ 12 | python3 \ 13 | py3-pip \ 14 | && rm -rf /var/cache/apk/* 15 | 16 | RUN pip3 install --no-cache-dir --upgrade pip==19.1 17 | 18 | # healthcheck 19 | COPY healthcheck /healthcheck 20 | RUN pip3 install --no-cache-dir -r /healthcheck/requirements.txt 21 | ENV FLASK_APP /healthcheck/hc.py 22 | HEALTHCHECK --interval=15s --timeout=15s \ 23 | CMD curl --silent --fail http://localhost:5000/healthcheck || exit 1 24 | 25 | # get newer for worker list fix 26 | RUN pip3 install rq-dashboard==0.5.2 27 | 28 | COPY rq_dash_settings.py /rq_dash_settings.py 29 | 30 | COPY run.sh /run.sh 31 | RUN chmod 755 /run.sh 32 | 33 | EXPOSE 9181 34 | 35 | CMD (flask run > /dev/null 2>&1) & (/run.sh) 36 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/extras/rq_dashboard/__init__.py -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/healthcheck/hc.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from healthcheck import EnvironmentDump 4 | from healthcheck import HealthCheck 5 | 6 | app = Flask(__name__) 7 | 8 | health = HealthCheck(app, '/healthcheck') 9 | envdump = EnvironmentDump(app, '/environment') 10 | 11 | 12 | def application_data(): 13 | return {'maintainer': 'Charlie Lewis', 14 | 'git_repo': 'https://github.com/CyberReboot/vent', 15 | 'app': 'rq_dashboard'} 16 | 17 | 18 | envdump.add_section('application', application_data) 19 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/healthcheck/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==1.1.1 2 | healthcheck==1.3.3 3 | six==1.13.0 4 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/rq_dash_settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | URL_PREFIX = os.environ['DASH_PREFIX'] 3 | REDIS_HOST = os.environ['REMOTE_REDIS_HOST'] 4 | REDIS_PORT = os.environ['REMOTE_REDIS_PORT'] 5 | REDIS_PASSWORD = os.environ['REMOTE_REDIS_PSWD'] 6 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | set -x 4 | 5 | if [[ -z "$REMOTE_REDIS_HOST" ]]; then 6 | echo "NEED TO SET REMOTE_REDIS_HOST env var. If redis on host machine, set to public ip for host machine" 7 | exit 11 8 | fi 9 | 10 | if [[ -z "$REMOTE_REDIS_PORT" ]]; then 11 | export REMOTE_REDIS_PORT="6379" 12 | fi 13 | 14 | if [[ -z "$REMOTE_REDIS_PSWD" ]]; then #should i fail if not set? 15 | echo "REMOTE_REDIS_PSWD not set. Please SET" 16 | export REMOTE_REDIS_PSWD="" 17 | fi 18 | 19 | if [[ -z "$DASH_PREFIX" ]]; then #should i fail if not set? 20 | export DASH_PREFIX="/rq" 21 | fi 22 | 23 | export RQ_DASHBOARD_SETTINGS="/rq_dash_settings.py" 24 | 25 | echo "REMOTE_REDIS_HOST=${REMOTE_REDIS_HOST} REMOTE_REDIS_PORT=${REMOTE_REDIS_PORT}" 26 | 27 | rq-dashboard 28 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/test_rq_dashboard.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def test_rq_dash_settings(): 5 | """ Tests the rq dashboard environment variable settings """ 6 | os.environ['DASH_PREFIX'] = 'test' 7 | os.environ['REMOTE_REDIS_HOST'] = 'test' 8 | os.environ['REMOTE_REDIS_PORT'] = 'test' 9 | os.environ['REMOTE_REDIS_PSWD'] = 'test' 10 | from . import rq_dash_settings 11 | -------------------------------------------------------------------------------- /vent/extras/rq_dashboard/vent.template: -------------------------------------------------------------------------------- 1 | [info] 2 | name = RQ-Dashboard 3 | groups = monitoring 4 | 5 | [service] 6 | uri_prefix1 = http:// 7 | uri_postfix1 = /rq 8 | 9 | [settings] 10 | instances = 1 11 | priority = 5 12 | 13 | [docker] 14 | environment = ["REMOTE_REDIS_HOST=redis"] 15 | links = {"Redis":"redis"} 16 | -------------------------------------------------------------------------------- /vent/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/helpers/__init__.py -------------------------------------------------------------------------------- /vent/helpers/errors.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger(__name__) 4 | 5 | 6 | def ErrorHandler(function): 7 | def wrapper(*args, **kwargs): 8 | try: 9 | return function(*args, **kwargs) 10 | except Exception as e: # pragma: no cover 11 | # log the exception 12 | logger.exception('Exception in {0}: {1}'.format( 13 | function.__name__, str(e))) 14 | return wrapper 15 | -------------------------------------------------------------------------------- /vent/helpers/logs.py: -------------------------------------------------------------------------------- 1 | import logging.handlers 2 | import os 3 | 4 | from vent.helpers.paths import PathDirs 5 | 6 | 7 | def Logger(name, **kargs): 8 | """ Create and return logger """ 9 | path_dirs = PathDirs(**kargs) 10 | logging.captureWarnings(True) 11 | logger = logging.getLogger(name) 12 | logger.setLevel(logging.INFO) 13 | handler = logging.handlers.WatchedFileHandler(os.path.join( 14 | path_dirs.meta_dir, 'vent.log')) 15 | handler.setLevel(logging.INFO) 16 | formatter = logging.Formatter('%(asctime)s - %(name)s:%(lineno)-4d - ' 17 | '%(levelname)s - %(message)s') 18 | handler.setFormatter(formatter) 19 | if not len(logger.handlers): 20 | logger.addHandler(handler) 21 | 22 | return logger 23 | -------------------------------------------------------------------------------- /vent/helpers/paths.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import platform 3 | from os import chdir 4 | from os import environ 5 | from os import getcwd 6 | from os import getenv 7 | from os import makedirs 8 | from os.path import exists 9 | from os.path import expanduser 10 | from os.path import isdir 11 | from os.path import isfile 12 | from os.path import join 13 | 14 | import yaml 15 | 16 | from vent.helpers.templates import Template 17 | 18 | 19 | class PathDirs: 20 | """ Global path directories for vent """ 21 | 22 | def __init__(self, 23 | base_dir=join(expanduser('~'), '.vent/'), 24 | plugins_dir='plugins/', 25 | meta_dir=join(expanduser('~'), '.vent')): 26 | self.base_dir = base_dir 27 | self.plugins_dir = base_dir + plugins_dir 28 | self.meta_dir = meta_dir 29 | self.init_file = base_dir + 'vent.init' 30 | self.cfg_file = base_dir + 'vent.cfg' 31 | self.startup_file = join(expanduser('~'), 32 | '.vent_startup.yml') 33 | self.plugin_config_file = join(expanduser('~'), 34 | '.plugin_config.yml') 35 | 36 | # make sure the paths exists, if not create them 37 | self.ensure_dir(self.base_dir) 38 | self.ensure_dir(self.plugins_dir) 39 | self.ensure_dir(self.meta_dir) 40 | 41 | @staticmethod 42 | def ensure_dir(path): 43 | """ 44 | Tries to create directory, if fails, checks if path already exists 45 | """ 46 | try: 47 | makedirs(path) 48 | except OSError as e: # pragma: no cover 49 | if e.errno == errno.EEXIST and isdir(path): 50 | return (True, 'exists') 51 | else: 52 | return (False, e) 53 | return (True, path) 54 | 55 | @staticmethod 56 | def ensure_file(path): 57 | """ Checks if file exists, if fails, tries to create file """ 58 | try: 59 | exists = isfile(path) 60 | if not exists: 61 | with open(path, 'w+') as fname: 62 | fname.write('initialized') 63 | return (True, path) 64 | return (True, 'exists') 65 | except OSError as e: # pragma: no cover 66 | return (False, e) 67 | 68 | @staticmethod 69 | def rel_path(name, available_tools): 70 | """ 71 | Extracts relative path to a tool (from the main cloned directory) out 72 | of available_tools based on the name it is given 73 | """ 74 | if name == '@' or name == '.' or name == '/': 75 | name = '' 76 | multi_tool = '@' in name 77 | for tool in available_tools: 78 | t_name = tool[0].lower() 79 | if multi_tool: 80 | if name.split('@')[-1] == t_name.split('@')[-1]: 81 | return t_name, t_name 82 | else: 83 | if name == t_name.split('/')[-1]: 84 | return t_name, tool[0] 85 | elif name == '' and t_name.split('@')[-1] == 'unspecified': 86 | return '', '' 87 | return None, None 88 | 89 | def host_config(self): 90 | """ Ensure the host configuration file exists """ 91 | if platform.system() == 'Darwin': 92 | default_file_dir = join(expanduser('~'), 93 | 'vent_files') 94 | else: 95 | default_file_dir = '/opt/vent_files' 96 | status = self.ensure_dir(default_file_dir) 97 | if not isfile(self.cfg_file): 98 | config = Template(template=self.cfg_file) 99 | sections = {'main': {'files': default_file_dir}, 100 | 'network-mapping': {}, 101 | 'nvidia-docker-plugin': {'port': '3476'}} 102 | for s in sections: 103 | if sections[s]: 104 | for option in sections[s]: 105 | config.add_option(s, option, sections[s][option]) 106 | else: 107 | config.add_section(s) 108 | config.write_config() 109 | return status 110 | 111 | def apply_path(self, repo): 112 | """ Set path to where the repo is and return original path """ 113 | try: 114 | # rewrite repo for consistency 115 | if repo.endswith('.git'): 116 | repo = repo.split('.git')[0] 117 | 118 | # get org and repo name and path repo will be cloned to 119 | org, name = repo.split('/')[-2:] 120 | path = join(self.plugins_dir, org, name) 121 | 122 | # save current path 123 | cwd = getcwd() 124 | 125 | # set to new repo path 126 | self.ensure_dir(path) 127 | chdir(path) 128 | status = (True, cwd, path) 129 | except Exception as e: # pragma: no cover 130 | status = (False, str(e)) 131 | return status 132 | 133 | def get_path(self, repo): 134 | """ Return the path for the repo """ 135 | if repo.endswith('.git'): 136 | repo = repo.split('.git')[0] 137 | org, name = repo.split('/')[-2:] 138 | path = self.plugins_dir 139 | path = join(path, org, name) 140 | return path, org, name 141 | 142 | def override_config(self, path): 143 | """ 144 | Will take a yml located in home directory titled '.plugin_config.yml'. 145 | It'll then override, using the yml, the plugin's config file 146 | """ 147 | status = (True, None) 148 | config_override = False 149 | 150 | try: 151 | # parse the yml file 152 | c_dict = {} 153 | if exists(self.plugin_config_file): 154 | with open(self.plugin_config_file, 'r') as config_file: 155 | c_dict = yaml.safe_load(config_file.read()) 156 | 157 | # check for environment variable overrides 158 | check_c_dict = c_dict.copy() 159 | for tool in check_c_dict: 160 | for section in check_c_dict[tool]: 161 | for key in check_c_dict[tool][section]: 162 | if key in environ: 163 | c_dict[tool][section][key] = getenv(key) 164 | 165 | # assume the name of the plugin is its directory 166 | plugin_name = path.split('/')[-1] 167 | if plugin_name == '': 168 | plugin_name = path.split('/')[-2] 169 | plugin_config_path = path + '/config/' + plugin_name + '.config' 170 | 171 | if exists(plugin_config_path): 172 | plugin_template = Template(plugin_config_path) 173 | plugin_options = c_dict[plugin_name] 174 | for section in plugin_options: 175 | for option in plugin_options[section]: 176 | plugin_template.set_option(section, option, 177 | str(plugin_options[section][option])) 178 | plugin_template.write_config() 179 | config_override = True 180 | except Exception as e: # pragma: no cover 181 | status = (False, str(e)) 182 | 183 | return status, config_override 184 | -------------------------------------------------------------------------------- /vent/helpers/templates.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | 3 | from vent.helpers.errors import ErrorHandler 4 | 5 | 6 | class Template: 7 | """ Handle parsing templates """ 8 | 9 | def __init__(self, template=None): 10 | self.config = configparser.ConfigParser(interpolation=None) 11 | self.config.optionxform = str 12 | if template: 13 | self.config.read(template) 14 | self.template = template 15 | 16 | @ErrorHandler 17 | def sections(self): 18 | """ Returns a list of sections """ 19 | return (True, self.config.sections()) 20 | 21 | @ErrorHandler 22 | def section(self, section): 23 | """ Returns a list of tuples of (option, value) for the section """ 24 | # check if the named section exists 25 | if self.config.has_section(section): 26 | return (True, self.config.items(section)) 27 | return (False, 'Section: ' + section + ' does not exist') 28 | 29 | @ErrorHandler 30 | def options(self, section): 31 | """ Returns a list of options for a section """ 32 | if self.config.has_section(section): 33 | return (True, self.config.options(section)) 34 | return (False, 'Section: ' + section + ' does not exist') 35 | 36 | @ErrorHandler 37 | def option(self, section, option): 38 | """ Returns the value of the option """ 39 | if self.config.has_section(section): 40 | if self.config.has_option(section, option): 41 | return (True, self.config.get(section, option)) 42 | return (False, 'Option: ' + option + ' does not exist') 43 | return (False, 'Section: ' + section + ' does not exist') 44 | 45 | @ErrorHandler 46 | def add_section(self, section): 47 | """ 48 | If section exists, returns log, 49 | otherwise adds section and returns list of sections. 50 | """ 51 | # check if section already exists 52 | if not self.config.has_section(section): 53 | self.config.add_section(section) 54 | # return updated sections 55 | return (True, self.config.sections()) 56 | return (False, 'Section: ' + section + ' already exists') 57 | 58 | @ErrorHandler 59 | def add_option(self, section, option, value=None): 60 | """ 61 | Creates an option for a section. If the section does 62 | not exist, it will create the section. 63 | """ 64 | # check if section exists; create if not 65 | if not self.config.has_section(section): 66 | message = self.add_section(section) 67 | if not message[0]: 68 | return message 69 | 70 | if not self.config.has_option(section, option): 71 | if value: 72 | self.config.set(section, option, value) 73 | else: 74 | self.config.set(section, option) 75 | return(True, self.config.options(section)) 76 | return(False, 'Option: {} already exists @ {}'.format(option, section)) 77 | 78 | @ErrorHandler 79 | def del_section(self, section): 80 | """ Deletes a section if it exists """ 81 | if self.config.has_section(section): 82 | self.config.remove_section(section) 83 | return (True, self.config.sections()) 84 | return (False, 'Section: ' + section + ' does not exist') 85 | 86 | @ErrorHandler 87 | def del_option(self, section, option): 88 | """ Deletes an option if the section and option exist """ 89 | if self.config.has_section(section): 90 | if self.config.has_option(section, option): 91 | self.config.remove_option(section, option) 92 | return (True, self.config.options(section)) 93 | return (False, 'Option: ' + option + ' does not exist') 94 | return (False, 'Section: ' + section + ' does not exist') 95 | 96 | @ErrorHandler 97 | def set_option(self, section, option, value): 98 | """ 99 | Sets an option to a value in the given section. Option is created if it 100 | does not already exist 101 | """ 102 | if self.config.has_section(section): 103 | self.config.set(section, option, value) 104 | return (True, self.config.options(section)) 105 | return (False, 'Section: ' + section + ' does not exist') 106 | 107 | @ErrorHandler 108 | def write_config(self): 109 | with open(self.template, 'w') as configfile: 110 | self.config.write(configfile) 111 | return 112 | 113 | @ErrorHandler 114 | def constrained_sections(self, constraints=None, options=None): 115 | """ 116 | Takes a dictionary of option/values (constraints) that must be present 117 | in a section, and returns a dictionary of sections and optionally a 118 | dictionary of option/values defined by a list of options called options 119 | that match the constraints 120 | """ 121 | sections = {} 122 | if not constraints: 123 | constraints = {} 124 | if not options: 125 | options = [] 126 | all_sections = self.sections() 127 | for a_section in all_sections[1]: 128 | include = True 129 | for constraint in constraints: 130 | result = self.option(a_section, constraint) 131 | if not result[0] or result[1] != constraints[constraint]: 132 | include = False 133 | # handle group membership 134 | if (result[0] and 135 | constraint == 'groups' and 136 | constraints[constraint] in result[1]): 137 | include = True 138 | if include: 139 | sections[a_section] = {} 140 | for option in options: 141 | result = self.option(a_section, option) 142 | if result[0]: 143 | sections[a_section][option] = result[1] 144 | return sections 145 | 146 | @ErrorHandler 147 | def constrain_opts(self, constraint_dict, options): 148 | """ Return result of constraints and options against a template """ 149 | constraints = {} 150 | for constraint in constraint_dict: 151 | if constraint != 'self': 152 | if (constraint_dict[constraint] or 153 | constraint_dict[constraint] == ''): 154 | constraints[constraint] = constraint_dict[constraint] 155 | results = self.constrained_sections(constraints=constraints, 156 | options=options) 157 | return results, self.template 158 | 159 | @ErrorHandler 160 | def list_tools(self): 161 | """ 162 | Return list of tuples of all tools 163 | """ 164 | tools = [] 165 | exists, sections = self.sections() 166 | if exists: 167 | for section in sections: 168 | options = {'section': section, 169 | 'built': None, 170 | 'version': None, 171 | 'repo': None, 172 | 'branch': None, 173 | 'name': None, 174 | 'groups': None, 175 | 'image_name': None} 176 | for option in list(options.keys()): 177 | exists, value = self.option(section, option) 178 | if exists: 179 | options[option] = value 180 | if 'core' not in options['groups'] and 'hidden' not in options['groups']: 181 | tools.append(options) 182 | return tools 183 | -------------------------------------------------------------------------------- /vent/menu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import curses 4 | import time 5 | from threading import Thread 6 | 7 | import npyscreen 8 | 9 | from vent.api.system import System 10 | from vent.helpers.meta import Version 11 | from vent.helpers.paths import PathDirs 12 | from vent.menus.help import HelpForm 13 | from vent.menus.main import MainForm 14 | from vent.menus.tutorial_forms import TutorialAddingFilesForm 15 | from vent.menus.tutorial_forms import TutorialAddingPluginsForm 16 | from vent.menus.tutorial_forms import TutorialBackgroundForm 17 | from vent.menus.tutorial_forms import TutorialGettingSetupForm 18 | from vent.menus.tutorial_forms import TutorialIntroForm 19 | from vent.menus.tutorial_forms import TutorialStartingCoresForm 20 | from vent.menus.tutorial_forms import TutorialTerminologyForm 21 | from vent.menus.tutorial_forms import TutorialTroubleshootingForm 22 | 23 | 24 | class VentApp(npyscreen.NPSAppManaged): 25 | """ Main menu app for vent CLI """ 26 | keypress_timeout_default = 10 27 | repo_value = {} 28 | paths = PathDirs() 29 | first_time = paths.ensure_file(paths.init_file) 30 | if first_time[0] and first_time[1] != 'exists': 31 | npyscreen.NPSAppManaged.STARTING_FORM = 'TUTORIALINTRO' 32 | else: 33 | npyscreen.NPSAppManaged.STARTING_FORM = 'MAIN' 34 | 35 | def onStart(self): 36 | """ Override onStart method for npyscreen """ 37 | curses.mousemask(0) 38 | self.paths.host_config() 39 | version = Version() 40 | 41 | # setup initial runtime stuff 42 | if self.first_time[0] and self.first_time[1] != 'exists': 43 | system = System() 44 | thr = Thread(target=system.start, args=(), kwargs={}) 45 | thr.start() 46 | countdown = 60 47 | while thr.is_alive(): 48 | npyscreen.notify_wait('Completing initialization:...' + str(countdown), 49 | title='Setting up things...') 50 | time.sleep(1) 51 | countdown -= 1 52 | thr.join() 53 | 54 | quit_s = '\t'*4 + '^Q to quit' 55 | tab_esc = '\t'*4 + 'ESC to close menu popup' 56 | self.addForm('MAIN', 57 | MainForm, 58 | name='Vent ' + version + 59 | '\t\t\t\t\t^T for help' + quit_s + tab_esc, 60 | color='IMPORTANT') 61 | self.addForm('HELP', 62 | HelpForm, 63 | name='Help\t\t\t\t\t\t\t\t^T to toggle previous' + 64 | quit_s, 65 | color='DANGER') 66 | self.addForm('TUTORIALINTRO', 67 | TutorialIntroForm, 68 | name='Vent Tutorial' + quit_s, 69 | color='DANGER') 70 | self.addForm('TUTORIALBACKGROUND', 71 | TutorialBackgroundForm, 72 | name='About Vent' + quit_s, 73 | color='DANGER') 74 | self.addForm('TUTORIALTERMINOLOGY', 75 | TutorialTerminologyForm, 76 | name='About Vent' + quit_s, 77 | color='DANGER') 78 | self.addForm('TUTORIALGETTINGSETUP', 79 | TutorialGettingSetupForm, 80 | name='About Vent' + quit_s, 81 | color='DANGER') 82 | self.addForm('TUTORIALSTARTINGCORES', 83 | TutorialStartingCoresForm, 84 | name='Working with Cores' + quit_s, 85 | color='DANGER') 86 | self.addForm('TUTORIALADDINGPLUGINS', 87 | TutorialAddingPluginsForm, 88 | name='Working with Plugins' + quit_s, 89 | color='DANGER') 90 | self.addForm('TUTORIALADDINGFILES', 91 | TutorialAddingFilesForm, 92 | name='Files' + quit_s, 93 | color='DANGER') 94 | self.addForm('TUTORIALTROUBLESHOOTING', 95 | TutorialTroubleshootingForm, 96 | name='Troubleshooting' + quit_s, 97 | color='DANGER') 98 | 99 | def change_form(self, name): 100 | """ Changes the form (window) that is displayed """ 101 | self.switchForm(name) 102 | -------------------------------------------------------------------------------- /vent/menus/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberReboot/vent/5ca8baf82b4198f38609e1fc471ebd278335536f/vent/menus/__init__.py -------------------------------------------------------------------------------- /vent/menus/add.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | import npyscreen 5 | 6 | from vent.api.image import Image 7 | from vent.api.repository import Repository 8 | from vent.api.system import System 9 | from vent.api.tools import Tools 10 | from vent.menus.add_options import AddOptionsForm 11 | from vent.menus.editor import EditorForm 12 | 13 | 14 | class AddForm(npyscreen.ActionForm): 15 | """ For for adding a new repo """ 16 | default_repo = 'https://github.com/cyberreboot/vent-plugins' 17 | 18 | def create(self): 19 | """ Create widgets for AddForm """ 20 | self.add_handlers({'^T': self.quit, '^Q': self.quit}) 21 | self.add(npyscreen.Textfield, 22 | value='Add a plugin from a Git repository or an image from a ' 23 | 'Docker registry.', 24 | editable=False, 25 | color='STANDOUT') 26 | self.add(npyscreen.Textfield, 27 | value='For Git repositories, you can optionally specify a ' 28 | 'username and password', 29 | editable=False, 30 | color='STANDOUT') 31 | self.add(npyscreen.Textfield, 32 | value='for private repositories.', 33 | editable=False, 34 | color='STANDOUT') 35 | self.add(npyscreen.Textfield, 36 | value='For Docker images, specify a name for referencing the ' 37 | 'image that is being', 38 | editable=False, 39 | color='STANDOUT') 40 | self.add(npyscreen.Textfield, 41 | value='added and optionally override the tag and/or the ' 42 | 'registry and specify', 43 | editable=False, 44 | color='STANDOUT') 45 | self.add(npyscreen.Textfield, 46 | value='comma-separated groups this image should belong to.', 47 | editable=False, 48 | color='STANDOUT') 49 | self.nextrely += 1 50 | self.repo = self.add(npyscreen.TitleText, 51 | name='Repository', 52 | value=self.default_repo) 53 | self.user = self.add(npyscreen.TitleText, name='Username') 54 | self.pw = self.add(npyscreen.TitlePassword, name='Password') 55 | self.nextrely += 1 56 | self.add(npyscreen.TitleText, 57 | name='OR', 58 | editable=False, 59 | labelColor='STANDOUT') 60 | self.nextrely += 1 61 | self.image = self.add(npyscreen.TitleText, name='Image') 62 | self.link_name = self.add(npyscreen.TitleText, 63 | name='Name') 64 | self.tag = self.add(npyscreen.TitleText, name='Tag', value='latest') 65 | self.registry = self.add(npyscreen.TitleText, 66 | name='Registry', 67 | value='docker.io') 68 | self.groups = self.add(npyscreen.TitleText, name='Groups') 69 | self.repo.when_value_edited() 70 | 71 | def quit(self, *args, **kwargs): 72 | """ Overridden to switch back to MAIN form """ 73 | self.parentApp.switchForm('MAIN') 74 | 75 | def on_ok(self): 76 | """ Add the repository """ 77 | def popup(thr, add_type, title): 78 | """ 79 | Start the thread and display a popup of the plugin being cloned 80 | until the thread is finished 81 | """ 82 | thr.start() 83 | tool_str = 'Cloning repository...' 84 | if add_type == 'image': 85 | tool_str = 'Pulling image...' 86 | npyscreen.notify_wait(tool_str, title=title) 87 | while thr.is_alive(): 88 | time.sleep(1) 89 | return 90 | 91 | if self.image.value and self.link_name.value: 92 | api_action = Tools() 93 | api_image = Image(System().manifest) 94 | api_system = System() 95 | thr = threading.Thread(target=api_image.add, args=(), 96 | kwargs={'image': self.image.value, 97 | 'link_name': self.link_name.value, 98 | 'tag': self.tag.value, 99 | 'registry': self.registry.value, 100 | 'groups': self.groups.value}) 101 | popup(thr, 'image', 'Please wait, adding image...') 102 | npyscreen.notify_confirm('Done adding image.', title='Added image') 103 | editor_args = {'tool_name': self.image.value, 104 | 'version': self.tag.value, 105 | 'get_configure': api_system.get_configure, 106 | 'save_configure': api_system.save_configure, 107 | 'restart_tools': api_system.restart_tools, 108 | 'start_tools': api_action.start, 109 | 'from_registry': True, 110 | 'just_downloaded': True, 111 | 'link_name': self.link_name.value, 112 | 'groups': self.groups.value} 113 | self.parentApp.addForm('CONFIGUREIMAGE', EditorForm, 114 | name='Specify vent.template settings for ' 115 | 'image pulled (optional)', **editor_args) 116 | self.parentApp.change_form('CONFIGUREIMAGE') 117 | elif self.image.value: 118 | npyscreen.notify_confirm('A name needs to be supplied for ' 119 | 'the image being added!', 120 | title='Specify a name for the image', 121 | form_color='CAUTION') 122 | elif self.repo.value: 123 | self.parentApp.repo_value['repo'] = self.repo.value.lower() 124 | api_repo = Repository(System().manifest) 125 | api_repo.repo = self.repo.value.lower() 126 | thr = threading.Thread(target=api_repo._clone, args=(), 127 | kwargs={'user': self.user.value, 128 | 'pw': self.pw.value}) 129 | popup(thr, 'repository', 'Please wait, adding repository...') 130 | self.parentApp.addForm('ADDOPTIONS', 131 | AddOptionsForm, 132 | name='Set options for new plugin' 133 | '\t\t\t\t\t\t^Q to quit', 134 | color='CONTROL') 135 | self.parentApp.change_form('ADDOPTIONS') 136 | else: 137 | npyscreen.notify_confirm('Either a repository or an image ' 138 | 'name must be specified!', 139 | title='Specify plugin to add', 140 | form_color='CAUTION') 141 | return 142 | 143 | def on_cancel(self): 144 | """ When user clicks cancel, will return to MAIN """ 145 | self.quit() 146 | -------------------------------------------------------------------------------- /vent/menus/add_options.py: -------------------------------------------------------------------------------- 1 | import npyscreen 2 | 3 | from vent.api.tools import Tools 4 | from vent.menus.choose_tools import ChooseToolsForm 5 | 6 | 7 | class AddOptionsForm(npyscreen.ActionForm): 8 | """ For specifying options when adding a repo """ 9 | branch_cb = {} 10 | commit_tc = {} 11 | build_tc = {} 12 | branches = [] 13 | commits = {} 14 | error = None 15 | 16 | def repo_values(self): 17 | """ 18 | Set the appropriate repo dir and get the branches and commits of it 19 | """ 20 | branches = [] 21 | commits = {} 22 | m_helper = Tools() 23 | status = m_helper.repo_branches(self.parentApp.repo_value['repo']) 24 | # branches and commits must both be retrieved successfully 25 | if status[0]: 26 | branches = status[1] 27 | status = m_helper.repo_commits(self.parentApp.repo_value['repo']) 28 | if status[0]: 29 | r_commits = status[1] 30 | for commit in r_commits: 31 | commits[commit[0]] = commit[1] 32 | else: 33 | # if commits failed, return commit errors 34 | return status 35 | else: 36 | # if branch failed, return branch errors 37 | return status 38 | # if everything is good, return branches with commits 39 | return branches, commits 40 | 41 | def create(self): 42 | """ Update with current branches and commits """ 43 | self.add_handlers({'^Q': self.quit}) 44 | self.add(npyscreen.TitleText, name='Branches:', editable=False) 45 | 46 | if not self.branches or not self.commits: 47 | repo_vals = self.repo_values() 48 | i = 3 49 | # check if repo_values returned successfully 50 | if (isinstance(repo_vals[0], list) and 51 | isinstance(repo_vals[1], dict)): 52 | self.branches, self.commits = repo_vals 53 | for branch in self.branches: 54 | self.branch_cb[branch] = self.add(npyscreen.CheckBox, 55 | name=branch, rely=i, 56 | relx=5, max_width=25) 57 | self.commit_tc[branch] = self.add(npyscreen.TitleCombo, 58 | value=0, rely=i+1, 59 | relx=10, max_width=30, 60 | name='Commit:', 61 | values=self.commits[branch]) 62 | i += 3 63 | else: 64 | self.error = self.add(npyscreen.MultiLineEdit, 65 | name='Errors', 66 | editable=False, labelColor='DANGER', 67 | rely=i, relx=5, color='DANGER', value=""" 68 | Errors were found... 69 | """ + str(repo_vals[1]) + """ 70 | 71 | Please confirm the repo url and credentials are valid! 72 | Vent will return to the main screen. 73 | """) 74 | self.error.display() 75 | 76 | def quit(self, *args, **kwargs): 77 | self.parentApp.switchForm('MAIN') 78 | 79 | def on_ok(self): 80 | """ 81 | Take the branch, commit, and build selection and add them as plugins 82 | """ 83 | self.parentApp.repo_value['versions'] = {} 84 | self.parentApp.repo_value['build'] = {} 85 | for branch in self.branch_cb: 86 | if self.branch_cb[branch].value: 87 | # process checkboxes 88 | self.parentApp.repo_value['versions'][branch] = self.commit_tc[branch].values[self.commit_tc[branch].value] 89 | self.parentApp.repo_value['build'][branch] = True 90 | if self.error: 91 | self.quit() 92 | self.parentApp.addForm('CHOOSETOOLS', 93 | ChooseToolsForm, 94 | name='Choose tools to add for new plugin' 95 | '\t\t\t\t\t\t^Q to quit', 96 | color='CONTROL') 97 | self.parentApp.change_form('CHOOSETOOLS') 98 | 99 | def on_cancel(self): 100 | self.quit() 101 | -------------------------------------------------------------------------------- /vent/menus/backup.py: -------------------------------------------------------------------------------- 1 | import npyscreen 2 | 3 | 4 | class BackupForm(npyscreen.ActionForm): 5 | """ Form that can be used to select a backup file to restore """ 6 | 7 | def __init__(self, *args, **keywords): 8 | """ Initialize backup form objects """ 9 | self.restore = keywords['restore'] 10 | self.dirs = keywords['dirs'] 11 | self.display_vals = [] 12 | for bdir in self.dirs: 13 | date_arr = bdir.split('-')[2:] 14 | self.display_vals.append('-'.join(date_arr[:3]) + ' at ' + 15 | ' '.join(date_arr[3:])) 16 | super(BackupForm, self).__init__(*args, **keywords) 17 | 18 | def create(self): 19 | """ Add backup files to select from """ 20 | self.add_handlers({'^T': self.quit}) 21 | self.add(npyscreen.Textfield, value='Pick a version to restore from: ', 22 | editable=False, color='STANDOUT') 23 | self.dir_select = self.add(npyscreen.SelectOne, 24 | values=self.display_vals, 25 | scroll_exit=True, rely=4) 26 | 27 | def quit(self, *args, **kwargs): 28 | self.parentApp.change_form('MAIN') 29 | 30 | def on_ok(self): 31 | """ Perform restoration on the backup file selected """ 32 | if self.dir_select.value: 33 | npyscreen.notify_wait('In the process of restoring', 34 | title='Restoring...') 35 | status = self.restore(self.dirs[self.dir_select.value[0]]) 36 | if status[0]: 37 | npyscreen.notify_confirm('Status of restore:\n' + 38 | status[1]) 39 | else: 40 | npyscreen.notify_confirm(status[1]) 41 | self.quit() 42 | else: 43 | npyscreen.notify_confirm('Choose a version to restore from') 44 | 45 | def on_cancel(self): 46 | """ When user clicks cancel, will return to MAIN """ 47 | self.quit() 48 | -------------------------------------------------------------------------------- /vent/menus/choose_tools.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | import npyscreen 5 | 6 | from vent.api.tools import Tools 7 | from vent.helpers.meta import ManifestTools 8 | 9 | 10 | class ChooseToolsForm(npyscreen.ActionForm): 11 | """ For picking which tools to add """ 12 | tools_tc = {} 13 | 14 | def repo_tools(self, branch): 15 | """ Set the appropriate repo dir and get the tools available of it """ 16 | tools = [] 17 | m_helper = Tools() 18 | repo = self.parentApp.repo_value['repo'] 19 | version = self.parentApp.repo_value['versions'][branch] 20 | status = m_helper.repo_tools(repo, branch, version) 21 | if status[0]: 22 | r_tools = status[1] 23 | for tool in r_tools: 24 | tools.append(tool[0]) 25 | return tools 26 | 27 | def create(self): 28 | """ Update with current tools for each branch at the version chosen """ 29 | self.add_handlers({'^Q': self.quit}) 30 | self.add(npyscreen.TitleText, 31 | name='Select which tools to add from each branch selected:', 32 | editable=False) 33 | self.add(npyscreen.Textfield, 34 | value='NOTE tools you have already installed will be ignored', 35 | color='STANDOUT', 36 | editable=False) 37 | 38 | i = 6 39 | for branch in self.parentApp.repo_value['versions']: 40 | self.tools_tc[branch] = {} 41 | self.add(npyscreen.TitleText, 42 | name='Branch: ' + branch, 43 | editable=False, 44 | rely=i, 45 | relx=5, 46 | max_width=25) 47 | tools = self.repo_tools(branch) 48 | i += 1 49 | for tool in tools: 50 | value = True 51 | if tool.startswith('/dev'): 52 | value = False 53 | # tool in base directory 54 | if tool == '' or tool.startswith(':'): 55 | tool = '/' + tool 56 | self.tools_tc[branch][tool] = self.add(npyscreen.CheckBox, 57 | name=tool, 58 | value=value, 59 | relx=10) 60 | i += 1 61 | i += 2 62 | 63 | def quit(self, *args, **kwargs): 64 | self.parentApp.switchForm('MAIN') 65 | 66 | def on_ok(self): 67 | """ 68 | Take the tool selections and add them as plugins 69 | """ 70 | def diff(first, second): 71 | """ 72 | Get the elements that exist in the first list and not in the second 73 | """ 74 | second = set(second) 75 | return [item for item in first if item not in second] 76 | 77 | def popup(original_tools, branch, thr, title): 78 | """ 79 | Start the thread and display a popup of the tools being added until 80 | the thread is finished 81 | """ 82 | thr.start() 83 | tool_str = 'Adding tools...' 84 | npyscreen.notify_wait(tool_str, title=title) 85 | while thr.is_alive(): 86 | tools = diff(ManifestTools(), original_tools) 87 | if tools: 88 | tool_str = '' 89 | for tool in tools: 90 | pre_tool = 'Added: ' + branch + '/' + tool + '\n' 91 | tool_str = pre_tool + tool_str 92 | npyscreen.notify_wait(tool_str, title=title) 93 | time.sleep(1) 94 | return 95 | 96 | original_tools = ManifestTools() 97 | for branch in self.tools_tc: 98 | tools = [] 99 | for tool in self.tools_tc[branch]: 100 | if self.tools_tc[branch][tool].value: 101 | # get rid of temporary show for multiple tools in same 102 | # directory 103 | if tool == '/': 104 | tools.append(('.', '')) 105 | else: 106 | tools.append((tool, '')) 107 | repo = self.parentApp.repo_value['repo'] 108 | version = self.parentApp.repo_value['versions'][branch] 109 | api_action = Tools(version=version, branch=branch) 110 | thr = threading.Thread(target=api_action.new, args=(), 111 | kwargs={'tool_type': 'repo', 112 | 'uri': repo, 113 | 'tools': tools}) 114 | popup(original_tools, branch, thr, 115 | 'Please wait, adding tools for the ' + branch + ' branch...') 116 | npyscreen.notify_confirm('Done adding repository: ' + 117 | self.parentApp.repo_value['repo'], 118 | title='Added Repository') 119 | self.quit() 120 | 121 | def on_cancel(self): 122 | self.quit() 123 | -------------------------------------------------------------------------------- /vent/menus/del_instances.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | import npyscreen 5 | 6 | from vent.helpers.meta import Dependencies 7 | 8 | 9 | class InstanceSelect(npyscreen.MultiSelect): 10 | """ 11 | A widget class for selecting an exact amount of instances to perform 12 | actions on 13 | """ 14 | 15 | def __init__(self, *args, **kargs): 16 | """ Initialize an instance select object """ 17 | self.instance_num = kargs['instance_num'] 18 | super(InstanceSelect, self).__init__(*args, **kargs) 19 | 20 | def when_value_edited(self, *args, **kargs): 21 | """ Overrided to prevent user from selecting too many instances """ 22 | if len(self.value) > self.instance_num: 23 | self.value.pop(-2) 24 | self.display() 25 | 26 | def safe_to_exit(self, *args, **kargs): 27 | """ 28 | Overrided to prevent user from exiting selection until 29 | they have selected the right amount of instances 30 | """ 31 | if len(self.value) == self.instance_num: 32 | return True 33 | return False 34 | 35 | 36 | class DeleteForm(npyscreen.ActionForm): 37 | """ A form for selecting instances to delete and deleting them """ 38 | 39 | def __init__(self, *args, **keywords): 40 | """ Initialize a delete form object """ 41 | self.new_instances = int(keywords['new_instances']) 42 | self.next_tool = keywords['next_tool'] 43 | self.manifest = keywords['manifest'] 44 | self.clean = keywords['clean'] 45 | self.prep_start = keywords['prep_start'] 46 | self.start_tools = keywords['start_tools'] 47 | self.cur_instances = [] 48 | self.display_to_section = {} 49 | self.old_instances = int(keywords['old_instances']) 50 | section = keywords['section'] 51 | for i in range(1, self.old_instances + 1): 52 | i_section = section.rsplit(':', 2) 53 | i_section[0] += str(i) if i != 1 else '' 54 | i_section = ':'.join(i_section) 55 | display = i_section.rsplit('/', 1)[-1] 56 | self.cur_instances.append(display) 57 | self.display_to_section[display] = i_section 58 | super(DeleteForm, self).__init__(*args, **keywords) 59 | 60 | def create(self): 61 | """ Creates the necessary display for this form """ 62 | self.add_handlers({'^E': self.quit, '^Q': self.quit}) 63 | to_delete = self.old_instances - self.new_instances 64 | self.add(npyscreen.Textfield, value='Select which instances to delete' 65 | ' (you must select ' + str(to_delete) + 66 | ' instance(s) to delete):', editable=False, color='GOOD') 67 | self.del_instances = self.add(InstanceSelect, 68 | values=self.cur_instances, 69 | scroll_exit=True, rely=3, 70 | instance_num=to_delete) 71 | 72 | def change_screens(self): 73 | """ Change to the next tool to edit or back to MAIN form """ 74 | if self.next_tool: 75 | self.parentApp.change_form(self.next_tool) 76 | else: 77 | self.parentApp.change_form('MAIN') 78 | 79 | def quit(self, *args, **kargs): 80 | """ Quit without making any changes to the tool """ 81 | npyscreen.notify_confirm('No changes made to instance(s)', 82 | title='Instance confiugration cancelled') 83 | self.change_screens() 84 | 85 | def on_ok(self): 86 | """ Delete the instances that the user chose to delete """ 87 | npyscreen.notify_wait('Deleting instances given...', 88 | title='In progress') 89 | # keep track of number for shifting instances down for alignment 90 | shift_num = 1 91 | to_update = [] 92 | for i, val in enumerate(self.del_instances.values): 93 | # clean all tools for renmaing and relabeling purposes 94 | t = val.split(':') 95 | section = self.display_to_section[val] 96 | prev_running = self.manifest.option(section, 'running') 97 | run = prev_running[0] and prev_running[1] == 'yes' 98 | if i in self.del_instances.value: 99 | if run: 100 | # grab dependencies of tools that linked to previous one 101 | if i == 0: 102 | dependent_tools = [self.manifest.option( 103 | section, 'link_name')[1]] 104 | for dependency in Dependencies(dependent_tools): 105 | self.clean(**dependency) 106 | to_update.append(dependency) 107 | self.clean(name=t[0], branch=t[1], version=t[2]) 108 | self.manifest.del_section(section) 109 | else: 110 | try: 111 | # update instances for tools remaining 112 | section = self.display_to_section[val] 113 | settings_dict = json.loads(self.manifest.option 114 | (section, 'settings')[1]) 115 | settings_dict['instances'] = self.new_instances 116 | self.manifest.set_option(section, 'settings', 117 | json.dumps(settings_dict)) 118 | # check if tool name doesn't need to be shifted because 119 | # it's already correct 120 | identifier = str(shift_num) if shift_num != 1 else '' 121 | new_section = section.rsplit(':', 2) 122 | new_section[0] = re.sub(r'\d+$', identifier, 123 | new_section[0]) 124 | new_section = ':'.join(new_section) 125 | if section != new_section: 126 | # clean tool so that we can rename its container and 127 | # labels with new information 128 | self.clean(name=t[0], branch=t[1], version=t[2]) 129 | prev_name = self.manifest.option(section, 'name')[1] 130 | new_name = re.split(r'[0-9]', prev_name)[0] + \ 131 | identifier 132 | self.manifest.set_option(section, 'name', new_name) 133 | # copy new contents into shifted version 134 | self.manifest.add_section(new_section) 135 | for val_pair in self.manifest.section(section)[1]: 136 | self.manifest.set_option(new_section, val_pair[0], 137 | val_pair[1]) 138 | self.manifest.del_section(section) 139 | if run: 140 | to_update.append({'name': new_name, 141 | 'branch': t[1], 142 | 'version': t[2]}) 143 | shift_num += 1 144 | except Exception as e: 145 | npyscreen.notify_confirm('Trouble deleting tools' 146 | ' because ' + str(e)) 147 | self.manifest.write_config() 148 | tool_d = {} 149 | for tool in to_update: 150 | tool_d.update(self.prep_start(**tool)[1]) 151 | if tool_d: 152 | self.start_tools(tool_d) 153 | npyscreen.notify_confirm('Done deleting instances.', 154 | title='Finished') 155 | self.change_screens() 156 | 157 | def on_cancel(self): 158 | """ Exits the form without performing any action """ 159 | self.quit() 160 | -------------------------------------------------------------------------------- /vent/menus/help.py: -------------------------------------------------------------------------------- 1 | import npyscreen 2 | 3 | 4 | class HelpForm(npyscreen.ActionFormWithMenus): 5 | """ Help form for the Vent CLI """ 6 | 7 | @staticmethod 8 | def switch(page): 9 | def popup(page): 10 | info_str = '' 11 | if page == 'Menu': 12 | info_str = """ 13 | Menu interactions are simple! Here is a quick guide to get you 14 | familiar. 15 | 16 | Navigation of a page: Up, Down, Left, Right, or TAB. Note that 17 | SHIFT+TAB can be used to reverse cycle! 18 | 19 | Editing a page: Simply navigating to an editable field and 20 | typing should be enough to edit most pages. ENTER can you be 21 | used to select or deselect options, or to open drop down menus. 22 | 23 | CTRL+T: Will toggle between two pages. 24 | CTRL+Q: Will take you back to main. Or from main, will exit the 25 | application. 26 | CTRL+X: Can be used to open up menus on certain pages. 27 | """ 28 | elif page == 'Plugins': 29 | info_str = """ 30 | Plugins are user created software hosted on GitHub that Vent 31 | can install and run. Plugins are developed following a hybrid 32 | of requirements specified both by Docker and Vent. Vent uses 33 | Docker to run all plugins so all plugins should be designed to 34 | run as a system of containers. Knowledge of linking docker 35 | containers may be necessary for more complex tasks that require 36 | creating multiple containers for your plugin. For Help on 37 | building Plugins, check out the Working with Plugins section in 38 | our Help Menu.""" 39 | elif page == 'Tools': 40 | info_str = """ 41 | Tools are the individual building blocks of a Plugin. Each tool 42 | should follow S.R.P, and over the entirety of the Plugin should 43 | be able accomplish any task desired! For Help on building 44 | Tools, check out the Working with Plugins section in our Help 45 | Menu.""" 46 | elif page == 'Filetypes': 47 | info_str = """ 48 | The filetypes Vent can support are entirely based on the 49 | installed Plugins. Each plugin is ultimately responsible for 50 | doing some form of processing.""" 51 | elif page == 'Status': 52 | info_str = """ 53 | You'll notice Vent offers several status types amongst 54 | tools/plugins. Built means that each tool has a Docker image 55 | successfully built based off the provided specs for that 56 | tool/plugin. Enabled/Disabled correspond to user defined 57 | settings to enable or disable a tool or set of tools (plugin). 58 | Installed means simply that the plugin has been cloned from 59 | GitHub and installed to the Vent filesystem. No Docker image 60 | has been created yet. Running means that a Docker container has 61 | successfully been created from the corresponding Docker image 62 | for a specific tool in a Plugin.""" 63 | elif page == 'Plugin Adding': 64 | info_str = """ 65 | To add a plugin that you've created, simply open up the Menu 66 | from the main page using ^X. After, press "p" to open up the 67 | Plugin menu and then "a" to drop down into our Plugin 68 | installation screen. To add a Plugin, we require a valid 69 | GitHub repository. If your repository is private, you will 70 | need to enter a username and password. Once you have finished 71 | that, select OK. If we are successfully able to connect, you 72 | should see your repositories branches listed in our Plugin 73 | options menu. From here, press TAB to cycle between the 74 | options, and ENTER to select different branches to install and 75 | build from. You can even choose a specific commit if you like! 76 | Once you've selected those tools and selected OK, Vent will 77 | notify you about all tools it has detected. For more 78 | information about how Vent detects tools, see our "Building a 79 | Plugin" section. You may select or deselect the tools you wish 80 | to install as part of your Plugin. When you are done, select 81 | OK. If everything works you should get a successful Add. Select 82 | OK, to be returned to the main screen!""" 83 | elif page == 'Plugin Building': 84 | # !! TODO 85 | info_str = """Stay tuned!""" 86 | npyscreen.notify_confirm(info_str, 87 | title='About Vent ' + page, 88 | wide=True) 89 | popup(page) 90 | 91 | def create(self): 92 | """ Override method for creating FormBaseNew form """ 93 | self.add_handlers({'^T': self.change_forms, '^Q': self.exit}) 94 | self.addfield = self.add(npyscreen.TitleFixedText, name='Vent', 95 | labelColor='DEFAULT', editable=False) 96 | self.multifield1 = self.add(npyscreen.MultiLineEdit, editable=False, 97 | value=""" 98 | About Vent 99 | 100 | Vent is a library that includes a CLI designed to serve as a general 101 | platform for analyzing network traffic. Built with some basic 102 | functionality, Vent serves as a user-friendly platform to build custom 103 | plugins on to perform user-defined processing on incoming network data. 104 | Vent supports any filetype, but only processes ones based on the types 105 | of plugins installed for that instance of Vent. Simply create your 106 | plugins, point vent to them & install them, and drop a file in vent to 107 | begin processing! 108 | 109 | For a detailed explanation of Vent Concepts, check out the General 110 | section in our Help Menu. Topics include: Vent Plugins, Tools, 111 | Filetypes, and Statuses! Use ^X to access the menu and ESC to 112 | close it. 113 | 114 | Select CANCEL or ^Q to return to the Main Menu. Select OK or ^T to 115 | return to your previous menu. 116 | 117 | PRO TIP: You can use TAB to cycle through options. 118 | """) 119 | self.m2 = self.add_menu(name='Vent Basics', shortcut='b') 120 | self.m2.addItem(text='Menu Interactions', onSelect=HelpForm.switch, 121 | arguments=['Menu'], shortcut='m') 122 | self.m2.addItem(text='Plugins', onSelect=HelpForm.switch, 123 | arguments=['Plugins'], shortcut='p') 124 | self.m2.addItem(text='Tools', onSelect=HelpForm.switch, 125 | arguments=['Tools'], shortcut='t') 126 | self.m2.addItem(text='Filetypes', onSelect=HelpForm.switch, 127 | arguments=['Filetypes'], shortcut='f') 128 | self.m2.addItem(text='Statuses', onSelect=HelpForm.switch, 129 | arguments=['Status'], shortcut='s') 130 | self.m3 = self.add_menu(name='Working with Plugins', shortcut='p') 131 | self.m3.addItem(text='Adding a Plugin', onSelect=HelpForm.switch, 132 | arguments=['Plugin Adding'], shortcut='a') 133 | self.m3.addItem(text='Building a Plugin', onSelect=HelpForm.switch, 134 | arguments=['Plugin Building'], shortcut='b') 135 | 136 | def exit(self, *args, **keywords): 137 | self.parentApp.switchForm('MAIN') 138 | 139 | def on_cancel(self): 140 | self.exit() 141 | 142 | def on_ok(self): 143 | self.change_forms() 144 | 145 | def change_forms(self, *args, **keywords): 146 | """ 147 | Checks which form is currently displayed and toggles to the other one 148 | """ 149 | # Returns to previous Form in history if there is a previous Form 150 | try: 151 | self.parentApp.switchFormPrevious() 152 | except Exception as e: # pragma: no cover 153 | self.parentApp.switchForm('MAIN') 154 | -------------------------------------------------------------------------------- /vent/menus/inventory.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | 3 | import npyscreen 4 | 5 | from vent.helpers.templates import Template 6 | 7 | 8 | class InventoryForm(npyscreen.FormBaseNew): 9 | """ Inventory form for the Vent CLI """ 10 | 11 | def __init__(self, action=None, logger=None, *args, **keywords): 12 | """ Initialize inventory form objects """ 13 | self.action = action 14 | self.logger = logger 15 | self.api_action = self.action['api_action'] 16 | # get list of all possible group views to display 17 | self.views = deque() 18 | possible_groups = set() 19 | manifest = Template(self.api_action.manifest) 20 | tools = self.api_action.inventory(choices=['tools'])[1]['tools'] 21 | for tool in tools: 22 | groups = manifest.option(tool, 'groups')[1].split(',') 23 | for group in groups: 24 | # don't do core because that's the purpose of all in views 25 | if group != '' and group != 'core': 26 | possible_groups.add(group) 27 | self.views += possible_groups 28 | self.views.append('all groups') 29 | super(InventoryForm, self).__init__(*args, **keywords) 30 | 31 | def quit(self, *args, **kwargs): 32 | """ Overridden to switch back to MAIN form """ 33 | self.parentApp.switchForm('MAIN') 34 | 35 | def toggle_view(self, *args, **kwargs): 36 | group = self.views.popleft() 37 | new_display = [] 38 | new_display.append('Tools for group ' + group + ' found:') 39 | manifest = Template(self.api_action.manifest) 40 | cur_repo = '' 41 | for i in range(1, len(self.all_tools) - 1): 42 | val = self.all_tools[i] 43 | # get repo val 44 | if val.startswith(' Plugin:'): 45 | new_display.append(val) 46 | cur_repo = val.split(':', 1)[1].strip() 47 | # determine if tool should be displayed in this group 48 | elif val.startswith(' ') and not val.startswith(' '): 49 | name = val.strip() 50 | constraints = {'repo': cur_repo, 'name': name} 51 | t_section = manifest.constrain_opts(constraints, [])[0] 52 | t_section = list(t_section.keys())[0] 53 | if group in manifest.option(t_section, 'groups')[1].split(','): 54 | new_display += self.all_tools[i:i+4] 55 | elif val == '': 56 | new_display.append(val) 57 | # if all groups display all groups 58 | if group == 'all groups': 59 | self.display_val.values = self.all_tools 60 | else: 61 | self.display_val.values = new_display 62 | # redraw 63 | self.display() 64 | # add group back into cycle 65 | self.views.append(group) 66 | 67 | def create(self): 68 | """ Override method for creating FormBaseNew form """ 69 | self.add_handlers({'^T': self.quit, '^Q': self.quit, 70 | '^V': self.toggle_view}) 71 | self.add(npyscreen.TitleFixedText, name=self.action['title'], value='') 72 | response = self.action['api_action'].inventory(choices=['repos', 73 | 'tools', 74 | 'images', 75 | 'built', 76 | 'running']) 77 | if response[0]: 78 | inventory = response[1] 79 | if len(inventory['repos']) == 0: 80 | value = 'No tools were found.\n' 81 | else: 82 | value = 'Tools for all groups found:\n' 83 | tools = None 84 | if inventory['tools']: 85 | tools = inventory['tools'] 86 | 87 | for repo in inventory['repos']: 88 | s_value = '' 89 | repo_name = repo.rsplit('/', 2)[1:] 90 | if len(repo_name) == 1: 91 | repo_name = repo.split('/') 92 | if tools: 93 | p_value = '\n Plugin: ' + repo + '\n' 94 | for tool in tools: 95 | t_name = tool.split(':') 96 | if (t_name[0] == repo_name[0] and 97 | t_name[1] == repo_name[1]): 98 | s_value += ' ' + tools[tool] + '\n Built: ' 99 | s_value += inventory['built'][tool] + '\n' 100 | s_value += ' Image name: ' 101 | s_value += inventory['images'][tool] + '\n' 102 | s_value += ' Status: ' 103 | s_value += inventory['running'][tool] + '\n' 104 | if s_value: 105 | value += p_value + s_value 106 | else: 107 | value = 'There was an issue with ' + self.action['name'] 108 | value += ' retrieval:\n' + str(response[1]) 109 | value += '\nPlease see vent.log for more details.' 110 | self.all_tools = value.split('\n') 111 | self.display_val = self.add(npyscreen.Pager, values=value.split('\n')) 112 | -------------------------------------------------------------------------------- /vent/menus/inventory_forms.py: -------------------------------------------------------------------------------- 1 | from vent.api.tools import Tools 2 | from vent.helpers.logs import Logger 3 | from vent.menus.inventory import InventoryForm 4 | 5 | 6 | class BaseInventoryForm(InventoryForm): 7 | """ Base form to inherit from """ 8 | 9 | def __init__(self, action_dict=None, action_name=None, *args, **keywords): 10 | api_action = Tools() 11 | action = {'api_action': api_action} 12 | if action_dict: 13 | action.update(action_dict) 14 | logger = Logger(action_name) 15 | InventoryForm.__init__(self, action, logger, *args, **keywords) 16 | 17 | 18 | class InventoryToolsForm(BaseInventoryForm): 19 | """ Inventory Tools form for the Vent CLI """ 20 | 21 | def __init__(self, *args, **keywords): 22 | """ Initialize inventory tools form objects """ 23 | action_name = 'inventory' 24 | action_dict = {'title': 'Inventory of tools:', 25 | 'name': 'inventory', 26 | 'cores': False} 27 | BaseInventoryForm.__init__(self, 28 | action_dict, 29 | action_name, 30 | *args, 31 | **keywords) 32 | -------------------------------------------------------------------------------- /vent/menus/services.py: -------------------------------------------------------------------------------- 1 | import npyscreen 2 | 3 | from vent.helpers.meta import Services 4 | 5 | 6 | class ServicesForm(npyscreen.FormBaseNew): 7 | """ Services form for the Vent CLI """ 8 | 9 | def __init__(self, *args, **keywords): 10 | """ Initialize service form objects """ 11 | self.core = keywords['core'] 12 | if 'external' in keywords: 13 | self.external = keywords['external'] 14 | else: 15 | self.external = False 16 | super(ServicesForm, self).__init__(*args, **keywords) 17 | 18 | def quit(self, *args, **kwargs): 19 | """ Overridden to switch back to MAIN form """ 20 | self.parentApp.switchForm('MAIN') 21 | 22 | def create(self): 23 | """ Override method for creating FormBaseNew form """ 24 | self.add_handlers({'^T': self.quit, '^Q': self.quit}) 25 | self.services_tft = self.add(npyscreen.TitleFixedText, 26 | name='No services running.', 27 | value='') 28 | services = Services(self.core, external=self.external) 29 | if services: 30 | self.services_tft.hidden = True 31 | for service in services: 32 | value = '' 33 | for val in service[1]: 34 | value += val+', ' 35 | self.add(npyscreen.TitleFixedText, 36 | name=service[0], 37 | value=value[:-2]) 38 | -------------------------------------------------------------------------------- /vent/menus/tutorials.py: -------------------------------------------------------------------------------- 1 | import npyscreen 2 | 3 | 4 | class TutorialForm(npyscreen.ActionFormWithMenus): 5 | """ Tutorial form for the Vent CLI """ 6 | 7 | def __init__(self, title='', text='', next_tutorial='', *args, **keywords): 8 | """ Initialize tutorial form fields """ 9 | self.title = title 10 | self.text = text 11 | self.next_tutorial = next_tutorial 12 | super(TutorialForm, self).__init__(*args, **keywords) 13 | 14 | def switch(self, name): 15 | """ Wrapper that switches to provided form """ 16 | self.parentApp.change_form(name) 17 | 18 | def quit(self, *args, **kwargs): 19 | """ Overridden to switch back to MAIN form """ 20 | self.parentApp.switchForm('MAIN') 21 | 22 | def create(self): 23 | """ Overridden to add handlers and content """ 24 | self.add_handlers({'^Q': self.quit}) 25 | self.add(npyscreen.TitleText, name=self.title, editable=False) 26 | self.add(npyscreen.MultiLineEdit, editable=False, value=self.text, 27 | max_width=75, slow_scroll=True) 28 | self.m2 = self.add_menu(name='About Vent', shortcut='v') 29 | self.m2.addItem(text='Background', onSelect=self.switch, 30 | arguments=['TUTORIALBACKGROUND'], shortcut='b') 31 | self.m2.addItem(text='Terminology', onSelect=self.switch, 32 | arguments=['TUTORIALTERMINOLOGY'], shortcut='t') 33 | self.m2.addItem(text='Getting Setup', onSelect=self.switch, 34 | arguments=['TUTORIALGETTINGSETUP'], shortcut='s') 35 | self.m3 = self.add_menu(name='Working with Cores', shortcut='c') 36 | self.m3.addItem(text='Starting Cores', onSelect=self.switch, 37 | arguments=['TUTORIALSTARTINGCORES'], shortcut='c') 38 | self.m4 = self.add_menu(name='Working with Plugins', shortcut='p') 39 | self.m4.addItem(text='Adding Plugins', onSelect=self.switch, 40 | arguments=['TUTORIALADDINGPLUGINS'], shortcut='a') 41 | self.m5 = self.add_menu(name='Files', shortcut='f') 42 | self.m5.addItem(text='Adding Files', onSelect=self.switch, 43 | arguments=['TUTORIALADDINGFILES'], shortcut='a') 44 | self.m6 = self.add_menu(name='Help', shortcut='s') 45 | self.m6.addItem(text='Basic Troubleshooting', onSelect=self.switch, 46 | arguments=['TUTORIALTROUBLESHOOTING'], shortcut='t') 47 | 48 | def on_cancel(self): 49 | """ When user clicks cancel, will return to MAIN """ 50 | self.quit() 51 | 52 | def on_ok(self): 53 | """ When user clicks ok, will proceed to next tutorial """ 54 | self.switch(self.next_tutorial) 55 | --------------------------------------------------------------------------------