├── CODE_OF_CONDUCT.md ├── docs ├── developer.md ├── workflows.md ├── pre-commit-config.md ├── pyproject.md └── pylint.md ├── src ├── osintbuddy │ ├── templates │ │ ├── __init__.py │ │ └── default.py │ ├── server.py │ ├── errors.py │ ├── utils │ │ ├── __init__.py │ │ └── generic.py │ ├── elements │ │ ├── __init__.py │ │ ├── base.py │ │ ├── displays.py │ │ └── inputs.py │ ├── __init__.py │ ├── ob.py │ ├── ascii.py │ └── plugins.py └── README.md ├── .pypirc ├── .github └── workflows │ ├── publish.yml │ └── CI.yml ├── Dockerfile ├── requirements.txt ├── tests ├── test_methods.py ├── plugins.py └── conftest.py ├── LICENSE ├── .pre-commit-config.yaml ├── .gitignore ├── README.md └── pyproject.toml /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | @todo -------------------------------------------------------------------------------- /docs/developer.md: -------------------------------------------------------------------------------- 1 | # Developer Guide 2 | 3 | ## Testing Template Project 4 | -------------------------------------------------------------------------------- /src/osintbuddy/templates/__init__.py: -------------------------------------------------------------------------------- 1 | from .default import plugin_source_template -------------------------------------------------------------------------------- /src/osintbuddy/server.py: -------------------------------------------------------------------------------- 1 | 2 | from fastapi import FastAPI 3 | import osintbuddy 4 | 5 | app = FastAPI(title=f"OSINTBuddy Plugins v{osintbuddy.__version__}") 6 | -------------------------------------------------------------------------------- /.pypirc: -------------------------------------------------------------------------------- 1 | [distutils] 2 | index-servers = 3 | pypi 4 | testpypi 5 | 6 | [pypi] 7 | repository = https://upload.pypi.org/legacy/ 8 | 9 | [testpypi] 10 | repository = https://test.pypi.org/legacy/ 11 | -------------------------------------------------------------------------------- /src/osintbuddy/errors.py: -------------------------------------------------------------------------------- 1 | class OBPluginError(Exception): 2 | pass 3 | 4 | 5 | class NodeInvalidValueError(OBPluginError): 6 | pass 7 | 8 | 9 | class NodeMissingValueError(OBPluginError): 10 | pass 11 | -------------------------------------------------------------------------------- /src/osintbuddy/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .generic import ( 2 | MAP_KEY, 3 | chunks, 4 | find_emails, 5 | to_clean_domain, 6 | slugify, 7 | to_camel_case, 8 | to_snake_case, 9 | dkeys_to_snake_case 10 | ) -------------------------------------------------------------------------------- /docs/workflows.md: -------------------------------------------------------------------------------- 1 | # GitHub Workflow for Python 2 | 3 | The main workflow file is ".github/workflows/CI.yml". This performs linting, testing, and publishing for Python packages. 4 | It can also be triggered manually on a specific branch. 5 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Python Publish Workflow 2 | on: 3 | workflow_call: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | publish: 8 | uses: microsoft/action-python/.github/workflows/publish.yml@0.2.0 9 | secrets: 10 | PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 11 | TEST_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.16-alpine 2 | 3 | RUN python -m pip install --upgrade pip \ 4 | && python -m pip install 'flit>=3.8.0' 5 | 6 | ENV FLIT_ROOT_INSTALL=1 7 | 8 | COPY pyproject.toml . 9 | RUN touch README.md \ 10 | && mkdir -p src/osintbuddy \ 11 | && python -m flit install --only-deps --deps develop \ 12 | && rm -r pyproject.toml README.md src 13 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | release: 8 | types: [created] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | publish: 13 | uses: microsoft/action-python/.github/workflows/publish.yml@0.2.0 14 | secrets: 15 | PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 16 | TEST_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} 17 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | selenium>=4.9.0 2 | sqlalchemy-json==0.7.0 3 | SQLAlchemy-Utils==0.41.1 4 | playwright==1.39.0 5 | httpx>=0.25.0 6 | beautifulsoup4==4.12.2 7 | gremlinpy @ git+https://github.com/jerlendds/gremlinpy.git@eaba7dca12ad0156eb0d6d8ba2eb5751551c6a6d 8 | pyfiglet==0.8.post1 9 | termcolor==2.3.0 10 | fastapi==0.103.2 11 | uvicorn==0.22.0 12 | uvloop==0.17.0 13 | pydantic==2.4.2 14 | pydantic-settings==2.0.3 15 | yq==3.2.3 16 | jedi-language-server==0.41.1 17 | websockets==11.0.3 18 | -------------------------------------------------------------------------------- /docs/pre-commit-config.md: -------------------------------------------------------------------------------- 1 | # pre-commit-config.yaml 2 | 3 | Pre-commit is a Python package which can be used to create 'git' hooks which scan can prior to checkins. 4 | The included configuration focuses on python actions which will help to prevent users from commiting code which will fail during builds. 5 | In general, only formatting actions are automatiicaly performed. These include auto-formatting with 'black', or sorting dependacies with 'isort'. 6 | Linting actions are left to the discretion of the user. 7 | -------------------------------------------------------------------------------- /docs/pyproject.md: -------------------------------------------------------------------------------- 1 | # pypyroject.toml 2 | 3 | The pyproject.toml is the main configuration file used for the Python project. 4 | It contains configurations for building, linting, testing, and publishing the Python package. 5 | 6 | The pyproject.toml replaces the "setup.py" package. When using 'flit' or 'poetry', only the pyproject.toml is required. 7 | This project currently uses 'flit', but in the future may also include a 'poetry' example. Both are considered viable options. 8 | 9 | When using setuptools, and setup.cfg is still required. 10 | -------------------------------------------------------------------------------- /src/osintbuddy/elements/__init__.py: -------------------------------------------------------------------------------- 1 | from osintbuddy.elements.base import BaseElement, BaseInput, BaseDisplay # noqa 2 | from osintbuddy.elements.displays import ( # noqa 3 | Title, 4 | Text, 5 | CopyText, 6 | CopyCode, 7 | Json, 8 | Image, 9 | Video, 10 | Pdf, 11 | List, 12 | Table, 13 | Empty 14 | ) 15 | from osintbuddy.elements.inputs import ( # noqa 16 | UploadFileInput, 17 | TextInput, 18 | DropdownInput, 19 | NumberInput, 20 | DecimalInput, 21 | TextAreaInput 22 | ) 23 | -------------------------------------------------------------------------------- /src/osintbuddy/__init__.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Licensed under the MIT License. See LICENSE in project root for information. 3 | # ------------------------------------------------------------- 4 | """Python Package Template""" 5 | from __future__ import annotations 6 | from osintbuddy.plugins import ( 7 | OBRegistry as Registry, 8 | OBPlugin as Plugin, 9 | OBAuthorUse as PluginUse, 10 | discover_plugins, 11 | transform, 12 | load_plugin, 13 | load_plugins 14 | ) 15 | 16 | __version__ = "0.0.4" 17 | -------------------------------------------------------------------------------- /tests/test_methods.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # Licensed under the MIT License. See LICENSE in project root for information. 4 | # --------------------------------------------------------------------------------- 5 | from __future__ import annotations 6 | from osintbuddy import OBRegistry, OBPlugin, transform, discover_plugins 7 | from osintbuddy.node import TextInput 8 | 9 | 10 | # Mock 11 | def test_ob_registry(): 12 | assert OBRegistry.plugins == [] 13 | -------------------------------------------------------------------------------- /tests/plugins.py: -------------------------------------------------------------------------------- 1 | from osintbuddy import OBPlugin, transform 2 | from osintbuddy.node import TextInput 3 | 4 | class WebsitePlugin(OBPlugin): 5 | label = 'Website' 6 | name = 'Website' 7 | color = '#1D1DB8' 8 | icon = 'world-www' 9 | node = [ 10 | TextInput(label='Domain', icon='world-www'), 11 | ] 12 | 13 | 14 | class UrlPlugin(OBPlugin): 15 | label = 'URL' 16 | name = 'URL' 17 | color = '#642CA9' 18 | node = [ 19 | TextInput(label='URL', icon='link'), 20 | ] 21 | 22 | @transform(label='To website', icon='world-www') 23 | def transform_to_website(self, node, **kwargs): 24 | return WebsitePlugin.blueprint(domain='google.com') 25 | 26 | -------------------------------------------------------------------------------- /src/osintbuddy/templates/default.py: -------------------------------------------------------------------------------- 1 | def plugin_source_template(label: str, description: str, author: str) -> str: 2 | class_name = ''.join(x for x in filter(str.isalnum, label.title()) if not x.isspace()) 3 | 4 | return f"""import osintbuddy as ob 5 | from osintbuddy.elements import TextInput 6 | 7 | class {class_name}(ob.Plugin): 8 | label = '{label}' 9 | icon = 'atom-2' # https://tabler-icons.io/ 10 | color = '#FFD166' 11 | 12 | author = '{author}' 13 | description = '{description}' 14 | 15 | node = [ 16 | TextInput(label='Example', icon='radioactive') 17 | ] 18 | 19 | @ob.transform(label='To example', icon='atom-2') 20 | async def transform_example(self, node, use): 21 | WebsitePlugin = await ob.Registry.get_plugin('website') 22 | website_plugin = WebsitePlugin() 23 | return website_plugin.blueprint(domain=node.example) 24 | """ 25 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | 3 | Copyright 2023 jerlendds 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # Licensed under the MIT License. See LICENSE in project root for information. 4 | # --------------------------------------------------------------------------------- 5 | """ 6 | This is a configuration file for pytest containing customizations and fixtures. 7 | 8 | In VSCode, Code Coverage is recorded in config.xml. Delete this file to reset reporting. 9 | """ 10 | 11 | from __future__ import annotations 12 | 13 | import pytest 14 | from _pytest.nodes import Item 15 | 16 | 17 | def pytest_collection_modifyitems(items: list[Item]): 18 | for item in items: 19 | if "spark" in item.nodeid: 20 | item.add_marker(pytest.mark.spark) 21 | elif "_int_" in item.nodeid: 22 | item.add_marker(pytest.mark.integration) 23 | 24 | 25 | @pytest.fixture 26 | def unit_test_mocks(monkeypatch: None): 27 | """Include Mocks here to execute all commands offline and fast.""" 28 | pass 29 | -------------------------------------------------------------------------------- /src/osintbuddy/elements/base.py: -------------------------------------------------------------------------------- 1 | class BaseElement(object): 2 | """ 3 | The BaseElement class represents a basic building block used in OsintBuddy 4 | plugins. It is designed to implement the base styles used 5 | in other nodes that can render a nodes element 6 | with a specific element type, label, and style on the OSINTbuddy UI. 7 | 8 | label : str 9 | A string representing the label for the node. 10 | style : dict 11 | A dictionary representing the react style properties for the node. 12 | placeholder : str 13 | A string representing the placeholder for the node. 14 | 15 | _base_blueprint(self) -> dict: 16 | Returns a dictionary containing essential data for a node 17 | element type, such as the node type, label, 18 | placeholder, and style. 19 | """ 20 | def __init__(self, **kwargs): 21 | self.label: str = '' 22 | if entity_type := kwargs.get('label'): 23 | setattr(self, 'label', entity_type) 24 | 25 | def _base_entity_element(self, **kwargs): 26 | base_element = {} 27 | if kwargs: 28 | base_element = { 29 | k: v for k, v in kwargs.items() 30 | } 31 | base_element['label'] = self.label 32 | base_element['type'] = self.element_type 33 | return base_element 34 | 35 | class BaseInput(BaseElement): 36 | pass 37 | 38 | 39 | class BaseDisplay(BaseElement): 40 | pass 41 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autoupdate_commit_msg: "chore: update pre-commit hooks" 3 | autofix_commit_msg: "style: pre-commit fixes" 4 | 5 | repos: 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v4.1.0 8 | hooks: 9 | - id: check-added-large-files 10 | - id: check-case-conflict 11 | - id: check-merge-conflict 12 | - id: check-symlinks 13 | - id: check-yaml 14 | - id: debug-statements 15 | - id: end-of-file-fixer 16 | - id: mixed-line-ending 17 | - id: requirements-txt-fixer 18 | - id: trailing-whitespace 19 | 20 | - repo: https://github.com/PyCQA/isort 21 | rev: 5.12.0 22 | hooks: 23 | - id: isort 24 | args: ["-a", "from __future__ import annotations"] 25 | 26 | - repo: https://github.com/asottile/pyupgrade 27 | rev: v2.31.0 28 | hooks: 29 | - id: pyupgrade 30 | args: [--py37-plus] 31 | 32 | - repo: https://github.com/hadialqattan/pycln 33 | rev: v1.2.5 34 | hooks: 35 | - id: pycln 36 | args: [--config=pyproject.toml] 37 | stages: [manual] 38 | 39 | - repo: https://github.com/codespell-project/codespell 40 | rev: v2.1.0 41 | hooks: 42 | - id: codespell 43 | 44 | - repo: https://github.com/pre-commit/pygrep-hooks 45 | rev: v1.9.0 46 | hooks: 47 | - id: python-check-blanket-noqa 48 | - id: python-check-blanket-type-ignore 49 | - id: python-no-log-warn 50 | - id: python-no-eval 51 | - id: python-use-type-annotations 52 | - id: rst-backticks 53 | - id: rst-directive-colons 54 | - id: rst-inline-touching-normal 55 | 56 | - repo: https://github.com/mgedmin/check-manifest 57 | rev: "0.47" 58 | hooks: 59 | - id: check-manifest 60 | stages: [manual] 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | .vscode 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | pip-wheel-metadata/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # pipenv 89 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 90 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 91 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 92 | # install all needed dependencies. 93 | #Pipfile.lock 94 | 95 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 96 | __pypackages__/ 97 | 98 | # Celery stuff 99 | celerybeat-schedule 100 | celerybeat.pid 101 | 102 | # SageMath parsed files 103 | *.sage.py 104 | 105 | # Environments 106 | .env 107 | .venv 108 | env/ 109 | venv/ 110 | ENV/ 111 | env.bak/ 112 | venv.bak/ 113 | 114 | # Spyder project settings 115 | .spyderproject 116 | .spyproject 117 | 118 | # Rope project settings 119 | .ropeproject 120 | 121 | # mkdocs documentation 122 | /site 123 | 124 | # mypy 125 | .mypy_cache/ 126 | .dmypy.json 127 | dmypy.json 128 | 129 | # Pyre type checker 130 | .pyre/ 131 | -------------------------------------------------------------------------------- /src/osintbuddy/utils/generic.py: -------------------------------------------------------------------------------- 1 | import re 2 | import unicodedata 3 | from typing import List, Union 4 | from urllib import parse 5 | from pydantic import EmailStr 6 | 7 | 8 | MAP_KEY = '___obmap___' 9 | 10 | 11 | def chunks(lst, n): 12 | """Yield successive n-sized chunks from lst.""" 13 | for i in range(0, len(lst), n): 14 | yield lst[i:i + n] 15 | 16 | 17 | def find_emails(value: str) -> List[EmailStr]: 18 | emails = [] 19 | match = re.search(r"[\w.+-]+@[\w-]+\.[\w.-]+", value) 20 | if match is not None: 21 | email = match.group(0) 22 | # if trailing dot, remove. @todo improve regex 23 | if email[len(email) - 1] == ".": 24 | emails.append(email[0: len(email) - 2]) 25 | else: 26 | emails.append(email) 27 | return list(set(emails)) 28 | 29 | 30 | def to_clean_domain(value: str) -> str: 31 | if "http://" not in value and "https://" not in value: 32 | value = "https://" + value 33 | url = parse.urlparse(value) 34 | split_domain = url.netloc.split(".") 35 | if len(split_domain) >= 3: 36 | split_domain.pop(0) 37 | domain = ".".join(split_domain) 38 | return domain 39 | 40 | 41 | # Slugify and related code is from the Django project, thanks guys! 42 | # Project URL: https://github.com/django/django 43 | # https://github.com/django/django/blob/main/django/utils/text.py 44 | def slugify(value, allow_unicode=False): 45 | """ 46 | Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated 47 | dashes to single dashes. Remove characters that aren't alphanumerics, 48 | underscores, or hyphens. Convert to lowercase. Also strip leading and 49 | trailing whitespace, dashes, and underscores. 50 | """ 51 | value = str(value) 52 | if allow_unicode: 53 | value = unicodedata.normalize("NFKC", value) 54 | else: 55 | value = ( 56 | unicodedata.normalize("NFKD", value) 57 | .encode("ascii", "ignore") 58 | .decode("ascii") 59 | ) 60 | value = re.sub(r"[^\w\s-]", "", value.lower()) 61 | return re.sub(r"[-\s]+", "-", value).strip("-_") 62 | 63 | 64 | def to_camel_case(value: str): 65 | value_list = value.replace(' ', '_').lower().split('_') 66 | return value_list[0] + ''.join(e.title() for e in value_list[1:]) 67 | 68 | 69 | def to_snake_case(name): 70 | name = to_camel_case(name.replace('-', '_')) 71 | name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) 72 | name = re.sub('__([A-Z])', r'_\1', name) 73 | name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', name) 74 | return name.lower() 75 | 76 | # Convert all keys in dict to snake_case 77 | def dkeys_to_snake_case(data: dict) -> Union[dict, List[dict]]: 78 | def to_snake(s): 79 | return re.sub('([A-Z]\w+$)', '_\\1', s).lower() 80 | 81 | if isinstance(data, list): 82 | return [dkeys_to_snake_case(i) if isinstance(i, (dict, list)) else i for i in data] 83 | return {to_snake(a):dkeys_to_snake_case(b) if isinstance(b, (dict, list)) else b for a, b in data.items()} 84 | -------------------------------------------------------------------------------- /src/osintbuddy/elements/displays.py: -------------------------------------------------------------------------------- 1 | from osintbuddy.elements.base import BaseDisplay 2 | 3 | 4 | class Title(BaseDisplay): 5 | element_type: str = 'title' 6 | 7 | def __init__(self, value='', **kwargs): 8 | super().__init__(**kwargs) 9 | self.element = { 10 | "value": value, 11 | } 12 | 13 | def to_dict(self): 14 | return self._base_entity_element(**self.element) 15 | 16 | 17 | class Text(BaseDisplay): 18 | element_type: str = 'section' 19 | 20 | def __init__(self, value='', icon="123", **kwargs): 21 | super().__init__(**kwargs) 22 | self.element = { 23 | "value": value, 24 | "icon": icon 25 | } 26 | 27 | def to_dict(self): 28 | return self._base_entity_element(**self.element) 29 | 30 | 31 | class Empty(BaseDisplay): 32 | element_type: str = 'empty' 33 | 34 | def __init__(self, **kwargs): 35 | super().__init__(**kwargs) 36 | 37 | def to_dict(self): 38 | return self._base_entity_element() 39 | 40 | 41 | class CopyText(BaseDisplay): 42 | element_type: str = 'copy-text' 43 | 44 | def __init__(self, value='', **kwargs): 45 | super().__init__(**kwargs) 46 | self.element = { 47 | "value": value 48 | } 49 | 50 | def to_dict(self): 51 | return self._base_entity_element(**self.element) 52 | 53 | 54 | class CopyCode(BaseDisplay): 55 | element_type: str = 'copy-code' 56 | 57 | def __init__(self, value='', **kwargs): 58 | super().__init__(**kwargs) 59 | self.element = { 60 | "value": value 61 | } 62 | 63 | def to_dict(self): 64 | return self._base_entity_element(**self.element) 65 | 66 | 67 | class Json(BaseDisplay): 68 | element_type: str = 'json' 69 | 70 | def __init__(self, **kwargs): 71 | super().__init__(**kwargs) 72 | 73 | def to_dict(self): 74 | return self._base_entity_element() 75 | 76 | 77 | class Image(BaseDisplay): 78 | element_type: str = 'image' 79 | 80 | def __init__(self, **kwargs): 81 | super().__init__(**kwargs) 82 | 83 | def to_dict(self): 84 | return self._base_entity_element() 85 | 86 | 87 | class Pdf(BaseDisplay): 88 | element_type: str = 'pdf' 89 | 90 | def __init__(self, **kwargs): 91 | super().__init__(**kwargs) 92 | 93 | def to_dict(self): 94 | return self._base_entity_element() 95 | 96 | 97 | class Video(BaseDisplay): 98 | element_type: str = 'video' 99 | 100 | def __init__(self, **kwargs): 101 | super().__init__(**kwargs) 102 | 103 | def to_dict(self): 104 | return self._base_entity_element() 105 | 106 | 107 | class List(BaseDisplay): 108 | element_type: str = 'list' 109 | 110 | def __init__(self, **kwargs): 111 | super().__init__(**kwargs) 112 | 113 | def to_dict(self): 114 | return self._base_entity_element() 115 | 116 | 117 | class Table(BaseDisplay): 118 | element_type: str = 'table' 119 | 120 | def __init__(self, **kwargs): 121 | super().__init__(**kwargs) 122 | 123 | def to_dict(self): 124 | return self._base_entity_element() 125 | -------------------------------------------------------------------------------- /src/osintbuddy/ob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """OSINTBuddy plugins server script 3 | 4 | This script contains the commands needed to manage an OSINTBuddy Plugins service, which is used by the OSINTBuddy project. 5 | 6 | Basic Commands: 7 | Plugins service command(s): 8 | `start` : Starts the FastAPI microservice (`ctrl+c` to stop the microservice) 9 | `lsp` : Start the language server for code completion in the OSINTBuddy app 10 | Database Command(s): 11 | `plugin create` : Run the setup wizard for creating new plugin(s) 12 | `load $GIT_URL` : Load plugin(s) from a remote git repository 13 | """ 14 | 15 | from os import getpid, devnull 16 | from argparse import ArgumentParser, BooleanOptionalAction 17 | from pyfiglet import figlet_format 18 | from termcolor import colored 19 | import osintbuddy 20 | 21 | APP_INFO = \ 22 | """____________________________________________________________________ 23 | | Find, share, and get help with OSINTBuddy plugins: 24 | | https://forum.osintbuddy.com/c/plugin-devs/5 25 | |___________________________________________________________________ 26 | | If you run into any bugs, please file an issue on Github: 27 | | https://github.com/jerlendds/osintbuddy 28 | |___________________________________________________________________ 29 | | 30 | | OSINTBuddy plugins: v{osintbuddy_version} 31 | | PID: {pid} 32 | | Endpoint: 0.0.0.0:42562 33 | """.rstrip() 34 | 35 | 36 | def _print_server_details(): 37 | print(colored(figlet_format(f"OSINTBuddy plugins", font='smslant'), color="blue")) 38 | print(colored(APP_INFO.format( 39 | osintbuddy_version=osintbuddy.__version__, 40 | pid=getpid(), 41 | ), color="blue")) 42 | colored("Created by", color="blue"), colored("jerlendds", color="red") 43 | 44 | 45 | def _print_lsp_details(): 46 | import jedi_language_server 47 | print(colored(figlet_format(f"OSINTBuddy LSP", font='smslant'), color="blue")) 48 | colored("Created by", color="blue"), colored("jerlendds", color="red") 49 | print(colored(f"""____________________________________________________________________ 50 | | Jedi Language Server: v{jedi_language_server.__version__} 51 | | Endpoint: {'ws://0.0.0.0:54332'} 52 | """, color="blue")) 53 | 54 | def start_lsp(): 55 | _print_lsp_details() 56 | import subprocess 57 | FNULL = open(devnull, 'w') 58 | jedi_language_server = subprocess.Popen( 59 | ["ps", "aux", "|", "pkill", "jedi-language-", "&&","jedi-language-server", "--ws", "--host", "0.0.0.0", "--port", "54332"], 60 | stdout=FNULL, 61 | stderr=subprocess.STDOUT 62 | ) 63 | return jedi_language_server 64 | 65 | 66 | def start(): 67 | # jedi_language_server = start_lsp() 68 | _print_server_details() 69 | # import signal 70 | import uvicorn 71 | uvicorn.run( 72 | "osintbuddy.server:app", 73 | host="127.0.0.1", 74 | port=42562, 75 | loop='asyncio', 76 | reload=True, 77 | workers=4, 78 | headers=[('server', f"OSINTBuddy")], 79 | log_level='info' 80 | ) 81 | 82 | # def signal_handler(sig, frame): 83 | # jedi_language_server.wait(timeout=1) 84 | 85 | # signal.signal(signal.SIGINT, signal_handler) 86 | # signal.pause() 87 | 88 | 89 | def create_plugin_wizard(): 90 | # TODO: setup prompt process for initializing an osintbuddy plugin(s) project 91 | pass 92 | 93 | 94 | def main(): 95 | commands = { 96 | "lsp": start_lsp, 97 | "start": start, 98 | "plugin create": create_plugin_wizard, 99 | } 100 | parser = ArgumentParser() 101 | parser.add_argument('command', type=str, nargs="*", help="[CATEGORY (Optional)] [ACTION]") 102 | 103 | args = parser.parse_args() 104 | command = commands.get(' '.join(args.command)) 105 | 106 | if command: 107 | command() 108 | else: 109 | parser.error("Command not recognized") 110 | 111 | 112 | if __name__ == '__main__': 113 | main() 114 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | # OSINTBuddy plugins and extensions 2 | 3 | The plugins library for [jerlendds/osintbuddy](https://github.com/jerlendds/osintbuddy). Currently in beta, expect changes to come... 4 | 5 | ***Please note:** [OSINTBuddy plugins](https://github.com/jerlendds/osintbuddy-plugins) are still in early alpha and breaking changes may occasionally occur in the API. That said, if you remain on the `main` branch and avoid accessing protected methods we will try our best to avoid introducing breaking changes.* 6 | 7 | ### **NOTICE:** There has been a major update to plugins, any created plugins will have to be updated to use the new, and more convenient data access method: 8 | - Remove `name` from any `ob.Plugin` 9 | - Update `node['data']` access to be `node.label_defined_in_node` 10 | - Please see the introduction to the plugin system below 11 | 12 | 13 | The osintbuddy plugin system at its core is very simple. An `OBRegistry` class holds all registered `OBPlugin` classes within the application. This registry is loaded into the [osintbuddy application](https://github.com/jerlendds/osintbuddy/) where it is then used to load the available entities for the user when they access a project graph, load transforms when a user opens the context menu of a node, and perform transformations which expect a `Plugin.blueprint()` to be returned. The returned data of a transform decorated method will be automatically mapped to a [JanusGraph](https://janusgraph.org/) database through [gremlin](https://tinkerpop.apache.org/) according to the labels *(as snakecase)* you previously set in the classes `node` for whatever `Plugin.blueprint()` 14 | you return. 15 | 16 | To make this a bit more clear please see the below example... 17 | 18 | ```py 19 | from pydantic import BaseModel 20 | import osintbuddy import transform, Plugin 21 | from osintbuddy.elements import TextInput, DropdownInput, Title, CopyText 22 | from osintbuddy.errors import OBPluginError 23 | 24 | 25 | class CSESearchResults(Plugin): 26 | label = "CSE Result" 27 | name = "CSE result" 28 | show_label = False # do not show this on the entities dialog 29 | # the user sees on the left of the project graph screen 30 | color = "#058F63" 31 | node = [ 32 | Title(label="Result"), 33 | CopyText(label="URL"), 34 | CopyText(label="Cache URL"), 35 | ] 36 | 37 | 38 | class CSESearchPlugin(Plugin): 39 | label = "CSE Search" 40 | name = "CSE search" 41 | color = "#2C7237" 42 | node = [ 43 | [ 44 | TextInput(label="Query", icon="search"), 45 | TextInput(label="Pages", icon="123", default="1"), 46 | ], 47 | DropdownInput(label="CSE Categories", options=cse_link_options) 48 | ] 49 | 50 | @transform(label="To cse results", icon="search") 51 | async def transform_to_cse_results( 52 | self, 53 | node: BaseModel, # dynamically generated pydantic model 54 | # that is mapped from the above labels contained within `node` 55 | use # a pydantic model allowing you to access a selenium instance 56 | # (and eventually a gremlin graph and settings api) 57 | ): 58 | results = [] 59 | 60 | if not node.query: 61 | raise OBPluginError(( 62 | 'You can send error messages to the user here' 63 | 'if they forget to submit data or if some other error occurs' 64 | )) 65 | 66 | # notice how you can access data returned from the context menu 67 | # of this node; using the label name in snake case 68 | print(node.cse_categories, node.query, node.pages) 69 | 70 | ... # (removed code for clarity) 71 | 72 | if resp: 73 | for result in resp["results"]: 74 | url = result.get("breadcrumbUrl", {}) 75 | # some elements you can store more than just a string, 76 | # (these elements storing dicts are mapped 77 | # to janusgraph as properties with the names 78 | # result_title, result_subtitle, and result_text) 79 | blueprint = CSESearchResults.blueprint( 80 | result={ 81 | "title": result.get("titleNoFormatting"), 82 | "subtitle": url.get("host") + url.get("crumbs"), 83 | "text": result.get("contentNoFormatting"), 84 | }, 85 | url=result.get("unescapedUrl"), 86 | cache_url=result.get("cacheUrl"), 87 | ) 88 | results.append(blueprint) 89 | # here we return a list of blueprints (blueprints are dicts) 90 | # but you can also return a single blueprint without a list 91 | return results 92 | 93 | ``` 94 | 95 | 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OSINTBuddy plugins and extensions 2 | 3 | The plugins library for [jerlendds/osintbuddy](https://github.com/jerlendds/osintbuddy). 4 | 5 | 6 | [2023-12-02_demo.webm](https://github.com/jerlendds/osintbuddy/assets/29207058/a7feba13-d1ca-43a0-ba25-b5c899eae89c) 7 | 8 | 9 | ***Please note:** [OSINTBuddy plugins](https://github.com/jerlendds/osintbuddy-plugins) are still in early alpha and breaking changes may occasionally occur in the API. That said, if you remain on the `main` branch and avoid accessing protected methods we will try our best to avoid introducing breaking changes.* 10 | 11 | ### **NOTICE:** There has been a major update to plugins, any created plugins will have to be updated to use the new, and more convenient data access method: 12 | - Remove `name` from any `ob.Plugin` 13 | - Update `node['data']` access to be `node.label_defined_in_node` 14 | - Please see the introduction to the plugin system below 15 | 16 | 17 | The osintbuddy plugin system at its core is very simple. An `OBRegistry` class holds all registered `OBPlugin` classes within the application. This registry is loaded into the [osintbuddy application](https://github.com/jerlendds/osintbuddy/) where it is then used to load the available entities for the user when they access a project graph, load transforms when a user opens the context menu of a node, and perform transformations which expect a `Plugin.blueprint()` to be returned. The returned data of a transform decorated method will be automatically mapped to a [JanusGraph](https://janusgraph.org/) database through [gremlin](https://tinkerpop.apache.org/) according to the labels *(as snakecase)* you previously set in the classes `node` for whatever `Plugin.blueprint()` 18 | you return. 19 | 20 | To make this a bit more clear please see the below example... 21 | 22 | ```py 23 | from pydantic import BaseModel 24 | import osintbuddy import transform, Plugin 25 | from osintbuddy.elements import TextInput, DropdownInput, Title, CopyText 26 | from osintbuddy.errors import OBPluginError 27 | 28 | 29 | class CSESearchResults(Plugin): 30 | label = "CSE Result" 31 | name = "CSE result" 32 | show_label = False # do not show this on the entities dialog 33 | # the user sees on the left of the project graph screen 34 | color = "#058F63" 35 | node = [ 36 | Title(label="Result"), 37 | CopyText(label="URL"), 38 | CopyText(label="Cache URL"), 39 | ] 40 | 41 | 42 | class CSESearchPlugin(Plugin): 43 | label = "CSE Search" 44 | name = "CSE search" 45 | color = "#2C7237" 46 | node = [ 47 | [ 48 | TextInput(label="Query", icon="search"), 49 | TextInput(label="Pages", icon="123", default="1"), 50 | ], 51 | DropdownInput(label="CSE Categories", options=cse_link_options) 52 | ] 53 | 54 | @transform(label="To cse results", icon="search") 55 | async def transform_to_cse_results( 56 | self, 57 | node: BaseModel, # dynamically generated pydantic model 58 | # that is mapped from the above labels contained within `node` 59 | use # a pydantic model allowing you to access a selenium instance 60 | # (and eventually a gremlin graph and settings api) 61 | ): 62 | results = [] 63 | 64 | if not node.query: 65 | raise OBPluginError(( 66 | 'You can send error messages to the user here' 67 | 'if they forget to submit data or if some other error occurs' 68 | )) 69 | 70 | # notice how you can access data returned from the context menu 71 | # of this node; using the label name in snake case 72 | print(node.cse_categories, node.query, node.pages) 73 | 74 | ... # (removed code for clarity) 75 | 76 | if resp: 77 | for result in resp["results"]: 78 | url = result.get("breadcrumbUrl", {}) 79 | # some elements you can store more than just a string, 80 | # (these elements storing dicts are mapped 81 | # to janusgraph as properties with the names 82 | # result_title, result_subtitle, and result_text) 83 | blueprint = CSESearchResults.blueprint( 84 | result={ 85 | "title": result.get("titleNoFormatting"), 86 | "subtitle": url.get("host") + url.get("crumbs"), 87 | "text": result.get("contentNoFormatting"), 88 | }, 89 | url=result.get("unescapedUrl"), 90 | cache_url=result.get("cacheUrl"), 91 | ) 92 | results.append(blueprint) 93 | # here we return a list of blueprints (blueprints are dicts) 94 | # but you can also return a single blueprint without a list 95 | return results 96 | 97 | ``` 98 | 99 | 100 | -------------------------------------------------------------------------------- /src/osintbuddy/elements/inputs.py: -------------------------------------------------------------------------------- 1 | from typing import List, Any 2 | from osintbuddy.elements.base import BaseInput 3 | 4 | 5 | class UploadFileInput(BaseInput): 6 | """ 7 | !!WORK_IN_PROGRESS!! The UploadFileInput class represents an upload file input node used 8 | in the OsintBuddy plugin system. 9 | value : Any 10 | The initial value for the input node. 11 | icon : str 12 | The icon to be displayed with the node. 13 | supported_files : List[str] 14 | A list of supported file extensions for the node. 15 | 16 | Usage Example: 17 | class Plugin(OBPlugin): 18 | node = [UploadFileInput(supported_files=['.pdf', '.docx'])] 19 | """ 20 | element_type: str = "upload" 21 | 22 | def __init__(self, value="", supported_files=[], icon="IconFileUpload", **kwargs): 23 | super().__init__(**kwargs) 24 | self.element = { 25 | "value": value, 26 | "icon": icon, 27 | "supported_files": supported_files, 28 | } 29 | 30 | def to_dict(self): 31 | return self._base_entity_element(**self.element) 32 | 33 | 34 | class TextInput(BaseInput): 35 | """The TextInput class represents a text input node used 36 | in the OsintBuddy plugin system. 37 | value : str 38 | The value stored in the element. 39 | icon : str 40 | The icon to be displayed with the input element. 41 | default : str 42 | The default value for the input element. 43 | 44 | Usage Example: 45 | class Plugin(OBPlugin): 46 | node = [TextInput(label='Email search', placeholder='Enter email')] 47 | """ 48 | element_type: str = "text" 49 | 50 | def __init__(self, value="", default="", icon="IconAlphabetLatin", **kwargs): 51 | super().__init__(**kwargs) 52 | self.element = { 53 | "value": value, 54 | "icon": icon 55 | } 56 | def to_dict(self): 57 | return self._base_entity_element(**self.element) 58 | 59 | 60 | class TextAreaInput(BaseInput): 61 | """The TextInput class represents a text input node used 62 | in the OsintBuddy plugin system. 63 | value : str 64 | The value stored in the element. 65 | icon : str 66 | The icon to be displayed with the input element. 67 | default : str 68 | The default value for the input element. 69 | 70 | Usage Example: 71 | class Plugin(OBPlugin): 72 | node = [TextInput(label='Email search', placeholder='Enter email')] 73 | """ 74 | element_type: str = "textarea" 75 | 76 | def __init__(self, value="", default="", icon="IconAlphabetLatin", **kwargs): 77 | super().__init__(**kwargs) 78 | self.element = { 79 | "value": value, 80 | } 81 | def to_dict(self): 82 | return self._base_entity_element(**self.element) 83 | 84 | 85 | class DropdownInput(BaseInput): 86 | """ 87 | The DropdownInput class represents a dropdown menu node used 88 | in the OsintBuddy plugin system. 89 | options : List[any] 90 | A list of options for the dropdown menu. 91 | value : str 92 | The initially selected option in the dropdown menu. 93 | 94 | Usage Example: 95 | class Plugin(OBPlugin): 96 | node = [ 97 | DropdownInput( 98 | options=[{'label': 'Option 1', 'tooltip': 'Hello on hover!'}], 99 | value='Option 1' 100 | ) 101 | ] 102 | """ 103 | element_type: str = "dropdown" 104 | 105 | def __init__(self, options=[], value={'label': '', 'tooltip': '', 'value': ''}, **kwargs): 106 | super().__init__(**kwargs) 107 | self.element = { 108 | "options": options, 109 | "value": value 110 | } 111 | 112 | def to_dict(self): 113 | return self._base_entity_element(**self.element) 114 | 115 | 116 | 117 | class NumberInput(BaseInput): 118 | """ 119 | !!WORK_IN_PROGRESS!! The NumberInput class represents a whole number input node used 120 | in the OsintBuddy plugin system. 121 | 122 | value : int 123 | The integer value stored in the node. 124 | 125 | Usage Example: 126 | class Plugin(OBPlugin): 127 | node = [NumberInput(value=10, placeholder='Enter a whole number')] 128 | """ 129 | element_type: str = "number" 130 | 131 | def __init__(self, value=1, icon="123", **kwargs): 132 | super().__init__(**kwargs) 133 | self.element = { 134 | "value": value, 135 | "icon": icon 136 | } 137 | 138 | def to_dict(self): 139 | return self._base_entity_element(**self.element) 140 | 141 | 142 | class DecimalInput(BaseInput): 143 | """ 144 | !!WORK_IN_PROGRESS!! The DecimalInput class represents a decimal number input node used 145 | in the OsintBuddy plugin system. 146 | value : float 147 | The float value stored in the node. 148 | 149 | Usage Example: 150 | class Plugin(OBPlugin): 151 | node = [DecimalInput(value=3.14, placeholder='Enter a decimal number')] 152 | """ 153 | element_type: str = "decimal" 154 | 155 | def __init__(self, value=3.14, icon="123", **kwargs): 156 | super().__init__(**kwargs) 157 | self.element = { 158 | "value": value, 159 | "icon": icon 160 | } 161 | 162 | def to_dict(self): 163 | return self._base_entity_element(**self.element) 164 | 165 | -------------------------------------------------------------------------------- /src/osintbuddy/ascii.py: -------------------------------------------------------------------------------- 1 | 2 | OB_LOGO_SM = """ 3 | ██████████████████████████████████████ 4 | ██████████████████████████████████████ 5 | ██████████████████████████████████████ 6 | █████ ██████ 7 | █████ ██████ 8 | █████ █ ███ ██████ 9 | █████ ███ ████ ██████ 10 | █████ ██ ███ ████ ██████ 11 | █████ ██ ███ ████ ██ ██████ 12 | █████ ██ ███ ████ ███ ██████ 13 | █████ ██ ███ ████ ███ ██████ 14 | █████ ██ ███ ████ ███ ██████ 15 | █████ ███ ███ ███ ██ ██████ 16 | █████ ███ ███ ████ ██ ██████ 17 | █████ ███ ███ ███ ██ ██████ 18 | █████ █ ███ ███ ██ ██████ 19 | █████ ███ ███ ██ ██████ 20 | ██████ ███ ██ █████ 21 | ███████ ██ ██████ 22 | ███████ ███████ 23 | ████████ ████████ 24 | ██████████ ██████████ 25 | ██████████████████████████ 26 | █████████████████████ 27 | ████████████ 28 | """ 29 | 30 | OB_LOGO_LG = """ 31 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 32 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 33 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 34 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 35 | ▓▓▓▓▓▓ ▓▓▓▓▓▓ 36 | ▓▓▓▓▓▓ ▓▓▓▓▓▓ 37 | ▓▓▓▓▓▓ ▓ ▓▓▓ ▓▓▓▓▓▓ 38 | ▓▓▓▓▓▓ ▓▓▓ ▓▓▓ ▓▓▓▓▓▓ 39 | ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓ ▓▓▓▓▓▓ 40 | ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ 41 | ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓▓ ▓▓▓ ▓▓▓▓▓▓ 42 | ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ 43 | ▓▓▓▓▓▓ ▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ 44 | ▓▓▓▓▓▓ ▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ 45 | ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓ ▓▓▓▓▓▓ 46 | ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓ ▓▓▓▓▓▓ 47 | ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ 48 | ▓▓▓▓▓▓ ▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ 49 | ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ 50 | ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ 51 | ▓▓▓▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓▓ 52 | ▓▓▓▓▓▓▓ ▓▓ ▓▓▓▓▓▓▓ 53 | ▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓ 54 | ▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓ 55 | ▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓ 56 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 57 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 58 | ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 59 | ▓▓▓▓▓▓ 60 | """ 61 | 62 | OB_LOGO_LIGHT = """ 63 | rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr 64 | rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr 65 | rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr 66 | rrrrr rrrrr 67 | rrrrr rrr rrrrr 68 | rrrrr rr rr rrr rrrrr 69 | rrrrr rr rrr rrr rrrrr 70 | rrrrr rr rrr rrr rrr rrrrr 71 | rrrrr rr rrr rrr rrr rrrrr 72 | rrrrr rr rrr rrr rrr rrrrr 73 | rrrrr rrr rrr rrr rr rrrrr 74 | rrrrr rrr rrr rrr rr rrrrr 75 | rrrrr rrr rrrr rr rr rrrrr 76 | rrrrr r rrrr rr rr rrrrr 77 | rrrrr rrr rrr rr rrrrr 78 | rrrrrr rrr rr rrrrrr 79 | rrrrrr rr rrrrrr 80 | rrrrrr rrrrrr 81 | rrrrrrrr rrrrrrr 82 | rrrrrrrrrrrrrrrrrrrrrrrrr 83 | rrrrrrrrrrrrrrrrrrrrr 84 | rrrrrrrrrrrrrr 85 | """ 86 | 87 | OB_LOGO_XL = """ 88 | █████████████████████████████████████████████████████████ 89 | █████████████████████████████████████████████████████████ 90 | ██████████████████████████████████████████████████████████ 91 | ██████████████████████████████████████████████████████████ 92 | ██████████████████████████████████████████████████████████ 93 | █████████ ████████ 94 | █████████ ████████ 95 | █████████ ██ ████████ 96 | █████████ ████ ████████ 97 | █████████ ████ █████ ████████ 98 | █████████ █████ █████ ████████ 99 | █████████ ███ █████ █████ ████████ 100 | █████████ ████ ████ █████ █ ████████ 101 | █████████ ████ ████ ██████ ████ ████████ 102 | █████████ ███ ████ ██████ ████ ████████ 103 | █████████ ███ ████ ██████ █████ ████████ 104 | █████████ ███ █████ ██████ █████ ████████ 105 | █████████ ███ █████ ██████ ████ ████████ 106 | █████████ █████ ██████ ████ ██ ████████ 107 | █████████ █████ █████ ████ ████ ████████ 108 | █████████ █████ █████ ████ ████ ████████ 109 | ████████ █████ █████ █████ ████ ████████ 110 | ████████ █████ █████ █████ ████ ████████ 111 | ████████ ████ █████ █████ ████ ████████ 112 | ████████ █████ █████ ████ ████████ 113 | █████████ █████ █████ ██ █████████ 114 | █████████ █████ █████ █████████ 115 | █████████ █████ ████ █████████ 116 | ██████████ ████ █████████ 117 | ██████████ █████████ 118 | ██████████ ██████████ 119 | ███████████ ███████████ 120 | █████████████ ████████████ 121 | ███████████████ ████████████████ 122 | ██████████████████████████████████████████ 123 | ██████████████████████████████████████ 124 | ████████████████████████████████ 125 | ████████████████████████ 126 | ████████████ 127 | """ -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit"] 3 | build-backend = "flit.buildapi" 4 | 5 | [project] 6 | name = "osintbuddy" 7 | authors = [ 8 | {name = "jerlendds", email = "support@forum.osintbuddy.com"}, 9 | ] 10 | description = "OSINTBuddy - mine, merge, and map data for novel insights" 11 | readme = "README.md" 12 | classifiers = [ 13 | "Development Status :: 3 - Alpha", 14 | "Intended Audience :: Developers", 15 | "License :: OSI Approved :: GNU Affero General Public License v3", 16 | "Programming Language :: Python :: 3.11" 17 | ] 18 | requires-python = ">=3.11" 19 | dynamic = ["version"] 20 | dependencies = [ 21 | "selenium>=4.9.0", 22 | "sqlalchemy-json==0.7.0", 23 | "SQLAlchemy-Utils==0.41.1", 24 | "playwright>=1.39.0", 25 | "httpx>=0.25.0", 26 | "beautifulsoup4==4.12.2", 27 | "pyfiglet==0.8.post1", 28 | "termcolor==2.3.0", 29 | "fastapi==0.103.2", 30 | "uvicorn==0.22.0", 31 | "uvloop==0.17.0", 32 | "pydantic==2.4.2", 33 | "pydantic-settings==2.0.3", 34 | "yq==3.2.3", 35 | "jedi-language-server==0.41.1", 36 | "websockets==11.0.3" 37 | ] 38 | [project.optional-dependencies] 39 | test = [ 40 | "astroid==2.15.4", 41 | "colorama==0.4.6", 42 | "dill==0.3.6", 43 | "eradicate==2.2.0", 44 | "exceptiongroup==1.1.1", 45 | "iniconfig==2.0.0", 46 | "isort==5.12.0", 47 | "lazy-object-proxy==1.9.0", 48 | "mando==0.7.1", 49 | "mccabe==0.7.0", 50 | "mypy==1.3.0", 51 | "mypy-extensions==1.0.0", 52 | "packaging==23.1", 53 | "platformdirs==3.5.1", 54 | "pluggy==1.0.0", 55 | "pycodestyle==2.10.0", 56 | "pydocstyle==6.3.0", 57 | "pyflakes==3.0.1", 58 | "pylama==8.4.1", 59 | "pylint==2.17.4", 60 | "pytest==7.3.1", 61 | "radon==6.0.1", 62 | "six==1.16.0", 63 | "snowballstemmer==2.2.0", 64 | "toml==0.10.2", 65 | "tomli==2.0.1", 66 | "tomlkit==0.11.8", 67 | "typing-extensions==4.5.0", 68 | "vulture==2.7", 69 | "wrapt==1.15.0", 70 | ] 71 | 72 | [project.urls] 73 | Documentation = "https://docs.osintbuddy.com/" 74 | Source = "https://github.com/jerlendds/osintbuddy-plugins" 75 | Tracker = "https://github.com/jerlendds/osintbuddy/issues" 76 | 77 | [tool.flit.module] 78 | name = "osintbuddy" 79 | 80 | [tool.bandit] 81 | exclude_dirs = ["build","dist","tests","scripts"] 82 | number = 4 83 | recursive = true 84 | targets = "src" 85 | 86 | [tool.black] 87 | line-length = 120 88 | fast = true 89 | 90 | [tool.coverage.run] 91 | branch = true 92 | 93 | [tool.coverage.report] 94 | fail_under = 100 95 | 96 | [tool.flake8] 97 | max-line-length = 120 98 | select = "F,E,W,B,B901,B902,B903" 99 | exclude = [ 100 | ".eggs", 101 | ".git", 102 | ".tox", 103 | "nssm", 104 | "obj", 105 | "out", 106 | "packages", 107 | "pywin32", 108 | "tests", 109 | "swagger_client" 110 | ] 111 | ignore = [ 112 | "E722", 113 | "B001", 114 | "W503", 115 | "E203" 116 | ] 117 | 118 | [tool.pyright] 119 | include = ["src"] 120 | exclude = [ 121 | "**/node_modules", 122 | "**/__pycache__", 123 | ] 124 | venv = "env37" 125 | 126 | reportMissingImports = true 127 | reportMissingTypeStubs = false 128 | 129 | pythonVersion = "3.7" 130 | pythonPlatform = "Linux" 131 | 132 | executionEnvironments = [ 133 | { root = "src" } 134 | ] 135 | 136 | [tool.pytest.ini_options] 137 | addopts = "" 138 | pythonpath = [ 139 | "src" 140 | ] 141 | testpaths = "tests" 142 | junit_family = "xunit2" 143 | markers = [ 144 | "integration: marks as integration test", 145 | "notebooks: marks as notebook test", 146 | "gpu: marks as gpu test", 147 | "spark: marks tests which need Spark", 148 | "slow: marks tests as slow", 149 | "unit: fast offline tests", 150 | ] 151 | 152 | [tool.tox] 153 | legacy_tox_ini = """ 154 | [tox] 155 | envlist = py, integration, spark, all 156 | 157 | [testenv] 158 | commands = 159 | pytest -m "not integration and not spark" {posargs} 160 | 161 | [testenv:integration] 162 | commands = 163 | pytest -m "integration" {posargs} 164 | 165 | [testenv:spark] 166 | extras = spark 167 | setenv = 168 | PYSPARK_DRIVER_PYTHON = {envpython} 169 | PYSPARK_PYTHON = {envpython} 170 | commands = 171 | pytest -m "spark" {posargs} 172 | 173 | [testenv:all] 174 | extras = all 175 | setenv = 176 | PYSPARK_DRIVER_PYTHON = {envpython} 177 | PYSPARK_PYTHON = {envpython} 178 | commands = 179 | pytest {posargs} 180 | """ 181 | 182 | [tool.pylint] 183 | extension-pkg-whitelist= [ 184 | "numpy", 185 | "torch", 186 | "cv2", 187 | "pyodbc", 188 | "pydantic", 189 | "ciso8601", 190 | "netcdf4", 191 | "scipy" 192 | ] 193 | ignore="CVS" 194 | ignore-patterns="test.*?py,conftest.py" 195 | init-hook='import sys; sys.setrecursionlimit(8 * sys.getrecursionlimit())' 196 | jobs=0 197 | limit-inference-results=100 198 | persistent="yes" 199 | suggestion-mode="yes" 200 | unsafe-load-any-extension="no" 201 | 202 | [tool.pylint.'MESSAGES CONTROL'] 203 | enable="c-extension-no-member" 204 | 205 | [tool.pylint.'REPORTS'] 206 | evaluation="10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)" 207 | output-format="text" 208 | reports="no" 209 | score="yes" 210 | 211 | [tool.pylint.'REFACTORING'] 212 | max-nested-blocks=5 213 | never-returning-functions="sys.exit" 214 | 215 | [tool.pylint.'BASIC'] 216 | argument-naming-style="snake_case" 217 | attr-naming-style="snake_case" 218 | bad-names= [ 219 | "foo", 220 | "bar" 221 | ] 222 | class-attribute-naming-style="any" 223 | class-naming-style="PascalCase" 224 | const-naming-style="UPPER_CASE" 225 | docstring-min-length=-1 226 | function-naming-style="snake_case" 227 | good-names= [ 228 | "i", 229 | "j", 230 | "k", 231 | "ex", 232 | "Run", 233 | "_" 234 | ] 235 | include-naming-hint="yes" 236 | inlinevar-naming-style="any" 237 | method-naming-style="snake_case" 238 | module-naming-style="any" 239 | no-docstring-rgx="^_" 240 | property-classes="abc.abstractproperty" 241 | variable-naming-style="snake_case" 242 | 243 | [tool.pylint.'FORMAT'] 244 | ignore-long-lines="^\\s*(# )?.*['\"]??" 245 | indent-after-paren=4 246 | indent-string=' ' 247 | max-line-length=120 248 | max-module-lines=1000 249 | single-line-class-stmt="no" 250 | single-line-if-stmt="no" 251 | 252 | [tool.pylint.'LOGGING'] 253 | logging-format-style="old" 254 | logging-modules="logging" 255 | 256 | [tool.pylint.'MISCELLANEOUS'] 257 | notes= [ 258 | "FIXME", 259 | "XXX", 260 | "TODO" 261 | ] 262 | 263 | [tool.pylint.'SIMILARITIES'] 264 | ignore-comments="yes" 265 | ignore-docstrings="yes" 266 | ignore-imports="yes" 267 | min-similarity-lines=7 268 | 269 | [tool.pylint.'SPELLING'] 270 | max-spelling-suggestions=4 271 | spelling-store-unknown-words="no" 272 | 273 | [tool.pylint.'STRING'] 274 | check-str-concat-over-line-jumps="no" 275 | 276 | [tool.pylint.'TYPECHECK'] 277 | contextmanager-decorators="contextlib.contextmanager" 278 | generated-members="numpy.*,np.*,pyspark.sql.functions,collect_list" 279 | ignore-mixin-members="yes" 280 | ignore-none="yes" 281 | ignore-on-opaque-inference="yes" 282 | ignored-classes="optparse.Values,thread._local,_thread._local,numpy,torch,swagger_client" 283 | ignored-modules="numpy,torch,swagger_client,netCDF4,scipy" 284 | missing-member-hint="yes" 285 | missing-member-hint-distance=1 286 | missing-member-max-choices=1 287 | 288 | [tool.pylint.'VARIABLES'] 289 | additional-builtins="dbutils" 290 | allow-global-unused-variables="yes" 291 | callbacks= [ 292 | "cb_", 293 | "_cb" 294 | ] 295 | dummy-variables-rgx="_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" 296 | ignored-argument-names="_.*|^ignored_|^unused_" 297 | init-import="no" 298 | redefining-builtins-modules="six.moves,past.builtins,future.builtins,builtins,io" 299 | 300 | [tool.pylint.'CLASSES'] 301 | defining-attr-methods= [ 302 | "__init__", 303 | "__new__", 304 | "setUp", 305 | "__post_init__" 306 | ] 307 | exclude-protected= [ 308 | "_asdict", 309 | "_fields", 310 | "_replace", 311 | "_source", 312 | "_make" 313 | ] 314 | valid-classmethod-first-arg="cls" 315 | valid-metaclass-classmethod-first-arg="cls" 316 | 317 | [tool.pylint.'DESIGN'] 318 | max-args=5 319 | max-attributes=7 320 | max-bool-expr=5 321 | max-branches=12 322 | max-locals=15 323 | max-parents=7 324 | max-public-methods=20 325 | max-returns=6 326 | max-statements=50 327 | min-public-methods=2 328 | 329 | [tool.pylint.'IMPORTS'] 330 | allow-wildcard-with-all="no" 331 | analyse-fallback-blocks="no" 332 | deprecated-modules="optparse,tkinter.tix" 333 | 334 | [tool.pylint.'EXCEPTIONS'] 335 | overgeneral-exceptions= [ 336 | "BaseException", 337 | "Exception" 338 | ] 339 | [tool.hatch.build.targets.sdist.force-include] 340 | "bin/ob.py" = "osintbuddy/ob.py" 341 | 342 | [tool.hatch.build.targets.wheel.force-include] 343 | "bin/ob.py" = "osintbuddy/ob.py" 344 | 345 | [project.scripts] 346 | ob = "osintbuddy.ob:main" -------------------------------------------------------------------------------- /src/osintbuddy/plugins.py: -------------------------------------------------------------------------------- 1 | import os, imp, importlib, inspect, sys 2 | from typing import List, Any, Callable 3 | from collections import defaultdict 4 | from pydantic import BaseModel, ConfigDict 5 | from osintbuddy.elements.base import BaseElement 6 | from osintbuddy.errors import OBPluginError 7 | from osintbuddy.utils import to_snake_case 8 | 9 | 10 | OBNodeConfig = ConfigDict(extra="allow", frozen=False, populate_by_name=True, arbitrary_types_allowed=True) 11 | 12 | class OBNode(BaseModel): 13 | model_config = OBNodeConfig 14 | 15 | 16 | def plugin_results_middleman(f): 17 | def return_result(r): 18 | return r 19 | def yield_result(r): 20 | for i in r: 21 | yield i 22 | def decorator(*a, **kwa): 23 | if inspect.isgeneratorfunction(f): 24 | return yield_result(f(*a, **kwa)) 25 | else: 26 | return return_result(f(*a, **kwa)) 27 | return decorator 28 | 29 | 30 | class OBAuthorUse(BaseModel): 31 | get_driver: Callable[[], None] 32 | 33 | 34 | class OBRegistry(type): 35 | plugins = [] 36 | labels = [] 37 | ui_labels = [] 38 | 39 | def __init__(cls, name, bases, attrs): 40 | """ 41 | Initializes the OBRegistry metaclass by adding the plugin class 42 | and its label if it is a valid plugin. 43 | """ 44 | if name != 'OBPlugin' and name != 'Plugin' and issubclass(cls, OBPlugin): 45 | label = cls.label.strip() 46 | if cls.show_label is True: 47 | if isinstance(cls.author, list): 48 | cls.author = ', '.join(cls.author) 49 | OBRegistry.ui_labels.append({ 50 | 'label': label, 51 | 'description': cls.description, 52 | 'author': cls.author 53 | }) 54 | OBRegistry.labels.append(label) 55 | OBRegistry.plugins.append(cls) 56 | 57 | @classmethod 58 | async def get_plugin(cls, plugin_label: str): 59 | """ 60 | Returns the corresponding plugin class for a given plugin_label or 61 | 'None' if not found. 62 | 63 | :param plugin_label: The label of the plugin to be returned. 64 | :return: The plugin class or None if not found. 65 | """ 66 | for idx, label in enumerate(cls.labels): 67 | if label == plugin_label or to_snake_case(label) == to_snake_case(plugin_label): 68 | return cls.plugins[idx] 69 | return None 70 | 71 | @classmethod 72 | def get_plug(cls, plugin_label: str): 73 | """ 74 | Returns the corresponding plugin class for a given plugin_label or 75 | 'None' if not found. 76 | 77 | :param plugin_label: The label of the plugin to be returned. 78 | :return: The plugin class or None if not found. 79 | """ 80 | for idx, label in enumerate(cls.labels): 81 | if to_snake_case(label) == to_snake_case(plugin_label): 82 | return cls.plugins[idx] 83 | return None 84 | 85 | def __getitem__(self, i: str): 86 | return self.get_plug[i] 87 | 88 | # https://stackoverflow.com/a/7548190 89 | def load_plugin( 90 | mod_name: str, 91 | plugin_code: str, 92 | ): 93 | """ 94 | Load plugins from a string of code 95 | 96 | :param module_name: The desired module name of the plugin. 97 | :param plugin_code: The code of the plugin. 98 | :return: 99 | """ 100 | new_mod = imp.new_module(mod_name) 101 | exec(plugin_code, new_mod.__dict__) 102 | return OBRegistry.plugins 103 | 104 | 105 | def load_plugins( 106 | entities: list 107 | ): 108 | """ 109 | Loads plugins from the osintbuddy db 110 | 111 | :param entities: list of entities from the db 112 | :return: 113 | """ 114 | for entity in entities: 115 | mod_name = to_snake_case(entity.label) 116 | new_mod = imp.new_module(mod_name) 117 | exec(entity.source, new_mod.__dict__) 118 | return OBRegistry.plugins 119 | 120 | 121 | def discover_plugins( 122 | dir_path: str = '/plugins.osintbuddy.com/src/osintbuddy/core/', 123 | ): 124 | """ 125 | Scans the specified 'dir_path' for '.py' files, imports them as plugins, 126 | and populates the OBRegistry with classes. 127 | 128 | :param dir_path: The directory path where the plugins are located. 129 | :return: List of plugin classes 130 | """ 131 | for r, _, files in os.walk(dir_path): 132 | for filename in files: 133 | modname, ext = os.path.splitext(filename) 134 | if ext == '.py': 135 | try: 136 | modpath = r.replace("/app/", "") 137 | if 'osintbuddy/core' in dir_path: 138 | modpath = r.replace("/plugins.osintbuddy.com/src/", "") 139 | modpath = modpath.replace("/", ".") 140 | importlib.import_module(f'{modpath}{modname}') 141 | except ImportError as e: 142 | print(f"Error importing plugin '{modpath}{modname}': {e}") 143 | 144 | return OBRegistry.plugins 145 | 146 | 147 | def transform(label, icon='list', edge_label='transformed_to'): 148 | """ 149 | A decorator add transforms to an osintbuddy plugin. 150 | 151 | Usage: 152 | @transform(label=, icon=) 153 | def transform_to_ip(self, node, **kwargs): 154 | # Method implementation 155 | 156 | :param label: str, A string representing the label for the transform 157 | method, which can be utilized for displaying in the context menu. 158 | :param icon: str, Optional icon name, representing the icon associated 159 | displayed by the transform label. Default is "list". 160 | :return: A decorator for the plugin transform method. 161 | """ 162 | def decorator_transform(func, edge_label=edge_label): 163 | async def wrapper(self, node, **kwargs): 164 | return await func(self=self, node=node, **kwargs) 165 | wrapper.label = label 166 | wrapper.icon = icon 167 | wrapper.edge_label = edge_label 168 | return wrapper 169 | return decorator_transform 170 | 171 | 172 | class OBPlugin(object, metaclass=OBRegistry): 173 | """ 174 | OBPlugin is the base class for all plugin classes in this application. 175 | It provides the required structure and methods for a plugin. 176 | """ 177 | entity: List[BaseElement] 178 | color: str = '#145070' 179 | label: str = '' 180 | icon: str = 'atom-2' 181 | show_label = True 182 | 183 | author = 'Unknown' 184 | description = 'No description.' 185 | 186 | def __init__(self): 187 | transforms = self.__class__.__dict__.values() 188 | self.transforms = { 189 | to_snake_case(func.label): func for func in transforms if hasattr(func, 'label') 190 | } 191 | self.transform_labels = [ 192 | { 193 | 'label': func.label, 194 | 'icon': func.icon, 195 | } for func in transforms 196 | if hasattr(func, 'icon') and hasattr(func, 'label') 197 | ] 198 | 199 | def __call__(self): 200 | return self.blueprint() 201 | 202 | @staticmethod 203 | def _map_graph_data_labels(element, **kwargs): 204 | label = to_snake_case(element['label']) 205 | for element_key in kwargs.keys(): 206 | if element_key == label: 207 | if isinstance(kwargs[label], str): 208 | element['value'] = kwargs[label] 209 | elif isinstance(kwargs[label], dict): 210 | for t in kwargs[label]: 211 | element[t] = kwargs[label][t] 212 | return element 213 | 214 | @classmethod 215 | def blueprint(cls, **kwargs): 216 | """ 217 | Generate and return a dictionary representing the plugins node. 218 | Includes label, name, color, icon, and a list of all elements 219 | for the node/plugin. 220 | """ 221 | entity_ui_node = defaultdict(None) 222 | entity_ui_node['label'] = cls.label 223 | entity_ui_node['color'] = cls.color if cls.color else '#145070' 224 | entity_ui_node['icon'] = cls.icon 225 | entity_ui_node['elements'] = [] 226 | if cls.entity: 227 | for element in cls.entity: 228 | # if an entity element is a nested list, 229 | # elements will be positioned next to each other horizontally 230 | if isinstance(element, list): 231 | entity_ui_node['elements'].append([ 232 | cls._map_graph_data_labels(elm.to_dict(), **kwargs) 233 | for elm in element 234 | ]) 235 | # otherwise position the entity elements vertically on the actual UI entity node 236 | else: 237 | element_row = cls._map_graph_data_labels(element.to_dict(), **kwargs) 238 | entity_ui_node['elements'].append(element_row) 239 | return entity_ui_node 240 | if cls.node: 241 | print("WARNING! Using node in plugins is being deprecated! Please switch to entity = [TextInput(...), ...] ") 242 | for element in cls.node: 243 | # if an entity element is a nested list, 244 | # elements will be positioned next to each other horizontally 245 | if isinstance(element, list): 246 | row_elms = [] 247 | for elm in element: 248 | row_elms.append(cls._map_graph_data_labels(elm.to_dict(), **kwargs)) 249 | entity_ui_node['elements'].append(row_elms) 250 | # otherwise position the entity elements vertically on the actual UI entity node 251 | else: 252 | element_row = cls._map_graph_data_labels(element.to_dict(), **kwargs) 253 | entity_ui_node['elements'].append(element_row) 254 | return entity_ui_node 255 | 256 | 257 | async def get_transform(self, transform_type: str, entity, use: OBAuthorUse) -> Any: 258 | """ Return output from a function accepting node data. 259 | The function will be called with a single argument, the node data 260 | from when a node context menu action is taken - and should return 261 | a list of Nodes. 262 | None if the plugin doesn't provide a transform 263 | for the transform_type. 264 | """ 265 | transform_type = to_snake_case(transform_type) 266 | if self.transforms and self.transforms[transform_type]: 267 | try: 268 | transform = await self.transforms[transform_type]( 269 | self=self, 270 | node=self._map_to_transform_data(entity), 271 | use=use 272 | ) 273 | edge_label = self.transforms[transform_type].edge_label 274 | if not isinstance(transform, list): 275 | transform['edge_label'] = edge_label 276 | return [transform] 277 | [ 278 | n.__setitem__('edge_label', edge_label) 279 | for n in transform 280 | ] 281 | return transform 282 | except (Exception, OBPluginError) as e: 283 | raise e 284 | # exc_type, exc_obj, exc_tb = sys.exc_info() 285 | # fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] 286 | # raise OBPluginError(f"Unhandled plugin error! {exc_type}\nPlease see {fname} on line no. {exc_tb.tb_lineno}\n{e}") 287 | return None 288 | 289 | @staticmethod 290 | def _map_element(transform_map: dict, element: dict): 291 | label = to_snake_case(element.pop('label', None)) 292 | transform_map[label] = {} 293 | element_type = element.pop('type', None) 294 | element.pop('icon', None) 295 | element.pop('placeholder', None) 296 | element.pop('style', None) 297 | element.pop('options', None) 298 | for k, v in element.items(): 299 | if (isinstance(v, str) and len(element.values()) == 1) or element_type == 'dropdown': 300 | transform_map[label] = v 301 | else: 302 | transform_map[label][k] = v 303 | 304 | @classmethod 305 | def _map_to_transform_data(cls, node: dict) -> OBNode: 306 | transform_map: dict = {} 307 | data: dict = node.get('data', {}) 308 | elements: list[dict] = data.get('elements', []) 309 | for element in elements: 310 | if isinstance(element, list): 311 | [cls._map_element(transform_map, elm) for elm in element] 312 | else: 313 | cls._map_element(transform_map, element) 314 | return OBNode(**transform_map) 315 | -------------------------------------------------------------------------------- /docs/pylint.md: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist=numpy,torch,cv2,pyodbc,pydantic,ciso8601,netcdf4,scipy 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns=test.*?py,conftest.py 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | init-hook='import sys; sys.setrecursionlimit(8 * sys.getrecursionlimit())' 19 | 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 21 | # number of processors available to use. 22 | jobs=0 23 | 24 | # Control the amount of potential inferred values when inferring a single 25 | # object. This can help the performance when dealing with large functions or 26 | # complex, nested conditions. 27 | limit-inference-results=100 28 | 29 | # List of plugins (as comma separated values of python module names) to load, 30 | # usually to register additional checkers. 31 | 32 | # Pickle collected data for later comparisons. 33 | persistent=yes 34 | 35 | # Specify a configuration file. 36 | #rcfile= 37 | 38 | # When enabled, pylint would attempt to guess common misconfiguration and emit 39 | # user-friendly hints instead of false-positive error messages. 40 | suggestion-mode=yes 41 | 42 | # Allow loading of arbitrary C extensions. Extensions are imported into the 43 | # active Python interpreter and may run arbitrary code. 44 | unsafe-load-any-extension=no 45 | 46 | 47 | [MESSAGES CONTROL] 48 | 49 | # Only show warnings with the listed confidence levels. Leave empty to show 50 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 51 | confidence= 52 | 53 | # Disable the message, report, category or checker with the given id(s). You 54 | # can either give multiple identifiers separated by comma (,) or put this 55 | # option multiple times (only on the command line, not in the configuration 56 | # file where it should appear only once). You can also use "--disable=all" to 57 | # disable everything first and then reenable specific checks. For example, if 58 | # you want to run only the similarities checker, you can use "--disable=all 59 | # --enable=similarities". If you want to run only the classes checker, but have 60 | # no Warning level messages displayed, use "--disable=all --enable=classes 61 | # --disable=W". 62 | #disable= 63 | # Enable the message, report, category or checker with the given id(s). You can 64 | # either give multiple identifier separated by comma (,) or put this option 65 | # multiple time (only on the command line, not in the configuration file where 66 | # it should appear only once). See also the "--disable" option for examples. 67 | enable=c-extension-no-member 68 | 69 | 70 | [REPORTS] 71 | 72 | # Python expression which should return a score less than or equal to 10. You 73 | # have access to the variables 'error', 'warning', 'refactor', and 'convention' 74 | # which contain the number of messages in each category, as well as 'statement' 75 | # which is the total number of statements analyzed. This score is used by the 76 | # global evaluation report (RP0004). 77 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 78 | 79 | # Template used to display messages. This is a python new-style format string 80 | # used to format the message information. See doc for all details. 81 | #msg-template= 82 | 83 | # Set the output format. Available formats are text, parseable, colorized, json 84 | # and msvs (visual studio). You can also give a reporter class, e.g. 85 | # mypackage.mymodule.MyReporterClass. 86 | output-format=text 87 | 88 | # Tells whether to display a full report or only the messages. 89 | reports=no 90 | 91 | # Activate the evaluation score. 92 | score=yes 93 | 94 | 95 | [REFACTORING] 96 | 97 | # Maximum number of nested blocks for function / method body 98 | max-nested-blocks=5 99 | 100 | # Complete name of functions that never returns. When checking for 101 | # inconsistent-return-statements if a never returning function is called then 102 | # it will be considered as an explicit return statement and no message will be 103 | # printed. 104 | never-returning-functions=sys.exit 105 | 106 | 107 | [BASIC] 108 | 109 | # Naming style matching correct argument names. 110 | argument-naming-style=snake_case 111 | 112 | # Regular expression matching correct argument names. Overrides argument- 113 | # naming-style. 114 | #argument-rgx= 115 | 116 | # Naming style matching correct attribute names. 117 | attr-naming-style=snake_case 118 | 119 | # Regular expression matching correct attribute names. Overrides attr-naming- 120 | # style. 121 | #attr-rgx= 122 | 123 | # Bad variable names which should always be refused, separated by a comma. 124 | bad-names=foo, 125 | bar, 126 | baz, 127 | toto, 128 | tutu, 129 | tata 130 | 131 | # Naming style matching correct class attribute names. 132 | class-attribute-naming-style=any 133 | 134 | # Regular expression matching correct class attribute names. Overrides class- 135 | # attribute-naming-style. 136 | #class-attribute-rgx= 137 | 138 | # Naming style matching correct class names. 139 | class-naming-style=PascalCase 140 | 141 | # Regular expression matching correct class names. Overrides class-naming- 142 | # style. 143 | #class-rgx= 144 | 145 | # Naming style matching correct constant names. 146 | const-naming-style=UPPER_CASE 147 | 148 | # Regular expression matching correct constant names. Overrides const-naming- 149 | # style. 150 | #const-rgx= 151 | 152 | # Minimum line length for functions/classes that require docstrings, shorter 153 | # ones are exempt. 154 | docstring-min-length=-1 155 | 156 | # Naming style matching correct function names. 157 | function-naming-style=snake_case 158 | 159 | # Regular expression matching correct function names. Overrides function- 160 | # naming-style. 161 | #function-rgx= 162 | 163 | # Good variable names which should always be accepted, separated by a comma. 164 | good-names=i, 165 | j, 166 | k, 167 | ex, 168 | Run, 169 | _, 170 | df, 171 | n, 172 | N, 173 | t, 174 | T, 175 | ax 176 | 177 | # Include a hint for the correct naming format with invalid-name. 178 | include-naming-hint=no 179 | 180 | # Naming style matching correct inline iteration names. 181 | inlinevar-naming-style=any 182 | 183 | # Regular expression matching correct inline iteration names. Overrides 184 | # inlinevar-naming-style. 185 | #inlinevar-rgx= 186 | 187 | # Naming style matching correct method names. 188 | method-naming-style=snake_case 189 | 190 | # Regular expression matching correct method names. Overrides method-naming- 191 | # style. 192 | #method-rgx= 193 | 194 | # Naming style matching correct module names. 195 | module-naming-style=any 196 | 197 | # Regular expression matching correct module names. Overrides module-naming- 198 | # style. 199 | #module-rgx= 200 | 201 | # Colon-delimited sets of names that determine each other's naming style when 202 | # the name regexes allow several styles. 203 | name-group= 204 | 205 | # Regular expression which should only match function or class names that do 206 | # not require a docstring. 207 | no-docstring-rgx=^_ 208 | 209 | # List of decorators that produce properties, such as abc.abstractproperty. Add 210 | # to this list to register other decorators that produce valid properties. 211 | # These decorators are taken in consideration only for invalid-name. 212 | property-classes=abc.abstractproperty 213 | 214 | # Naming style matching correct variable names. 215 | variable-naming-style=snake_case 216 | 217 | # Regular expression matching correct variable names. Overrides variable- 218 | # naming-style. 219 | #variable-rgx= 220 | 221 | 222 | [FORMAT] 223 | 224 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 225 | expected-line-ending-format= 226 | 227 | # Regexp for a line that is allowed to be longer than the limit. 228 | ignore-long-lines=^\s*(# )?.*['"]?? 229 | 230 | # Number of spaces of indent required inside a hanging or continued line. 231 | indent-after-paren=4 232 | 233 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 234 | # tab). 235 | indent-string=' ' 236 | 237 | # Maximum number of characters on a single line. 238 | max-line-length=120 239 | 240 | # Maximum number of lines in a module. 241 | max-module-lines=1000 242 | 243 | # List of optional constructs for which whitespace checking is disabled. `dict- 244 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 245 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 246 | # `empty-line` allows space-only lines. 247 | no-space-check=trailing-comma, 248 | dict-separator 249 | 250 | # Allow the body of a class to be on the same line as the declaration if body 251 | # contains single statement. 252 | single-line-class-stmt=no 253 | 254 | # Allow the body of an if to be on the same line as the test if there is no 255 | # else. 256 | single-line-if-stmt=no 257 | 258 | 259 | [LOGGING] 260 | 261 | # Format style used to check logging format string. `old` means using % 262 | # formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. 263 | logging-format-style=old 264 | 265 | # Logging modules to check that the string format arguments are in logging 266 | # function parameter format. 267 | logging-modules=logging 268 | 269 | 270 | [MISCELLANEOUS] 271 | 272 | # List of note tags to take in consideration, separated by a comma. 273 | notes=FIXME, 274 | XXX, 275 | TODO 276 | 277 | 278 | [SIMILARITIES] 279 | 280 | # Ignore comments when computing similarities. 281 | ignore-comments=yes 282 | 283 | # Ignore docstrings when computing similarities. 284 | ignore-docstrings=yes 285 | 286 | # Ignore imports when computing similarities. 287 | ignore-imports=yes 288 | 289 | # Minimum lines number of a similarity. 290 | min-similarity-lines=9 291 | 292 | 293 | [SPELLING] 294 | 295 | # Limits count of emitted suggestions for spelling mistakes. 296 | max-spelling-suggestions=4 297 | 298 | # Spelling dictionary name. Available dictionaries: none. To make it work, 299 | # install the python-enchant package. 300 | spelling-dict= 301 | 302 | # List of comma separated words that should not be checked. 303 | spelling-ignore-words= 304 | 305 | # A path to a file that contains the private dictionary; one word per line. 306 | spelling-private-dict-file= 307 | 308 | # Tells whether to store unknown words to the private dictionary (see the 309 | # --spelling-private-dict-file option) instead of raising a message. 310 | spelling-store-unknown-words=no 311 | 312 | 313 | [STRING] 314 | 315 | # This flag controls whether the implicit-str-concat-in-sequence should 316 | # generate a warning on implicit string concatenation in sequences defined over 317 | # several lines. 318 | check-str-concat-over-line-jumps=no 319 | 320 | 321 | [TYPECHECK] 322 | 323 | # List of decorators that produce context managers, such as 324 | # contextlib.contextmanager. Add to this list to register other decorators that 325 | # produce valid context managers. 326 | contextmanager-decorators=contextlib.contextmanager 327 | 328 | # List of members which are set dynamically and missed by pylint inference 329 | # system, and so shouldn't trigger E1101 when accessed. Python regular 330 | # expressions are accepted. 331 | generated-members=numpy.*,np.*,pyspark.sql.functions,collect_list 332 | 333 | # Tells whether missing members accessed in mixin class should be ignored. A 334 | # mixin class is detected if its name ends with "mixin" (case insensitive). 335 | ignore-mixin-members=yes 336 | 337 | # Tells whether to warn about missing members when the owner of the attribute 338 | # is inferred to be None. 339 | ignore-none=yes 340 | 341 | # This flag controls whether pylint should warn about no-member and similar 342 | # checks whenever an opaque object is returned when inferring. The inference 343 | # can return multiple potential results while evaluating a Python object, but 344 | # some branches might not be evaluated, which results in partial inference. In 345 | # that case, it might be useful to still emit no-member and other checks for 346 | # the rest of the inferred objects. 347 | ignore-on-opaque-inference=yes 348 | 349 | # List of class names for which member attributes should not be checked (useful 350 | # for classes with dynamically set attributes). This supports the use of 351 | # qualified names. 352 | ignored-classes=optparse.Values,thread._local,_thread._local,numpy,torch,swagger_client 353 | 354 | # List of module names for which member attributes should not be checked 355 | # (useful for modules/projects where namespaces are manipulated during runtime 356 | # and thus existing member attributes cannot be deduced by static analysis). It 357 | # supports qualified module names, as well as Unix pattern matching. 358 | ignored-modules=numpy,torch,swagger_client,netCDF4,scipy 359 | 360 | # Show a hint with possible names when a member name was not found. The aspect 361 | # of finding the hint is based on edit distance. 362 | missing-member-hint=yes 363 | 364 | # The minimum edit distance a name should have in order to be considered a 365 | # similar match for a missing member name. 366 | missing-member-hint-distance=1 367 | 368 | # The total number of similar names that should be taken in consideration when 369 | # showing a hint for a missing member. 370 | missing-member-max-choices=1 371 | 372 | # List of decorators that change the signature of a decorated function. 373 | signature-mutators= 374 | 375 | 376 | [VARIABLES] 377 | 378 | # List of additional names supposed to be defined in builtins. Remember that 379 | # you should avoid defining new builtins when possible. 380 | additional-builtins=dbutils 381 | 382 | # Tells whether unused global variables should be treated as a violation. 383 | allow-global-unused-variables=yes 384 | 385 | # List of strings which can identify a callback function by name. A callback 386 | # name must start or end with one of those strings. 387 | callbacks=cb_, 388 | _cb 389 | 390 | # A regular expression matching the name of dummy variables (i.e. expected to 391 | # not be used). 392 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 393 | 394 | # Argument names that match this expression will be ignored. Default to name 395 | # with leading underscore. 396 | ignored-argument-names=_.*|^ignored_|^unused_ 397 | 398 | # Tells whether we should check for unused import in __init__ files. 399 | init-import=no 400 | 401 | # List of qualified module names which can have objects that can redefine 402 | # builtins. 403 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 404 | 405 | 406 | [CLASSES] 407 | 408 | # List of method names used to declare (i.e. assign) instance attributes. 409 | defining-attr-methods=__init__, 410 | __new__, 411 | setUp, 412 | __post_init__ 413 | 414 | # List of member names, which should be excluded from the protected access 415 | # warning. 416 | exclude-protected=_asdict, 417 | _fields, 418 | _replace, 419 | _source, 420 | _make 421 | 422 | # List of valid names for the first argument in a class method. 423 | valid-classmethod-first-arg=cls 424 | 425 | # List of valid names for the first argument in a metaclass class method. 426 | valid-metaclass-classmethod-first-arg=cls 427 | 428 | 429 | [DESIGN] 430 | 431 | # Maximum number of arguments for function / method. 432 | max-args=5 433 | 434 | # Maximum number of attributes for a class (see R0902). 435 | max-attributes=7 436 | 437 | # Maximum number of boolean expressions in an if statement (see R0916). 438 | max-bool-expr=5 439 | 440 | # Maximum number of branch for function / method body. 441 | max-branches=12 442 | 443 | # Maximum number of locals for function / method body. 444 | max-locals=15 445 | 446 | # Maximum number of parents for a class (see R0901). 447 | max-parents=7 448 | 449 | # Maximum number of public methods for a class (see R0904). 450 | max-public-methods=20 451 | 452 | # Maximum number of return / yield for function / method body. 453 | max-returns=6 454 | 455 | # Maximum number of statements in function / method body. 456 | max-statements=50 457 | 458 | # Minimum number of public methods for a class (see R0903). 459 | min-public-methods=2 460 | 461 | 462 | [IMPORTS] 463 | 464 | # List of modules that can be imported at any level, not just the top level 465 | # one. 466 | allow-any-import-level= 467 | 468 | # Allow wildcard imports from modules that define __all__. 469 | allow-wildcard-with-all=no 470 | 471 | # Analyse import fallback blocks. This can be used to support both Python 2 and 472 | # 3 compatible code, which means that the block might have code that exists 473 | # only in one or another interpreter, leading to false positives when analysed. 474 | analyse-fallback-blocks=no 475 | 476 | # Deprecated modules which should not be used, separated by a comma. 477 | deprecated-modules=optparse,tkinter.tix 478 | 479 | # Create a graph of external dependencies in the given file (report RP0402 must 480 | # not be disabled). 481 | ext-import-graph= 482 | 483 | # Create a graph of every (i.e. internal and external) dependencies in the 484 | # given file (report RP0402 must not be disabled). 485 | import-graph= 486 | 487 | # Create a graph of internal dependencies in the given file (report RP0402 must 488 | # not be disabled). 489 | int-import-graph= 490 | 491 | # Force import order to recognize a module as part of the standard 492 | # compatibility libraries. 493 | known-standard-library= 494 | 495 | # Force import order to recognize a module as part of a third party library. 496 | known-third-party=enchant, azureiai-logistics-inventoryplanning 497 | 498 | # Couples of modules and preferred modules, separated by a comma. 499 | preferred-modules= 500 | 501 | 502 | [EXCEPTIONS] 503 | 504 | # Exceptions that will emit a warning when being caught. Defaults to 505 | # "BaseException, Exception". 506 | overgeneral-exceptions=BaseException, 507 | Exception 508 | --------------------------------------------------------------------------------